From 7989d1645f838e7badaae4294ae471ff5533268c Mon Sep 17 00:00:00 2001 From: elsid Date: Mon, 6 Jun 2022 01:10:33 +0200 Subject: [PATCH] Run integration tests in CI --- .gitlab-ci.yml | 20 ++++++++- CI/before_script.linux.sh | 1 - CI/install_debian_deps.sh | 33 +++++++++++++++ CI/run_integration_tests.sh | 10 +++++ .../integration_tests/test_lua_api/player.lua | 41 +++++++++++-------- .../integration_tests/test_lua_api/test.lua | 26 +++++++++--- .../test_lua_api/testing_util.lua | 6 +++ scripts/integration_tests.py | 40 +++++++++++++++--- scripts/osg_stats.py | 2 +- 9 files changed, 150 insertions(+), 29 deletions(-) create mode 100755 CI/run_integration_tests.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d2a1c1423b..67898dc0af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ default: # See https://docs.gitlab.com/ee/ci/yaml/#needs stages: - build + - test # https://blog.nimbleways.com/let-s-make-faster-gitlab-ci-cd-pipelines/ variables: @@ -21,7 +22,6 @@ variables: image: ubuntu:focal rules: - if: $CI_PIPELINE_SOURCE == "push" - .Ubuntu: extends: .Ubuntu_Image @@ -300,6 +300,24 @@ Ubuntu_Clang_tests_Debug: reports: junit: build/tests.xml +Ubuntu_Clang_integration_tests: + extends: .Ubuntu_Image + stage: test + needs: + - Ubuntu_Clang + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + cache: + key: Ubuntu_Clang_integration_tests.v1 + paths: + - .cache/pip + - apt-cache/ + before_script: + - CI/install_debian_deps.sh openmw-integration-tests + - pip3 install --user numpy matplotlib termtables click + script: + - CI/run_integration_tests.sh + .MacOS: image: macos-11-xcode-12 tags: diff --git a/CI/before_script.linux.sh b/CI/before_script.linux.sh index 8b0918827f..cae35dff68 100755 --- a/CI/before_script.linux.sh +++ b/CI/before_script.linux.sh @@ -28,7 +28,6 @@ declare -a CMAKE_CONF_OPTS=( -DBUILD_SHARED_LIBS=OFF -DUSE_SYSTEM_TINYXML=ON -DOPENMW_USE_SYSTEM_RECASTNAVIGATION=ON - -DCMAKE_INSTALL_PREFIX=install -DOPENMW_CXX_FLAGS="-Werror -Werror=implicit-fallthrough" # flags specific to OpenMW project ) diff --git a/CI/install_debian_deps.sh b/CI/install_debian_deps.sh index c089558208..8342d89722 100755 --- a/CI/install_debian_deps.sh +++ b/CI/install_debian_deps.sh @@ -42,6 +42,39 @@ declare -rA GROUPED_DEPS=( " [openmw-coverage]="gcovr" + + [openmw-integration-tests]=" + ca-certificates + git + git-lfs + libavcodec58 + libavformat58 + libavutil56 + libboost-filesystem1.71.0 + libboost-iostreams1.71.0 + libboost-program-options1.71.0 + libboost-system1.71.0 + libbullet2.88 + libcollada-dom2.4-dp0 + libicu66 + libjpeg8 + libluajit-5.1-2 + liblz4-1 + libmyguiengine3debian1v5 + libopenal1 + libopenscenegraph161 + libpng16-16 + libqt5opengl5 + librecast1 + libsdl2-2.0-0 + libsqlite3-0 + libswresample3 + libswscale5 + libtinyxml2.6.2v5 + libyaml-cpp0.6 + python3-pip + xvfb + " ) if [[ $# -eq 0 ]]; then diff --git a/CI/run_integration_tests.sh b/CI/run_integration_tests.sh new file mode 100755 index 0000000000..d7b025df52 --- /dev/null +++ b/CI/run_integration_tests.sh @@ -0,0 +1,10 @@ +#!/bin/bash -ex + +git clone --depth=1 https://gitlab.com/OpenMW/example-suite.git + +xvfb-run --auto-servernum --server-args='-screen 0 640x480x24x60' \ + scripts/integration_tests.py --omw build/install/bin/openmw --workdir integration_tests_output example-suite/ + +ls integration_tests_output/*.osg_stats.log | while read v; do + scripts/osg_stats.py --stats '.*' --regexp_match < "${v}" +done diff --git a/scripts/data/integration_tests/test_lua_api/player.lua b/scripts/data/integration_tests/test_lua_api/player.lua index c46ab5c46e..63e88b0556 100644 --- a/scripts/data/integration_tests/test_lua_api/player.lua +++ b/scripts/data/integration_tests/test_lua_api/player.lua @@ -12,23 +12,25 @@ input.setControlSwitch(input.CONTROL_SWITCH.Magic, false) input.setControlSwitch(input.CONTROL_SWITCH.VanityMode, false) input.setControlSwitch(input.CONTROL_SWITCH.ViewMode, false) -testing.registerLocalTest('playerMovement', +testing.registerLocalTest('playerRotation', function() - local startTime = core.getSimulationTime() - local pos = self.position - - while core.getSimulationTime() < startTime + 0.5 do + local endTime = core.getSimulationTime() + 1 + while core.getSimulationTime() < endTime do self.controls.jump = false self.controls.run = true self.controls.movement = 0 self.controls.sideMovement = 0 - local progress = (core.getSimulationTime() - startTime) / 0.5 - self.controls.yawChange = util.normalizeAngle(math.rad(90) * progress - self.rotation.z) + self.controls.yawChange = util.normalizeAngle(math.rad(90) - self.rotation.z) * 0.5 coroutine.yield() end testing.expectEqualWithDelta(self.rotation.z, math.rad(90), 0.05, 'Incorrect rotation') + end) - while core.getSimulationTime() < startTime + 1.5 do +testing.registerLocalTest('playerForwardRunning', + function() + local startPos = self.position + local endTime = core.getSimulationTime() + 1 + while core.getSimulationTime() < endTime do self.controls.jump = false self.controls.run = true self.controls.movement = 1 @@ -36,12 +38,18 @@ testing.registerLocalTest('playerMovement', self.controls.yawChange = 0 coroutine.yield() end - direction = (self.position - pos) / types.Actor.runSpeed(self) - testing.expectEqualWithDelta(direction.x, 1, 0.1, 'Run forward, X coord') - testing.expectEqualWithDelta(direction.y, 0, 0.1, 'Run forward, Y coord') + local direction, distance = (self.position - startPos):normalize() + local normalizedDistance = distance / types.Actor.runSpeed(self) + testing.expectEqualWithDelta(normalizedDistance, 1, 0.2, 'Normalized forward runned distance') + testing.expectEqualWithDelta(direction.x, 0, 0.1, 'Run forward, X coord') + testing.expectEqualWithDelta(direction.y, 1, 0.1, 'Run forward, Y coord') + end) - pos = self.position - while core.getSimulationTime() < startTime + 2.5 do +testing.registerLocalTest('playerDiagonalWalking', + function() + local startPos = self.position + local endTime = core.getSimulationTime() + 1 + while core.getSimulationTime() < endTime do self.controls.jump = false self.controls.run = false self.controls.movement = -1 @@ -49,9 +57,11 @@ testing.registerLocalTest('playerMovement', self.controls.yawChange = 0 coroutine.yield() end - direction = (self.position - pos) / types.Actor.walkSpeed(self) + local direction, distance = (self.position - startPos):normalize() + local normalizedDistance = distance / types.Actor.walkSpeed(self) + testing.expectEqualWithDelta(normalizedDistance, 1, 0.2, 'Normalized diagonally walked distance') testing.expectEqualWithDelta(direction.x, -0.707, 0.1, 'Walk diagonally, X coord') - testing.expectEqualWithDelta(direction.y, 0.707, 0.1, 'Walk diagonally, Y coord') + testing.expectEqualWithDelta(direction.y, -0.707, 0.1, 'Walk diagonally, Y coord') end) return { @@ -60,4 +70,3 @@ return { }, eventHandlers = testing.eventHandlers } - diff --git a/scripts/data/integration_tests/test_lua_api/test.lua b/scripts/data/integration_tests/test_lua_api/test.lua index 573844199b..ba0d45ab23 100644 --- a/scripts/data/integration_tests/test_lua_api/test.lua +++ b/scripts/data/integration_tests/test_lua_api/test.lua @@ -29,10 +29,10 @@ local function testTimers() while not (ts1 and ts2 and th1 and th2) do coroutine.yield() end - testing.expectAlmostEqual(th1, 36, 'async:newGameTimer failed') - testing.expectAlmostEqual(ts1, 0.5, 'async:newSimulationTimer failed') - testing.expectAlmostEqual(th2, 72, 'async:newUnsavableGameTimer failed') - testing.expectAlmostEqual(ts2, 1, 'async:newUnsavableSimulationTimer failed') + testing.expectGreaterOrEqual(th1, 36, 'async:newGameTimer failed') + testing.expectGreaterOrEqual(ts1, 0.5, 'async:newSimulationTimer failed') + testing.expectGreaterOrEqual(th2, 72, 'async:newUnsavableGameTimer failed') + testing.expectGreaterOrEqual(ts2, 1, 'async:newUnsavableSimulationTimer failed') end local function testTeleport() @@ -51,9 +51,25 @@ local function testTeleport() testing.expectEqualWithDelta(player.rotation.z, math.rad(-90), 0.05, 'teleporting changes rotation') end +local function initPlayer() + player:teleport('', util.vector3(4096, 4096, 867.237), util.vector3(0, 0, 0)) + coroutine.yield() +end + tests = { {'timers', testTimers}, - {'playerMovement', function() testing.runLocalTest(player, 'playerMovement') end}, + {'playerRotation', function() + initPlayer() + testing.runLocalTest(player, 'playerRotation') + end}, + {'playerForwardRunning', function() + initPlayer() + testing.runLocalTest(player, 'playerForwardRunning') + end}, + {'playerDiagonalWalking', function() + initPlayer() + testing.runLocalTest(player, 'playerDiagonalWalking') + end}, {'teleport', testTeleport}, } diff --git a/scripts/data/integration_tests/test_lua_api/testing_util.lua b/scripts/data/integration_tests/test_lua_api/testing_util.lua index 719502e741..5ce5b2ff26 100644 --- a/scripts/data/integration_tests/test_lua_api/testing_util.lua +++ b/scripts/data/integration_tests/test_lua_api/testing_util.lua @@ -52,6 +52,12 @@ function M.expectAlmostEqual(v1, v2, msg) end end +function M.expectGreaterOrEqual(v1, v2, msg) + if not (v1 >= v2) then + error(string.format('%s: %f >= %f', msg or '', v1, v2), 2) + end +end + local localTests = {} local localTestRunner = nil diff --git a/scripts/integration_tests.py b/scripts/integration_tests.py index b125e688fb..d106c71283 100755 --- a/scripts/integration_tests.py +++ b/scripts/integration_tests.py @@ -51,14 +51,30 @@ def runTest(name): ) if (test_dir / "test.omwscripts").exists(): omw_cfg.write("content=test.omwscripts\n") + with open(config_dir / "settings.cfg", "a", encoding="utf-8") as settings_cfg: + settings_cfg.write( + "[Video]\n" + "resolution x = 640\n" + "resolution y = 480\n" + "framerate limit = 60\n" + ) + stdout_lines = list() + exit_ok = True + test_success = True with subprocess.Popen( - [f"{openmw_binary}", "--replace=config", f"--config={config_dir}", "--skip-menu", "--no-grab"], + [openmw_binary, "--replace=config", "--config", config_dir, "--skip-menu", "--no-grab", "--no-sound"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", + env={ + "OPENMW_OSG_STATS_FILE": work_dir / f"{name}.{time_str}.osg_stats.log", + "OPENMW_OSG_STATS_LIST": "times", + **os.environ, + }, ) as process: quit_requested = False for line in process.stdout: + stdout_lines.append(line) words = line.split(" ") if len(words) > 1 and words[1] == "E]": print(line, end="") @@ -72,16 +88,30 @@ def runTest(name): elif "TEST_FAILED" in line: w = line.split("TEST_FAILED")[1].split("\t") print(f"FAILED {w[3]}\t\t") + test_success = False process.wait(5) if not quit_requested: print("ERROR: Unexpected termination") - shutil.copyfile(config_dir / "openmw.log", work_dir / f"{name}.{time_str}.log") - print(f"{name} finished") + exit_ok = False + if process.returncode != 0: + print(f"ERROR: openmw exited with code {process.returncode}") + exit_ok = False + if os.path.exists(config_dir / "openmw.log"): + shutil.copyfile(config_dir / "openmw.log", work_dir / f"{name}.{time_str}.log") + if not exit_ok: + sys.stdout.writelines(stdout_lines) + if test_success and exit_ok: + print(f"{name} succeeded") + else: + print(f"{name} failed") + return test_success and exit_ok +status = 0 for entry in tests_dir.glob("test_*"): if entry.is_dir(): - runTest(entry.name) + if not runTest(entry.name): + status = -1 shutil.rmtree(config_dir, ignore_errors=True) shutil.rmtree(userdata_dir, ignore_errors=True) - +exit(status) diff --git a/scripts/osg_stats.py b/scripts/osg_stats.py index cc4314cd04..3a5067851d 100755 --- a/scripts/osg_stats.py +++ b/scripts/osg_stats.py @@ -76,7 +76,7 @@ def main(print_keys, regexp_match, timeseries, hist, hist_ratio, stdev_hist, plo def matching_keys(patterns): if regexp_match: return [key for pattern in patterns for key in keys if re.search(pattern, key)] - return keys + return patterns if timeseries: draw_timeseries(sources=frames, keys=matching_keys(timeseries), add_sum=timeseries_sum, begin_frame=begin_frame, end_frame=end_frame)