diff --git a/CI/run_integration_tests.sh b/CI/run_integration_tests.sh index e79408926a..6cc3bd55bf 100755 --- a/CI/run_integration_tests.sh +++ b/CI/run_integration_tests.sh @@ -9,7 +9,7 @@ git checkout FETCH_HEAD cd .. xvfb-run --auto-servernum --server-args='-screen 0 640x480x24x60' \ - scripts/integration_tests.py --omw build/install/bin/openmw --workdir integration_tests_output example-suite/ + scripts/integration_tests.py --verbose --omw build/install/bin/openmw --workdir integration_tests_output example-suite/ ls integration_tests_output/*.osg_stats.log | while read v; do scripts/osg_stats.py --stats '.*' --regexp_match < "${v}" diff --git a/scripts/data/integration_tests/test_lua_api/test.lua b/scripts/data/integration_tests/test_lua_api/global.lua similarity index 79% rename from scripts/data/integration_tests/test_lua_api/test.lua rename to scripts/data/integration_tests/test_lua_api/global.lua index ff2cd9bb33..cc8240554a 100644 --- a/scripts/data/integration_tests/test_lua_api/test.lua +++ b/scripts/data/integration_tests/test_lua_api/global.lua @@ -7,7 +7,7 @@ local vfs = require('openmw.vfs') local world = require('openmw.world') local I = require('openmw.interfaces') -local function testTimers() +testing.registerGlobalTest('timers', function() testing.expectAlmostEqual(core.getGameTimeScale(), 30, 'incorrect getGameTimeScale() result') testing.expectAlmostEqual(core.getSimulationTimeScale(), 1, 'incorrect getSimulationTimeScale result') @@ -39,9 +39,10 @@ local function testTimers() testing.expectGreaterOrEqual(ts1, 0.5, 'async:newSimulationTimer failed') testing.expectGreaterOrEqual(th2, 72, 'async:newUnsavableGameTimer failed') testing.expectGreaterOrEqual(ts2, 1, 'async:newUnsavableSimulationTimer failed') -end +end) -local function testTeleport() +testing.registerGlobalTest('teleport', function() + local player = world.players[1] player:teleport('', util.vector3(100, 50, 500), util.transform.rotateZ(math.rad(90))) coroutine.yield() testing.expect(player.cell.isExterior, 'teleport to exterior failed') @@ -71,16 +72,16 @@ local function testTeleport() testing.expectEqualWithDelta(player.position.x, 50, 1, 'incorrect position after teleporting') testing.expectEqualWithDelta(player.position.y, -100, 1, 'incorrect position after teleporting') testing.expectEqualWithDelta(player.rotation:getYaw(), math.rad(-90), 0.05, 'teleporting changes rotation') -end +end) -local function testGetGMST() +testing.registerGlobalTest('getGMST', function() testing.expectEqual(core.getGMST('non-existed gmst'), nil) testing.expectEqual(core.getGMST('Water_RippleFrameCount'), 4) testing.expectEqual(core.getGMST('Inventory_DirectionalDiffuseR'), 0.5) testing.expectEqual(core.getGMST('Level_Up_Level2'), 'something') -end +end) -local function testMWScript() +testing.registerGlobalTest('MWScript', function() local variableStoreCount = 18 local variableStore = world.mwscript.getGlobalVariables(player) testing.expectEqual(variableStoreCount, #variableStore) @@ -100,7 +101,7 @@ local function testMWScript() indexCheck = indexCheck + 1 end testing.expectEqual(variableStoreCount, indexCheck) -end +end) local function testRecordStore(store, storeName, skipPairs) testing.expect(store.records) @@ -121,7 +122,7 @@ local function testRecordStore(store, storeName, skipPairs) testing.expectEqual(status, true, storeName) end -local function testRecordStores() +testing.registerGlobalTest('record stores', function() for key, type in pairs(types) do if type.records then testRecordStore(type, key) @@ -140,9 +141,9 @@ local function testRecordStores() testRecordStore(types.NPC.classes, "classes") testRecordStore(types.NPC.races, "races") testRecordStore(types.Player.birthSigns, "birthSigns") -end +end) -local function testRecordCreation() +testing.registerGlobalTest('record creation', function() local newLight = { isCarriable = true, isDynamic = true, @@ -165,9 +166,9 @@ local function testRecordCreation() for key, value in pairs(newLight) do testing.expectEqual(record[key], value) end -end +end) -local function testUTF8Chars() +testing.registerGlobalTest('UTF-8 characters', function() testing.expectEqual(utf8.codepoint("😀"), 0x1F600) local chars = {} @@ -192,9 +193,9 @@ local function testUTF8Chars() testing.expectEqual(utf8.codepoint(char), codepoint) testing.expectEqual(utf8.len(char), 1) end -end +end) -local function testUTF8Strings() +testing.registerGlobalTest('UTF-8 strings', function() local utf8str = "Hello, 你好, 🌎!" local str = "" @@ -205,9 +206,9 @@ local function testUTF8Strings() testing.expectEqual(utf8.len(utf8str), 13) testing.expectEqual(utf8.offset(utf8str, 9), 11) -end +end) -local function testMemoryLimit() +testing.registerGlobalTest('memory limit', function() local ok, err = pcall(function() local t = {} local n = 1 @@ -218,14 +219,16 @@ local function testMemoryLimit() end) testing.expectEqual(ok, false, 'Script reaching memory limit should fail') testing.expectEqual(err, 'not enough memory') -end +end) local function initPlayer() + local player = world.players[1] player:teleport('', util.vector3(4096, 4096, 1745), util.transform.identity) coroutine.yield() + return player end -local function testVFS() +testing.registerGlobalTest('vfs', function() local file = 'test_vfs_dir/lines.txt' local nosuchfile = 'test_vfs_dir/nosuchfile' testing.expectEqual(vfs.fileExists(file), true, 'lines.txt should exist') @@ -269,12 +272,11 @@ local function testVFS() for _,v in pairs(expectedLines) do testing.expectEqual(getLine(), v) end -end +end) -local function testCommitCrime() - initPlayer() - local player = world.players[1] - testing.expectEqual(player == nil, false, 'A viable player reference should exist to run `testCommitCrime`') +testing.registerGlobalTest('commit crime', function() + local player = initPlayer() + testing.expectEqual(player == nil, false, 'A viable player reference should exist to run `commit crime`') testing.expectEqual(I.Crimes == nil, false, 'Crimes interface should be available in global contexts') -- Reset crime level to have a clean slate @@ -292,82 +294,41 @@ local function testCommitCrime() types.Player.setCrimeLevel(player, 0) testing.expectEqual(I.Crimes.commitCrime(player, { victim = victim, type = types.Player.OFFENSE_TYPE.Theft, arg = 50 }).wasCrimeSeen, true, "Running a crime with a valid victim should notify them when the player is not sneaking, even if it's not explicitly passed in") testing.expectEqual(types.Player.getCrimeLevel(player), 0, "Crime level should not change if the victim's alarm value is low and there's no other witnesses") -end +end) -local function testRecordModelProperty() - initPlayer() +testing.registerGlobalTest('record model property', function() local player = world.players[1] testing.expectEqual(types.NPC.record(player).model, 'meshes/basicplayer.dae') +end) + +local function registerPlayerTest(name) + testing.registerGlobalTest(name, function() + local player = initPlayer() + testing.runLocalTest(player, name) + end) end -tests = { - {'timers', testTimers}, - {'rotating player with controls.yawChange should change rotation', function() - initPlayer() - testing.runLocalTest(player, 'playerYawRotation') - end}, - {'rotating player with controls.pitchChange should change rotation', function() - initPlayer() - testing.runLocalTest(player, 'playerPitchRotation') - end}, - {'rotating player with controls.pitchChange and controls.yawChange should change rotation', function() - initPlayer() - testing.runLocalTest(player, 'playerPitchAndYawRotation') - end}, - {'rotating player should not lead to nan rotation', function() - initPlayer() - testing.runLocalTest(player, 'playerRotation') - end}, - {'playerForwardRunning', function() - initPlayer() - testing.runLocalTest(player, 'playerForwardRunning') - end}, - {'playerDiagonalWalking', function() - initPlayer() - testing.runLocalTest(player, 'playerDiagonalWalking') - end}, - {'findPath', function() - initPlayer() - testing.runLocalTest(player, 'findPath') - end}, - {'findRandomPointAroundCircle', function() - initPlayer() - testing.runLocalTest(player, 'findRandomPointAroundCircle') - end}, - {'castNavigationRay', function() - initPlayer() - testing.runLocalTest(player, 'castNavigationRay') - end}, - {'findNearestNavMeshPosition', function() - initPlayer() - testing.runLocalTest(player, 'findNearestNavMeshPosition') - end}, - {'teleport', testTeleport}, - {'getGMST', testGetGMST}, - {'recordStores', testRecordStores}, - {'recordCreation', testRecordCreation}, - {'utf8Chars', testUTF8Chars}, - {'utf8Strings', testUTF8Strings}, - {'mwscript', testMWScript}, - {'testMemoryLimit', testMemoryLimit}, - {'playerMemoryLimit', function() - initPlayer() - testing.runLocalTest(player, 'playerMemoryLimit') - end}, - {'player with equipped weapon on attack should damage health of other actors', function() - initPlayer() - world.createObject('basic_dagger1h', 1):moveInto(player) - testing.runLocalTest(player, 'playerWeaponAttack') - end}, - {'vfs', testVFS}, - {'testCommitCrime', testCommitCrime}, - {'recordModelProperty', testRecordModelProperty}, -} +registerPlayerTest('player yaw rotation') +registerPlayerTest('player pitch rotation') +registerPlayerTest('player pitch and yaw rotation') +registerPlayerTest('player rotation') +registerPlayerTest('player forward running') +registerPlayerTest('player diagonal walking') +registerPlayerTest('findPath') +registerPlayerTest('findRandomPointAroundCircle') +registerPlayerTest('castNavigationRay') +registerPlayerTest('findNearestNavMeshPosition') +registerPlayerTest('player memory limit') + +testing.registerGlobalTest('player weapon attack', function() + local player = initPlayer() + world.createObject('basic_dagger1h', 1):moveInto(player) + testing.runLocalTest(player, 'player weapon attack') +end) return { engineHandlers = { - onUpdate = testing.testRunner(tests), - onPlayerAdded = function(p) player = p end, + onUpdate = testing.updateGlobal, }, - eventHandlers = testing.eventHandlers, + eventHandlers = testing.globalEventHandlers, } diff --git a/scripts/data/integration_tests/test_lua_api/menu.lua b/scripts/data/integration_tests/test_lua_api/menu.lua new file mode 100644 index 0000000000..37f7cb826e --- /dev/null +++ b/scripts/data/integration_tests/test_lua_api/menu.lua @@ -0,0 +1,43 @@ +local testing = require('testing_util') +local menu = require('openmw.menu') + +local function registerGlobalTest(name, description) + testing.registerMenuTest(description or name, function() + menu.newGame() + coroutine.yield() + testing.runGlobalTest(name) + end) +end + +registerGlobalTest('timers') +registerGlobalTest('teleport') +registerGlobalTest('getGMST') +registerGlobalTest('MWScript') +registerGlobalTest('record stores') +registerGlobalTest('record creation') +registerGlobalTest('UTF-8 characters') +registerGlobalTest('UTF-8 strings') +registerGlobalTest('memory limit') +registerGlobalTest('vfs') +registerGlobalTest('commit crime') +registerGlobalTest('record model property') + +registerGlobalTest('player yaw rotation', 'rotating player with controls.yawChange should change rotation') +registerGlobalTest('player pitch rotation', 'rotating player with controls.pitchChange should change rotation') +registerGlobalTest('player pitch and yaw rotation', 'rotating player with controls.pitchChange and controls.yawChange should change rotation') +registerGlobalTest('player rotation', 'rotating player should not lead to nan rotation') +registerGlobalTest('player forward running') +registerGlobalTest('player diagonal walking') +registerGlobalTest('findPath') +registerGlobalTest('findRandomPointAroundCircle') +registerGlobalTest('castNavigationRay') +registerGlobalTest('findNearestNavMeshPosition') +registerGlobalTest('player memory limit') +registerGlobalTest('player weapon attack', 'player with equipped weapon on attack should damage health of other actors') + +return { + engineHandlers = { + onFrame = testing.makeUpdateMenu(), + }, + eventHandlers = testing.menuEventHandlers, +} diff --git a/scripts/data/integration_tests/test_lua_api/openmw.cfg b/scripts/data/integration_tests/test_lua_api/openmw.cfg index 68e50644a0..efd1cf331c 100644 --- a/scripts/data/integration_tests/test_lua_api/openmw.cfg +++ b/scripts/data/integration_tests/test_lua_api/openmw.cfg @@ -1,4 +1,4 @@ -content=test.omwscripts +content=test_lua_api.omwscripts # Needed to test `core.getGMST` fallback=Water_RippleFrameCount,4 diff --git a/scripts/data/integration_tests/test_lua_api/player.lua b/scripts/data/integration_tests/test_lua_api/player.lua index 16f0f2eea3..74769da8e0 100644 --- a/scripts/data/integration_tests/test_lua_api/player.lua +++ b/scripts/data/integration_tests/test_lua_api/player.lua @@ -40,7 +40,7 @@ local function rotateByPitch(object, target) rotate(object, target, nil) end -testing.registerLocalTest('playerYawRotation', +testing.registerLocalTest('player yaw rotation', function() local initialAlphaXZ, initialGammaXZ = self.rotation:getAnglesXZ() local initialAlphaZYX, initialBetaZYX, initialGammaZYX = self.rotation:getAnglesZYX() @@ -60,7 +60,7 @@ testing.registerLocalTest('playerYawRotation', testing.expectEqualWithDelta(gamma2, initialGammaZYX, 0.05, 'Gamma rotation in ZYX convention should not change') end) -testing.registerLocalTest('playerPitchRotation', +testing.registerLocalTest('player pitch rotation', function() local initialAlphaXZ, initialGammaXZ = self.rotation:getAnglesXZ() local initialAlphaZYX, initialBetaZYX, initialGammaZYX = self.rotation:getAnglesZYX() @@ -80,7 +80,7 @@ testing.registerLocalTest('playerPitchRotation', testing.expectEqualWithDelta(gamma2, targetPitch, 0.05, 'Incorrect gamma rotation in ZYX convention') end) -testing.registerLocalTest('playerPitchAndYawRotation', +testing.registerLocalTest('player pitch and yaw rotation', function() local targetPitch = math.rad(-30) local targetYaw = math.rad(-60) @@ -99,7 +99,7 @@ testing.registerLocalTest('playerPitchAndYawRotation', testing.expectEqualWithDelta(gamma2, math.rad(-16), 0.05, 'Incorrect gamma rotation in ZYX convention') end) -testing.registerLocalTest('playerRotation', +testing.registerLocalTest('player rotation', function() local rotation = math.sqrt(2) local endTime = core.getSimulationTime() + 3 @@ -123,7 +123,7 @@ testing.registerLocalTest('playerRotation', end end) -testing.registerLocalTest('playerForwardRunning', +testing.registerLocalTest('player forward running', function() local startPos = self.position local endTime = core.getSimulationTime() + 1 @@ -141,7 +141,7 @@ testing.registerLocalTest('playerForwardRunning', testing.expectEqualWithDelta(direction.y, 1, 0.1, 'Run forward, Y coord') end) -testing.registerLocalTest('playerDiagonalWalking', +testing.registerLocalTest('player diagonal walking', function() local startPos = self.position local endTime = core.getSimulationTime() + 1 @@ -220,7 +220,7 @@ testing.registerLocalTest('findNearestNavMeshPosition', 'Navigation mesh position ' .. testing.formatActualExpected(result, expected)) end) -testing.registerLocalTest('playerMemoryLimit', +testing.registerLocalTest('player memory limit', function() local ok, err = pcall(function() local str = 'a' @@ -232,7 +232,7 @@ testing.registerLocalTest('playerMemoryLimit', testing.expectEqual(err, 'not enough memory') end) -testing.registerLocalTest('playerWeaponAttack', +testing.registerLocalTest('player weapon attack', function() camera.setMode(camera.MODE.ThirdPerson) @@ -346,5 +346,5 @@ return { engineHandlers = { onFrame = testing.updateLocal, }, - eventHandlers = testing.eventHandlers + eventHandlers = testing.localEventHandlers, } diff --git a/scripts/data/integration_tests/test_lua_api/test.omwscripts b/scripts/data/integration_tests/test_lua_api/test.omwscripts deleted file mode 100644 index 80507392f7..0000000000 --- a/scripts/data/integration_tests/test_lua_api/test.omwscripts +++ /dev/null @@ -1,2 +0,0 @@ -GLOBAL: test.lua -PLAYER: player.lua diff --git a/scripts/data/integration_tests/test_lua_api/test_lua_api.omwscripts b/scripts/data/integration_tests/test_lua_api/test_lua_api.omwscripts new file mode 100644 index 0000000000..4ce925e61d --- /dev/null +++ b/scripts/data/integration_tests/test_lua_api/test_lua_api.omwscripts @@ -0,0 +1,3 @@ +MENU: menu.lua +GLOBAL: global.lua +PLAYER: player.lua diff --git a/scripts/data/integration_tests/testing_util/testing_util.lua b/scripts/data/integration_tests/testing_util/testing_util.lua index 7b886636ed..5a69cb89ed 100644 --- a/scripts/data/integration_tests/testing_util/testing_util.lua +++ b/scripts/data/integration_tests/testing_util/testing_util.lua @@ -2,23 +2,22 @@ local core = require('openmw.core') local util = require('openmw.util') local M = {} + +local menuTestsOrder = {} +local menuTests = {} + +local globalTestsOrder = {} +local globalTests = {} +local globalTestRunner = nil +local currentGlobalTest = nil +local currentGlobalTestError = nil + +local localTests = {} +local localTestRunner = nil local currentLocalTest = nil local currentLocalTestError = nil -function M.testRunner(tests) - local fn = function() - for i, test in ipairs(tests) do - local name, fn = unpack(test) - print('TEST_START', i, name) - local status, err = pcall(fn) - if status then - print('TEST_OK', i, name) - else - print('TEST_FAILED', i, name, err) - end - end - core.quit() - end +local function makeTestCoroutine(fn) local co = coroutine.create(fn) return function() if coroutine.status(co) ~= 'dead' then @@ -27,6 +26,64 @@ function M.testRunner(tests) end end +local function runTests(tests) + for i, test in ipairs(tests) do + local name, fn = unpack(test) + print('TEST_START', i, name) + local status, err = pcall(fn) + if status then + print('TEST_OK', i, name) + else + print('TEST_FAILED', i, name, err) + end + end + core.quit() +end + +function M.makeUpdateMenu() + return makeTestCoroutine(function() + print('Running menu tests...') + runTests(menuTestsOrder) + end) +end + +function M.makeUpdateGlobal() + return makeTestCoroutine(function() + print('Running global tests...') + runTests(globalTestsOrder) + end) +end + +function M.registerMenuTest(name, fn) + menuTests[name] = fn + table.insert(menuTestsOrder, {name, fn}) +end + +function M.runGlobalTest(name) + currentGlobalTest = name + currentGlobalTestError = nil + core.sendGlobalEvent('runGlobalTest', name) + while currentGlobalTest do + coroutine.yield() + end + if currentGlobalTestError then + error(currentGlobalTestError, 2) + end +end + +function M.registerGlobalTest(name, fn) + globalTests[name] = fn + table.insert(globalTestsOrder, {name, fn}) +end + +function M.updateGlobal() + if globalTestRunner and coroutine.status(globalTestRunner) ~= 'dead' then + coroutine.resume(globalTestRunner) + else + globalTestRunner = nil + end +end + function M.runLocalTest(obj, name) currentLocalTest = name currentLocalTestError = nil @@ -39,7 +96,21 @@ function M.runLocalTest(obj, name) end end -function M.expect(cond, delta, msg) +function M.registerLocalTest(name, fn) + localTests[name] = fn +end + +function M.updateLocal() + if localTestRunner and coroutine.status(localTestRunner) ~= 'dead' then + if not core.isWorldPaused() then + coroutine.resume(localTestRunner) + end + else + localTestRunner = nil + end +end + +function M.expect(cond, msg) if not cond then error(msg or '"true" expected', 2) end @@ -182,28 +253,50 @@ function M.formatActualExpected(actual, expected) return string.format('actual: %s, expected: %s', actual, expected) end -local localTests = {} -local localTestRunner = nil - -function M.registerLocalTest(name, fn) - localTests[name] = fn -end - -function M.updateLocal() - if localTestRunner and coroutine.status(localTestRunner) ~= 'dead' then - if not core.isWorldPaused() then - coroutine.resume(localTestRunner) +-- used only in menu scripts +M.menuEventHandlers = { + globalTestFinished = function(data) + if data.name ~= currentGlobalTest then + error(string.format('globalTestFinished with incorrect name %s, expected %s', data.name, currentGlobalTest), 2) end - else - localTestRunner = nil - end -end + currentGlobalTest = nil + currentGlobalTestError = data.errMsg + end, +} -M.eventHandlers = { - runLocalTest = function(name) -- used only in local scripts +-- used only in global scripts +M.globalEventHandlers = { + runGlobalTest = function(name) + fn = globalTests[name] + local types = require('openmw.types') + local world = require('openmw.world') + if not fn then + types.Player.sendMenuEvent(world.players[1], 'globalTestFinished', {name=name, errMsg='Global test is not found'}) + return + end + globalTestRunner = coroutine.create(function() + local status, err = pcall(fn) + if status then + err = nil + end + types.Player.sendMenuEvent(world.players[1], 'globalTestFinished', {name=name, errMsg=err}) + end) + end, + localTestFinished = function(data) + if data.name ~= currentLocalTest then + error(string.format('localTestFinished with incorrect name %s, expected %s', data.name, currentLocalTest), 2) + end + currentLocalTest = nil + currentLocalTestError = data.errMsg + end, +} + +-- used only in local scripts +M.localEventHandlers = { + runLocalTest = function(name) fn = localTests[name] if not fn then - core.sendGlobalEvent('localTestFinished', {name=name, errMsg='Test not found'}) + core.sendGlobalEvent('localTestFinished', {name=name, errMsg='Local test is not found'}) return end localTestRunner = coroutine.create(function() @@ -214,13 +307,6 @@ M.eventHandlers = { core.sendGlobalEvent('localTestFinished', {name=name, errMsg=err}) end) end, - localTestFinished = function(data) -- used only in global scripts - if data.name ~= currentLocalTest then - error(string.format('localTestFinished with incorrect name %s, expected %s', data.name, currentLocalTest)) - end - currentLocalTest = nil - currentLocalTestError = data.errMsg - end, } return M diff --git a/scripts/data/morrowind_tests/global.lua b/scripts/data/morrowind_tests/global.lua index fb7113d1b1..f17e72cda9 100644 --- a/scripts/data/morrowind_tests/global.lua +++ b/scripts/data/morrowind_tests/global.lua @@ -8,28 +8,13 @@ if not core.contentFiles.has('Morrowind.esm') then error('This test requires Morrowind.esm') end -function makeTests(modules) - local tests = {} - - for _, moduleName in ipairs(modules) do - local module = require(moduleName) - for _, v in ipairs(module) do - table.insert(tests, {string.format('[%s] %s', moduleName, v[1]), v[2]}) - end - end - - return tests -end - -local testModules = { - 'global_issues', - 'global_dialogues', - 'global_mwscript', -} +require('global_issues') +require('global_dialogues') +require('global_mwscript') return { engineHandlers = { - onUpdate = testing.testRunner(makeTests(testModules)), + onUpdate = testing.makeUpdateGlobal(), }, - eventHandlers = testing.eventHandlers, + eventHandlers = testing.globalEventHandlers, } diff --git a/scripts/data/morrowind_tests/global_dialogues.lua b/scripts/data/morrowind_tests/global_dialogues.lua index 397eb8461c..68f4f4a747 100644 --- a/scripts/data/morrowind_tests/global_dialogues.lua +++ b/scripts/data/morrowind_tests/global_dialogues.lua @@ -13,35 +13,37 @@ function iterateOverRecords(records) return firstRecordId, lastRecordId, count end -return { - {'Should support iteration over journal dialogues', function() - local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.journal.records) - testing.expectEqual(firstRecordId, '11111 test journal') - testing.expectEqual(lastRecordId, 'va_vamprich') - testing.expectEqual(count, 632) - end}, - {'Should support iteration over topic dialogues', function() - local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.topic.records) - testing.expectEqual(firstRecordId, '1000-drake pledge') - testing.expectEqual(lastRecordId, 'zenithar') - testing.expectEqual(count, 1698) - end}, - {'Should support iteration over greeting dialogues', function() - local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.greeting.records) - testing.expectEqual(firstRecordId, 'greeting 0') - testing.expectEqual(lastRecordId, 'greeting 9') - testing.expectEqual(count, 10) - end}, - {'Should support iteration over persuasion dialogues', function() - local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.persuasion.records) - testing.expectEqual(firstRecordId, 'admire fail') - testing.expectEqual(lastRecordId, 'taunt success') - testing.expectEqual(count, 10) - end}, - {'Should support iteration over voice dialogues', function() - local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.voice.records) - testing.expectEqual(firstRecordId, 'alarm') - testing.expectEqual(lastRecordId, 'thief') - testing.expectEqual(count, 8) - end}, -} +testing.registerGlobalTest('[dialogues] Should support iteration over journal dialogues', function() + local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.journal.records) + testing.expectEqual(firstRecordId, '11111 test journal') + testing.expectEqual(lastRecordId, 'va_vamprich') + testing.expectEqual(count, 632) +end) + +testing.registerGlobalTest('[dialogues] Should support iteration over topic dialogues', function() + local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.topic.records) + testing.expectEqual(firstRecordId, '1000-drake pledge') + testing.expectEqual(lastRecordId, 'zenithar') + testing.expectEqual(count, 1698) +end) + +testing.registerGlobalTest('[dialogues] Should support iteration over greeting dialogues', function() + local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.greeting.records) + testing.expectEqual(firstRecordId, 'greeting 0') + testing.expectEqual(lastRecordId, 'greeting 9') + testing.expectEqual(count, 10) +end) + +testing.registerGlobalTest('[dialogues] Should support iteration over persuasion dialogues', function() + local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.persuasion.records) + testing.expectEqual(firstRecordId, 'admire fail') + testing.expectEqual(lastRecordId, 'taunt success') + testing.expectEqual(count, 10) +end) + +testing.registerGlobalTest('[dialogues] Should support iteration over voice dialogues', function() + local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.voice.records) + testing.expectEqual(firstRecordId, 'alarm') + testing.expectEqual(lastRecordId, 'thief') + testing.expectEqual(count, 8) +end) diff --git a/scripts/data/morrowind_tests/global_issues.lua b/scripts/data/morrowind_tests/global_issues.lua index 2afad085b0..a3400da87c 100644 --- a/scripts/data/morrowind_tests/global_issues.lua +++ b/scripts/data/morrowind_tests/global_issues.lua @@ -4,37 +4,37 @@ local world = require('openmw.world') local core = require('openmw.core') local types = require('openmw.types') -return { - {'Player should be able to walk up stairs in Ebonheart docks (#4247)', function() - world.players[1]:teleport('', util.vector3(19867, -102180, -79), util.transform.rotateZ(math.rad(91))) - coroutine.yield() - testing.runLocalTest(world.players[1], 'Player should be able to walk up stairs in Ebonheart docks (#4247)') - end}, - {'Guard in Imperial Prison Ship should find path (#7241)', function() - world.players[1]:teleport('Imperial Prison Ship', util.vector3(61, -135, -105), util.transform.rotateZ(math.rad(-20))) - coroutine.yield() - testing.runLocalTest(world.players[1], 'Guard in Imperial Prison Ship should find path (#7241)') - end}, - {'Should keep reference to an object moved into container (#7663)', function() - world.players[1]:teleport('ToddTest', util.vector3(2176, 3648, -191), util.transform.rotateZ(math.rad(0))) - coroutine.yield() - local barrel = world.createObject('barrel_01', 1) - local fargothRing = world.createObject('ring_keley', 1) - coroutine.yield() - testing.expectEqual(types.Container.inventory(barrel):find('ring_keley'), nil) - fargothRing:moveInto(types.Container.inventory(barrel)) - coroutine.yield() - testing.expectEqual(fargothRing.recordId, 'ring_keley') - local isFargothRing = function(actual) - if actual == nil then - return 'ring_keley is not found' - end - if actual.id ~= fargothRing.id then - return 'found ring_keley id does not match expected: actual=' .. tostring(actual.id) - .. ', expected=' .. tostring(fargothRing.id) - end - return '' +testing.registerGlobalTest('[issues] Player should be able to walk up stairs in Ebonheart docks (#4247)', function() + world.players[1]:teleport('', util.vector3(19867, -102180, -79), util.transform.rotateZ(math.rad(91))) + coroutine.yield() + testing.runLocalTest(world.players[1], 'Player should be able to walk up stairs in Ebonheart docks (#4247)') +end) + +testing.registerGlobalTest('[issues] Guard in Imperial Prison Ship should find path (#7241)', function() + world.players[1]:teleport('Imperial Prison Ship', util.vector3(61, -135, -105), util.transform.rotateZ(math.rad(-20))) + coroutine.yield() + testing.runLocalTest(world.players[1], 'Guard in Imperial Prison Ship should find path (#7241)') +end) + +testing.registerGlobalTest('[issues] Should keep reference to an object moved into container (#7663)', function() + world.players[1]:teleport('ToddTest', util.vector3(2176, 3648, -191), util.transform.rotateZ(math.rad(0))) + coroutine.yield() + local barrel = world.createObject('barrel_01', 1) + local fargothRing = world.createObject('ring_keley', 1) + coroutine.yield() + testing.expectEqual(types.Container.inventory(barrel):find('ring_keley'), nil) + fargothRing:moveInto(types.Container.inventory(barrel)) + coroutine.yield() + testing.expectEqual(fargothRing.recordId, 'ring_keley') + local isFargothRing = function(actual) + if actual == nil then + return 'ring_keley is not found' end - testing.expectThat(types.Container.inventory(barrel):find('ring_keley'), isFargothRing) - end}, -} + if actual.id ~= fargothRing.id then + return 'found ring_keley id does not match expected: actual=' .. tostring(actual.id) + .. ', expected=' .. tostring(fargothRing.id) + end + return '' + end + testing.expectThat(types.Container.inventory(barrel):find('ring_keley'), isFargothRing) +end) diff --git a/scripts/data/morrowind_tests/global_mwscript.lua b/scripts/data/morrowind_tests/global_mwscript.lua index a4347ea66e..fec9fcdba5 100644 --- a/scripts/data/morrowind_tests/global_mwscript.lua +++ b/scripts/data/morrowind_tests/global_mwscript.lua @@ -14,38 +14,38 @@ function iterateOverVariables(variables) return first, last, count end -return { - {'Should support iteration over an empty set of script variables', function() - local mainVars = world.mwscript.getGlobalScript('main').variables - local first, last, count = iterateOverVariables(mainVars) - testing.expectEqual(first, nil) - testing.expectEqual(last, nil) - testing.expectEqual(count, 0) - testing.expectEqual(count, #mainVars) - end}, - {'Should support iteration of script variables', function() - local jiub = world.getObjectByFormId(core.getFormId('Morrowind.esm', 172867)) - local jiubVars = world.mwscript.getLocalScript(jiub).variables - local first, last, count = iterateOverVariables(jiubVars) +testing.registerGlobalTest('[mwscript] Should support iteration over an empty set of script variables', function() + local mainVars = world.mwscript.getGlobalScript('main').variables + local first, last, count = iterateOverVariables(mainVars) + testing.expectEqual(first, nil) + testing.expectEqual(last, nil) + testing.expectEqual(count, 0) + testing.expectEqual(count, #mainVars) +end) - testing.expectEqual(first, 'state') - testing.expectEqual(last, 'timer') - testing.expectEqual(count, 3) - testing.expectEqual(count, #jiubVars) - end}, - {'Should support numeric and string indices for getting and setting', function() - local jiub = world.getObjectByFormId(core.getFormId('Morrowind.esm', 172867)) - local jiubVars = world.mwscript.getLocalScript(jiub).variables +testing.registerGlobalTest('[mwscript] Should support iteration of script variables', function() + local jiub = world.getObjectByFormId(core.getFormId('Morrowind.esm', 172867)) + local jiubVars = world.mwscript.getLocalScript(jiub).variables + local first, last, count = iterateOverVariables(jiubVars) - testing.expectEqual(jiubVars[1], jiubVars.state) - testing.expectEqual(jiubVars[2], jiubVars.wandering) - testing.expectEqual(jiubVars[3], jiubVars.timer) + testing.expectEqual(first, 'state') + testing.expectEqual(last, 'timer') + testing.expectEqual(count, 3) + testing.expectEqual(count, #jiubVars) +end) - jiubVars[1] = 123; - testing.expectEqual(jiubVars.state, 123) - jiubVars.wandering = 42; - testing.expectEqual(jiubVars[2], 42) - jiubVars[3] = 1.25; - testing.expectEqual(jiubVars.timer, 1.25) - end}, -} +testing.registerGlobalTest('[mwscript] Should support numeric and string indices for getting and setting', function() + local jiub = world.getObjectByFormId(core.getFormId('Morrowind.esm', 172867)) + local jiubVars = world.mwscript.getLocalScript(jiub).variables + + testing.expectEqual(jiubVars[1], jiubVars.state) + testing.expectEqual(jiubVars[2], jiubVars.wandering) + testing.expectEqual(jiubVars[3], jiubVars.timer) + + jiubVars[1] = 123; + testing.expectEqual(jiubVars.state, 123) + jiubVars.wandering = 42; + testing.expectEqual(jiubVars[2], 42) + jiubVars[3] = 1.25; + testing.expectEqual(jiubVars.timer, 1.25) +end) diff --git a/scripts/data/morrowind_tests/player.lua b/scripts/data/morrowind_tests/player.lua index 7435b49553..226d9754f0 100644 --- a/scripts/data/morrowind_tests/player.lua +++ b/scripts/data/morrowind_tests/player.lua @@ -80,5 +80,5 @@ return { engineHandlers = { onFrame = testing.updateLocal, }, - eventHandlers = testing.eventHandlers + eventHandlers = testing.localEventHandlers, } diff --git a/scripts/integration_tests.py b/scripts/integration_tests.py index 17f89fbe2f..80c97f8b73 100755 --- a/scripts/integration_tests.py +++ b/scripts/integration_tests.py @@ -1,6 +1,13 @@ #!/usr/bin/env python3 -import argparse, datetime, os, subprocess, sys, shutil +import argparse +import datetime +import os +import shutil +import subprocess +import sys +import time + from pathlib import Path parser = argparse.ArgumentParser(description="OpenMW integration tests.") @@ -40,12 +47,13 @@ testing_util_dir = tests_dir / "testing_util" time_str = datetime.datetime.now().strftime("%Y-%m-%d-%H.%M.%S") -def runTest(name): - print(f"Start {name}") +def run_test(test_name): + start = time.time() + print(f'[----------] Running tests from {test_name}') shutil.rmtree(config_dir, ignore_errors=True) config_dir.mkdir() shutil.copyfile(example_suite_dir / "settings.cfg", config_dir / "settings.cfg") - test_dir = tests_dir / name + test_dir = tests_dir / test_name with open(config_dir / "openmw.cfg", "w", encoding="utf-8") as omw_cfg: for path in content_paths: omw_cfg.write(f'data="{path.parent}"\n') @@ -58,10 +66,8 @@ def runTest(name): ) for path in content_paths: omw_cfg.write(f'content={path.name}\n') - if (test_dir / "openmw.cfg").exists(): - omw_cfg.write(open(test_dir / "openmw.cfg").read()) - elif (test_dir / "test.omwscripts").exists(): - omw_cfg.write("content=test.omwscripts\n") + with open(test_dir / "openmw.cfg") as stream: + omw_cfg.write(stream.read()) with open(config_dir / "settings.cfg", "a", encoding="utf-8") as settings_cfg: settings_cfg.write( "[Video]\n" @@ -74,61 +80,76 @@ def runTest(name): f"memory limit = {1024 * 1024 * 256}\n" ) stdout_lines = list() - exit_ok = True test_success = True + fatal_errors = list() with subprocess.Popen( - [openmw_binary, "--replace=config", "--config", config_dir, "--skip-menu", "--no-grab", "--no-sound"], + [openmw_binary, "--replace=config", "--config", config_dir, "--no-grab"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", env={ - "OPENMW_OSG_STATS_FILE": str(work_dir / f"{name}.{time_str}.osg_stats.log"), + "OPENMW_OSG_STATS_FILE": str(work_dir / f"{test_name}.{time_str}.osg_stats.log"), "OPENMW_OSG_STATS_LIST": "times", **os.environ, }, ) as process: quit_requested = False + running_test_number = None + running_test_name = None + count = 0 + failed_tests = list() + test_start = None for line in process.stdout: if args.verbose: sys.stdout.write(line) else: stdout_lines.append(line) - words = line.split(" ") - if len(words) > 1 and words[1] == "E]": - print(line, end="") - elif "Quit requested by a Lua script" in line: + if "Quit requested by a Lua script" in line: quit_requested = True elif "TEST_START" in line: - w = line.split("TEST_START")[1].split("\t") - print(f"TEST {w[2].strip()}\t\t", end="") + test_start = time.time() + number, name = line.split("TEST_START")[1].strip().split("\t", maxsplit=1) + running_test_number = int(number) + running_test_name = name + count += 1 + print(f"[ RUN ] {running_test_name}") elif "TEST_OK" in line: - print(f"OK") + duration = (time.time() - test_start) * 1000 + number, name = line.split("TEST_OK")[1].strip().split("\t", maxsplit=1) + assert running_test_number == int(number) + print(f"[ OK ] {running_test_name} ({duration:.3f} ms)") elif "TEST_FAILED" in line: - w = line.split("TEST_FAILED")[1].split("\t") - print(f"FAILED {w[3]}\t\t") - test_success = False + duration = (time.time() - test_start) * 1000 + number, name, error = line.split("TEST_FAILED")[1].strip().split("\t", maxsplit=2) + assert running_test_number == int(number) + print(error) + print(f"[ FAILED ] {running_test_name} ({duration:.3f} ms)") + failed_tests.append(running_test_name) process.wait(5) if not quit_requested: - print("ERROR: Unexpected termination") - exit_ok = False + fatal_errors.append("unexpected termination") if process.returncode != 0: - print(f"ERROR: openmw exited with code {process.returncode}") - exit_ok = False + fatal_errors.append(f"openmw exited with code {process.returncode}") if os.path.exists(config_dir / "openmw.log"): - shutil.copyfile(config_dir / "openmw.log", work_dir / f"{name}.{time_str}.log") - if not exit_ok and not args.verbose: + shutil.copyfile(config_dir / "openmw.log", work_dir / f"{test_name}.{time_str}.log") + if fatal_errors and not args.verbose: sys.stdout.writelines(stdout_lines) - if test_success and exit_ok: - print(f"{name} succeeded") - else: - print(f"{name} failed") - return test_success and exit_ok + total_duration = (time.time() - start) * 1000 + print(f'\n[----------] {count} tests from {test_name} ({total_duration:.3f} ms total)') + print(f"[ PASSED ] {count - len(failed_tests)} tests.") + if fatal_errors: + print(f"[ FAILED ] fatal error: {'; '.join(fatal_errors)}") + if failed_tests: + print(f"[ FAILED ] {len(failed_tests)} tests, listed below:") + for failed_test in failed_tests: + print(f"[ FAILED ] {failed_test}") + return len(failed_tests) == 0 and not fatal_errors status = 0 for entry in tests_dir.glob("test_*"): if entry.is_dir(): - if not runTest(entry.name): + if not run_test(entry.name): status = -1 if status == 0: shutil.rmtree(config_dir, ignore_errors=True)