Merge branch 'lua_test_menu' into 'master'

Run Lua integration tests starting with menu script

See merge request OpenMW/openmw!4556

CI/run_integration_tests.sh

@@ -9,7 +9,7 @@ git checkout FETCH_HEAD
 cd ..
 
 xvfb-run --auto-servernum --server-args='-screen 0 640x480x24x60' \
-    scripts/integration_tests.py --omw build/install/bin/openmw --workdir integration_tests_output example-suite/
+    scripts/integration_tests.py --verbose --omw build/install/bin/openmw --workdir integration_tests_output example-suite/
 
 ls integration_tests_output/*.osg_stats.log | while read v; do
     scripts/osg_stats.py --stats '.*' --regexp_match < "${v}"

scripts/data/integration_tests/test_lua_api/global.lua

@@ -7,7 +7,7 @@ local vfs = require('openmw.vfs')
 local world = require('openmw.world')
 local I = require('openmw.interfaces')
 
-local function testTimers()
+testing.registerGlobalTest('timers', function()
     testing.expectAlmostEqual(core.getGameTimeScale(), 30, 'incorrect getGameTimeScale() result')
     testing.expectAlmostEqual(core.getSimulationTimeScale(), 1, 'incorrect getSimulationTimeScale result')
 
@@ -39,9 +39,10 @@ local function testTimers()
     testing.expectGreaterOrEqual(ts1, 0.5, 'async:newSimulationTimer failed')
     testing.expectGreaterOrEqual(th2, 72, 'async:newUnsavableGameTimer failed')
     testing.expectGreaterOrEqual(ts2, 1, 'async:newUnsavableSimulationTimer failed')
-end
+end)
 
-local function testTeleport()
+testing.registerGlobalTest('teleport', function()
+    local player = world.players[1]
     player:teleport('', util.vector3(100, 50, 500), util.transform.rotateZ(math.rad(90)))
     coroutine.yield()
     testing.expect(player.cell.isExterior, 'teleport to exterior failed')
@@ -71,16 +72,16 @@ local function testTeleport()
     testing.expectEqualWithDelta(player.position.x, 50, 1, 'incorrect position after teleporting')
     testing.expectEqualWithDelta(player.position.y, -100, 1, 'incorrect position after teleporting')
     testing.expectEqualWithDelta(player.rotation:getYaw(), math.rad(-90), 0.05, 'teleporting changes rotation')
-end
+end)
 
-local function testGetGMST()
+testing.registerGlobalTest('getGMST', function()
     testing.expectEqual(core.getGMST('non-existed gmst'), nil)
     testing.expectEqual(core.getGMST('Water_RippleFrameCount'), 4)
     testing.expectEqual(core.getGMST('Inventory_DirectionalDiffuseR'), 0.5)
     testing.expectEqual(core.getGMST('Level_Up_Level2'), 'something')
-end
+end)
 
-local function testMWScript()
+testing.registerGlobalTest('MWScript', function()
     local variableStoreCount = 18
     local variableStore = world.mwscript.getGlobalVariables(player)
     testing.expectEqual(variableStoreCount, #variableStore)
@@ -100,7 +101,7 @@ local function testMWScript()
         indexCheck = indexCheck + 1
     end
     testing.expectEqual(variableStoreCount, indexCheck)
-end
+end)
 
 local function testRecordStore(store, storeName, skipPairs)
     testing.expect(store.records)
@@ -121,7 +122,7 @@ local function testRecordStore(store, storeName, skipPairs)
     testing.expectEqual(status, true, storeName)
 end
 
-local function testRecordStores()
+testing.registerGlobalTest('record stores', function()
     for key, type in pairs(types) do
         if type.records then
             testRecordStore(type, key)
@@ -140,9 +141,9 @@ local function testRecordStores()
     testRecordStore(types.NPC.classes, "classes")
     testRecordStore(types.NPC.races, "races")
     testRecordStore(types.Player.birthSigns, "birthSigns")
-end
+end)
 
-local function testRecordCreation()
+testing.registerGlobalTest('record creation', function()
     local newLight = {
         isCarriable = true,
         isDynamic = true,
@@ -165,9 +166,9 @@ local function testRecordCreation()
     for key, value in pairs(newLight) do
         testing.expectEqual(record[key], value)
     end
-end
+end)
 
-local function testUTF8Chars()
+testing.registerGlobalTest('UTF-8 characters', function()
     testing.expectEqual(utf8.codepoint("😀"), 0x1F600)
 
     local chars = {}
@@ -192,9 +193,9 @@ local function testUTF8Chars()
         testing.expectEqual(utf8.codepoint(char), codepoint)
         testing.expectEqual(utf8.len(char), 1)
     end
-end
+end)
 
-local function testUTF8Strings()
+testing.registerGlobalTest('UTF-8 strings', function()
     local utf8str = "Hello, 你好, 🌎!"
 
     local str = ""
@@ -205,9 +206,9 @@ local function testUTF8Strings()
 
     testing.expectEqual(utf8.len(utf8str), 13)
     testing.expectEqual(utf8.offset(utf8str, 9), 11)
-end
+end)
 
-local function testMemoryLimit()
+testing.registerGlobalTest('memory limit', function()
     local ok, err = pcall(function()
         local t = {}
         local n = 1
@@ -218,14 +219,16 @@ local function testMemoryLimit()
     end)
     testing.expectEqual(ok, false, 'Script reaching memory limit should fail')
     testing.expectEqual(err, 'not enough memory')
-end
+end)
 
 local function initPlayer()
+    local player = world.players[1]
     player:teleport('', util.vector3(4096, 4096, 1745), util.transform.identity)
     coroutine.yield()
+    return player
 end
 
-local function testVFS()
+testing.registerGlobalTest('vfs', function()
     local file = 'test_vfs_dir/lines.txt'
     local nosuchfile = 'test_vfs_dir/nosuchfile'
     testing.expectEqual(vfs.fileExists(file), true, 'lines.txt should exist')
@@ -269,12 +272,11 @@ local function testVFS()
     for _,v in pairs(expectedLines) do
         testing.expectEqual(getLine(), v)
     end
-end
+end)
 
-local function testCommitCrime()
+testing.registerGlobalTest('commit crime', function()
-    initPlayer()
+    local player = initPlayer()
-    local player = world.players[1]
+    testing.expectEqual(player == nil, false, 'A viable player reference should exist to run `commit crime`')
-    testing.expectEqual(player == nil, false, 'A viable player reference should exist to run `testCommitCrime`')
     testing.expectEqual(I.Crimes == nil, false, 'Crimes interface should be available in global contexts')
 
     -- Reset crime level to have a clean slate
@@ -292,82 +294,41 @@ local function testCommitCrime()
     types.Player.setCrimeLevel(player, 0)
     testing.expectEqual(I.Crimes.commitCrime(player, { victim = victim, type = types.Player.OFFENSE_TYPE.Theft, arg = 50 }).wasCrimeSeen, true, "Running a crime with a valid victim should notify them when the player is not sneaking, even if it's not explicitly passed in")
     testing.expectEqual(types.Player.getCrimeLevel(player), 0, "Crime level should not change if the victim's alarm value is low and there's no other witnesses")
-end
+end)
 
-local function testRecordModelProperty()
+testing.registerGlobalTest('record model property', function()
-    initPlayer()
     local player = world.players[1]
     testing.expectEqual(types.NPC.record(player).model, 'meshes/basicplayer.dae')
+end)
+
+local function registerPlayerTest(name)
+    testing.registerGlobalTest(name, function()
+        local player = initPlayer()
+        testing.runLocalTest(player, name)
+    end)
 end
 
-tests = {
+registerPlayerTest('player yaw rotation')
-    {'timers', testTimers},
+registerPlayerTest('player pitch rotation')
-    {'rotating player with controls.yawChange should change rotation', function()
+registerPlayerTest('player pitch and yaw rotation')
-        initPlayer()
+registerPlayerTest('player rotation')
-        testing.runLocalTest(player, 'playerYawRotation')
+registerPlayerTest('player forward running')
-    end},
+registerPlayerTest('player diagonal walking')
-    {'rotating player with controls.pitchChange should change rotation', function()
+registerPlayerTest('findPath')
-        initPlayer()
+registerPlayerTest('findRandomPointAroundCircle')
-        testing.runLocalTest(player, 'playerPitchRotation')
+registerPlayerTest('castNavigationRay')
-    end},
+registerPlayerTest('findNearestNavMeshPosition')
-    {'rotating player with controls.pitchChange and controls.yawChange should change rotation', function()
+registerPlayerTest('player memory limit')
-        initPlayer()
+
-        testing.runLocalTest(player, 'playerPitchAndYawRotation')
+testing.registerGlobalTest('player weapon attack', function()
-    end},
+    local player = initPlayer()
-    {'rotating player should not lead to nan rotation', function()
-        initPlayer()
-        testing.runLocalTest(player, 'playerRotation')
-    end},
-    {'playerForwardRunning', function()
-        initPlayer()
-        testing.runLocalTest(player, 'playerForwardRunning')
-    end},
-    {'playerDiagonalWalking', function()
-        initPlayer()
-        testing.runLocalTest(player, 'playerDiagonalWalking')
-    end},
-    {'findPath', function()
-        initPlayer()
-        testing.runLocalTest(player, 'findPath')
-    end},
-    {'findRandomPointAroundCircle', function()
-        initPlayer()
-        testing.runLocalTest(player, 'findRandomPointAroundCircle')
-    end},
-    {'castNavigationRay', function()
-        initPlayer()
-        testing.runLocalTest(player, 'castNavigationRay')
-    end},
-    {'findNearestNavMeshPosition', function()
-        initPlayer()
-        testing.runLocalTest(player, 'findNearestNavMeshPosition')
-    end},
-    {'teleport', testTeleport},
-    {'getGMST', testGetGMST},
-    {'recordStores', testRecordStores},
-    {'recordCreation', testRecordCreation},
-    {'utf8Chars', testUTF8Chars},
-    {'utf8Strings', testUTF8Strings},
-    {'mwscript', testMWScript},
-    {'testMemoryLimit', testMemoryLimit},
-    {'playerMemoryLimit', function()
-        initPlayer()
-        testing.runLocalTest(player, 'playerMemoryLimit')
-    end},
-    {'player with equipped weapon on attack should damage health of other actors', function()
-        initPlayer()
     world.createObject('basic_dagger1h', 1):moveInto(player)
-        testing.runLocalTest(player, 'playerWeaponAttack')
+    testing.runLocalTest(player, 'player weapon attack')
-    end},
+end)
-    {'vfs', testVFS},
-    {'testCommitCrime', testCommitCrime},
-    {'recordModelProperty', testRecordModelProperty},
-}
 
 return {
     engineHandlers = {
-        onUpdate = testing.testRunner(tests),
+        onUpdate = testing.updateGlobal,
-        onPlayerAdded = function(p) player = p end,
     },
-    eventHandlers = testing.eventHandlers,
+    eventHandlers = testing.globalEventHandlers,
 }

scripts/data/integration_tests/test_lua_api/menu.lua

@@ -0,0 +1,43 @@
+local testing = require('testing_util')
+local menu = require('openmw.menu')
+
+local function registerGlobalTest(name, description)
+    testing.registerMenuTest(description or name, function()
+        menu.newGame()
+        coroutine.yield()
+        testing.runGlobalTest(name)
+    end)
+end
+
+registerGlobalTest('timers')
+registerGlobalTest('teleport')
+registerGlobalTest('getGMST')
+registerGlobalTest('MWScript')
+registerGlobalTest('record stores')
+registerGlobalTest('record creation')
+registerGlobalTest('UTF-8 characters')
+registerGlobalTest('UTF-8 strings')
+registerGlobalTest('memory limit')
+registerGlobalTest('vfs')
+registerGlobalTest('commit crime')
+registerGlobalTest('record model property')
+
+registerGlobalTest('player yaw rotation', 'rotating player with controls.yawChange should change rotation')
+registerGlobalTest('player pitch rotation', 'rotating player with controls.pitchChange should change rotation')
+registerGlobalTest('player pitch and yaw rotation', 'rotating player with controls.pitchChange and controls.yawChange should change rotation')
+registerGlobalTest('player rotation', 'rotating player should not lead to nan rotation')
+registerGlobalTest('player forward running')
+registerGlobalTest('player diagonal walking')
+registerGlobalTest('findPath')
+registerGlobalTest('findRandomPointAroundCircle')
+registerGlobalTest('castNavigationRay')
+registerGlobalTest('findNearestNavMeshPosition')
+registerGlobalTest('player memory limit')
+registerGlobalTest('player weapon attack', 'player with equipped weapon on attack should damage health of other actors')
+
+return {
+    engineHandlers = {
+        onFrame = testing.makeUpdateMenu(),
+    },
+    eventHandlers = testing.menuEventHandlers,
+}

scripts/data/integration_tests/test_lua_api/openmw.cfg

@@ -1,4 +1,4 @@
-content=test.omwscripts
+content=test_lua_api.omwscripts
 
 # Needed to test `core.getGMST`
 fallback=Water_RippleFrameCount,4

scripts/data/integration_tests/test_lua_api/player.lua

@@ -40,7 +40,7 @@ local function rotateByPitch(object, target)
     rotate(object, target, nil)
 end
 
-testing.registerLocalTest('playerYawRotation',
+testing.registerLocalTest('player yaw rotation',
     function()
         local initialAlphaXZ, initialGammaXZ = self.rotation:getAnglesXZ()
         local initialAlphaZYX, initialBetaZYX, initialGammaZYX = self.rotation:getAnglesZYX()
@@ -60,7 +60,7 @@ testing.registerLocalTest('playerYawRotation',
         testing.expectEqualWithDelta(gamma2, initialGammaZYX, 0.05, 'Gamma rotation in ZYX convention should not change')
     end)
 
-testing.registerLocalTest('playerPitchRotation',
+testing.registerLocalTest('player pitch rotation',
     function()
         local initialAlphaXZ, initialGammaXZ = self.rotation:getAnglesXZ()
         local initialAlphaZYX, initialBetaZYX, initialGammaZYX = self.rotation:getAnglesZYX()
@@ -80,7 +80,7 @@ testing.registerLocalTest('playerPitchRotation',
         testing.expectEqualWithDelta(gamma2, targetPitch, 0.05, 'Incorrect gamma rotation in ZYX convention')
     end)
 
-testing.registerLocalTest('playerPitchAndYawRotation',
+testing.registerLocalTest('player pitch and yaw rotation',
     function()
         local targetPitch = math.rad(-30)
         local targetYaw = math.rad(-60)
@@ -99,7 +99,7 @@ testing.registerLocalTest('playerPitchAndYawRotation',
         testing.expectEqualWithDelta(gamma2, math.rad(-16), 0.05, 'Incorrect gamma rotation in ZYX convention')
     end)
 
-testing.registerLocalTest('playerRotation',
+testing.registerLocalTest('player rotation',
     function()
         local rotation = math.sqrt(2)
         local endTime = core.getSimulationTime() + 3
@@ -123,7 +123,7 @@ testing.registerLocalTest('playerRotation',
         end
     end)
 
-testing.registerLocalTest('playerForwardRunning',
+testing.registerLocalTest('player forward running',
     function()
         local startPos = self.position
         local endTime = core.getSimulationTime() + 1
@@ -141,7 +141,7 @@ testing.registerLocalTest('playerForwardRunning',
         testing.expectEqualWithDelta(direction.y, 1, 0.1, 'Run forward, Y coord')
     end)
 
-testing.registerLocalTest('playerDiagonalWalking',
+testing.registerLocalTest('player diagonal walking',
     function()
         local startPos = self.position
         local endTime = core.getSimulationTime() + 1
@@ -220,7 +220,7 @@ testing.registerLocalTest('findNearestNavMeshPosition',
             'Navigation mesh position ' .. testing.formatActualExpected(result, expected))
     end)
 
-testing.registerLocalTest('playerMemoryLimit',
+testing.registerLocalTest('player memory limit',
     function()
         local ok, err = pcall(function()
             local str = 'a'
@@ -232,7 +232,7 @@ testing.registerLocalTest('playerMemoryLimit',
         testing.expectEqual(err, 'not enough memory')
     end)
 
-testing.registerLocalTest('playerWeaponAttack',
+testing.registerLocalTest('player weapon attack',
     function()
         camera.setMode(camera.MODE.ThirdPerson)
 
@@ -346,5 +346,5 @@ return {
     engineHandlers = {
         onFrame = testing.updateLocal,
     },
-    eventHandlers = testing.eventHandlers
+    eventHandlers = testing.localEventHandlers,
 }

scripts/data/integration_tests/test_lua_api/test.omwscripts

@@ -1,2 +0,0 @@
-GLOBAL: test.lua
-PLAYER: player.lua

scripts/data/integration_tests/test_lua_api/test_lua_api.omwscripts

@@ -0,0 +1,3 @@
+MENU: menu.lua
+GLOBAL: global.lua
+PLAYER: player.lua

scripts/data/integration_tests/testing_util/testing_util.lua

@@ -2,11 +2,31 @@ local core = require('openmw.core')
 local util = require('openmw.util')
 
 local M = {}
+
+local menuTestsOrder = {}
+local menuTests = {}
+
+local globalTestsOrder = {}
+local globalTests = {}
+local globalTestRunner = nil
+local currentGlobalTest = nil
+local currentGlobalTestError = nil
+
+local localTests = {}
+local localTestRunner = nil
 local currentLocalTest = nil
 local currentLocalTestError = nil
 
-function M.testRunner(tests)
+local function makeTestCoroutine(fn)
-    local fn = function()
+    local co = coroutine.create(fn)
+    return function()
+        if coroutine.status(co) ~= 'dead' then
+            coroutine.resume(co)
+        end
+    end
+end
+
+local function runTests(tests)
     for i, test in ipairs(tests) do
         local name, fn = unpack(test)
         print('TEST_START', i, name)
@@ -18,12 +38,49 @@ function M.testRunner(tests)
         end
     end
     core.quit()
+end
+
+function M.makeUpdateMenu()
+    return makeTestCoroutine(function()
+        print('Running menu tests...')
+        runTests(menuTestsOrder)
+    end)
+end
+
+function M.makeUpdateGlobal()
+    return makeTestCoroutine(function()
+        print('Running global tests...')
+        runTests(globalTestsOrder)
+    end)
+end
+
+function M.registerMenuTest(name, fn)
+    menuTests[name] = fn
+    table.insert(menuTestsOrder, {name, fn})
+end
+
+function M.runGlobalTest(name)
+    currentGlobalTest = name
+    currentGlobalTestError = nil
+    core.sendGlobalEvent('runGlobalTest', name)
+    while currentGlobalTest do
+        coroutine.yield()
     end
-    local co = coroutine.create(fn)
+    if currentGlobalTestError then
-    return function()
+        error(currentGlobalTestError, 2)
-        if coroutine.status(co) ~= 'dead' then
-            coroutine.resume(co)
     end
+end
+
+function M.registerGlobalTest(name, fn)
+    globalTests[name] = fn
+    table.insert(globalTestsOrder, {name, fn})
+end
+
+function M.updateGlobal()
+    if globalTestRunner and coroutine.status(globalTestRunner) ~= 'dead' then
+        coroutine.resume(globalTestRunner)
+    else
+        globalTestRunner = nil
     end
 end
 
@@ -39,7 +96,21 @@ function M.runLocalTest(obj, name)
     end
 end
 
-function M.expect(cond, delta, msg)
+function M.registerLocalTest(name, fn)
+    localTests[name] = fn
+end
+
+function M.updateLocal()
+    if localTestRunner and coroutine.status(localTestRunner) ~= 'dead' then
+        if not core.isWorldPaused() then
+            coroutine.resume(localTestRunner)
+        end
+    else
+        localTestRunner = nil
+    end
+end
+
+function M.expect(cond, msg)
     if not cond then
         error(msg or '"true" expected', 2)
     end
@@ -182,28 +253,50 @@ function M.formatActualExpected(actual, expected)
     return string.format('actual: %s, expected: %s', actual, expected)
 end
 
-local localTests = {}
+-- used only in menu scripts
-local localTestRunner = nil
+M.menuEventHandlers = {
-
+    globalTestFinished = function(data)
-function M.registerLocalTest(name, fn)
+        if data.name ~= currentGlobalTest then
-    localTests[name] = fn
+            error(string.format('globalTestFinished with incorrect name %s, expected %s', data.name, currentGlobalTest), 2)
-end
-
-function M.updateLocal()
-    if localTestRunner and coroutine.status(localTestRunner) ~= 'dead' then
-        if not core.isWorldPaused() then
-            coroutine.resume(localTestRunner)
         end
-    else
+        currentGlobalTest = nil
-        localTestRunner = nil
+        currentGlobalTestError = data.errMsg
-    end
+    end,
-end
+}
 
-M.eventHandlers = {
+-- used only in global scripts
-    runLocalTest = function(name)  -- used only in local scripts
+M.globalEventHandlers = {
+    runGlobalTest = function(name)
+        fn = globalTests[name]
+        local types = require('openmw.types')
+        local world = require('openmw.world')
+        if not fn then
+            types.Player.sendMenuEvent(world.players[1], 'globalTestFinished', {name=name, errMsg='Global test is not found'})
+            return
+        end
+        globalTestRunner = coroutine.create(function()
+            local status, err = pcall(fn)
+            if status then
+                err = nil
+            end
+            types.Player.sendMenuEvent(world.players[1], 'globalTestFinished', {name=name, errMsg=err})
+        end)
+    end,
+    localTestFinished = function(data)
+        if data.name ~= currentLocalTest then
+            error(string.format('localTestFinished with incorrect name %s, expected %s', data.name, currentLocalTest), 2)
+        end
+        currentLocalTest = nil
+        currentLocalTestError = data.errMsg
+    end,
+}
+
+-- used only in local scripts
+M.localEventHandlers = {
+    runLocalTest = function(name)
         fn = localTests[name]
         if not fn then
-            core.sendGlobalEvent('localTestFinished', {name=name, errMsg='Test not found'})
+            core.sendGlobalEvent('localTestFinished', {name=name, errMsg='Local test is not found'})
             return
         end
         localTestRunner = coroutine.create(function()
@@ -214,13 +307,6 @@ M.eventHandlers = {
             core.sendGlobalEvent('localTestFinished', {name=name, errMsg=err})
         end)
     end,
-    localTestFinished = function(data)  -- used only in global scripts
-        if data.name ~= currentLocalTest then
-            error(string.format('localTestFinished with incorrect name %s, expected %s', data.name, currentLocalTest))
-        end
-        currentLocalTest = nil
-        currentLocalTestError = data.errMsg
-    end,
 }
 
 return M

scripts/data/morrowind_tests/global.lua

@@ -8,28 +8,13 @@ if not core.contentFiles.has('Morrowind.esm') then
     error('This test requires Morrowind.esm')
 end
 
-function makeTests(modules)
+require('global_issues')
-    local tests = {}
+require('global_dialogues')
-
+require('global_mwscript')
-    for _, moduleName in ipairs(modules) do
-        local module = require(moduleName)
-        for _, v in ipairs(module) do
-            table.insert(tests, {string.format('[%s] %s', moduleName, v[1]), v[2]})
-        end
-    end
-
-    return tests
-end
-
-local testModules = {
-    'global_issues',
-    'global_dialogues',
-    'global_mwscript',
-}
 
 return {
     engineHandlers = {
-        onUpdate = testing.testRunner(makeTests(testModules)),
+        onUpdate = testing.makeUpdateGlobal(),
     },
-    eventHandlers = testing.eventHandlers,
+    eventHandlers = testing.globalEventHandlers,
 }

scripts/data/morrowind_tests/global_dialogues.lua

@@ -13,35 +13,37 @@ function iterateOverRecords(records)
     return firstRecordId, lastRecordId, count
 end
 
-return {
+testing.registerGlobalTest('[dialogues] Should support iteration over journal dialogues', function()
-    {'Should support iteration over journal dialogues', function()
     local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.journal.records)
     testing.expectEqual(firstRecordId, '11111 test journal')
     testing.expectEqual(lastRecordId, 'va_vamprich')
     testing.expectEqual(count, 632)
-    end},
+end)
-    {'Should support iteration over topic dialogues', function()
+
+testing.registerGlobalTest('[dialogues] Should support iteration over topic dialogues', function()
     local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.topic.records)
     testing.expectEqual(firstRecordId, '1000-drake pledge')
     testing.expectEqual(lastRecordId, 'zenithar')
     testing.expectEqual(count, 1698)
-    end},
+end)
-    {'Should support iteration over greeting dialogues', function()
+
+testing.registerGlobalTest('[dialogues] Should support iteration over greeting dialogues', function()
     local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.greeting.records)
     testing.expectEqual(firstRecordId, 'greeting 0')
     testing.expectEqual(lastRecordId, 'greeting 9')
     testing.expectEqual(count, 10)
-    end},
+end)
-    {'Should support iteration over persuasion dialogues', function()
+
+testing.registerGlobalTest('[dialogues] Should support iteration over persuasion dialogues', function()
     local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.persuasion.records)
     testing.expectEqual(firstRecordId, 'admire fail')
     testing.expectEqual(lastRecordId, 'taunt success')
     testing.expectEqual(count, 10)
-    end},
+end)
-    {'Should support iteration over voice dialogues', function()
+
+testing.registerGlobalTest('[dialogues] Should support iteration over voice dialogues', function()
     local firstRecordId, lastRecordId, count = iterateOverRecords(core.dialogue.voice.records)
     testing.expectEqual(firstRecordId, 'alarm')
     testing.expectEqual(lastRecordId, 'thief')
     testing.expectEqual(count, 8)
-    end},
+end)
-}

scripts/data/morrowind_tests/global_issues.lua

@@ -4,18 +4,19 @@ local world = require('openmw.world')
 local core = require('openmw.core')
 local types = require('openmw.types')
 
-return {
+testing.registerGlobalTest('[issues] Player should be able to walk up stairs in Ebonheart docks (#4247)', function()
-    {'Player should be able to walk up stairs in Ebonheart docks (#4247)', function()
     world.players[1]:teleport('', util.vector3(19867, -102180, -79), util.transform.rotateZ(math.rad(91)))
     coroutine.yield()
     testing.runLocalTest(world.players[1], 'Player should be able to walk up stairs in Ebonheart docks (#4247)')
-    end},
+end)
-    {'Guard in Imperial Prison Ship should find path (#7241)', function()
+
+testing.registerGlobalTest('[issues] Guard in Imperial Prison Ship should find path (#7241)', function()
     world.players[1]:teleport('Imperial Prison Ship', util.vector3(61, -135, -105), util.transform.rotateZ(math.rad(-20)))
     coroutine.yield()
     testing.runLocalTest(world.players[1], 'Guard in Imperial Prison Ship should find path (#7241)')
-    end},
+end)
-    {'Should keep reference to an object moved into container (#7663)', function()
+
+testing.registerGlobalTest('[issues] Should keep reference to an object moved into container (#7663)', function()
     world.players[1]:teleport('ToddTest', util.vector3(2176, 3648, -191), util.transform.rotateZ(math.rad(0)))
     coroutine.yield()
     local barrel = world.createObject('barrel_01', 1)
@@ -36,5 +37,4 @@ return {
         return ''
     end
     testing.expectThat(types.Container.inventory(barrel):find('ring_keley'), isFargothRing)
-    end},
+end)
-}

scripts/data/morrowind_tests/global_mwscript.lua

@@ -14,16 +14,16 @@ function iterateOverVariables(variables)
     return first, last, count
 end
 
-return {
+testing.registerGlobalTest('[mwscript] Should support iteration over an empty set of script variables', function()
-    {'Should support iteration over an empty set of script variables', function()
     local mainVars = world.mwscript.getGlobalScript('main').variables
     local first, last, count = iterateOverVariables(mainVars)
     testing.expectEqual(first, nil)
     testing.expectEqual(last, nil)
     testing.expectEqual(count, 0)
     testing.expectEqual(count, #mainVars)
-    end},
+end)
-    {'Should support iteration of script variables', function()
+
+testing.registerGlobalTest('[mwscript] Should support iteration of script variables', function()
     local jiub = world.getObjectByFormId(core.getFormId('Morrowind.esm', 172867))
     local jiubVars = world.mwscript.getLocalScript(jiub).variables
     local first, last, count = iterateOverVariables(jiubVars)
@@ -32,8 +32,9 @@ return {
     testing.expectEqual(last, 'timer')
     testing.expectEqual(count, 3)
     testing.expectEqual(count, #jiubVars)
-    end},
+end)
-    {'Should support numeric and string indices for getting and setting', function()
+
+testing.registerGlobalTest('[mwscript] Should support numeric and string indices for getting and setting', function()
     local jiub = world.getObjectByFormId(core.getFormId('Morrowind.esm', 172867))
     local jiubVars = world.mwscript.getLocalScript(jiub).variables
 
@@ -47,5 +48,4 @@ return {
     testing.expectEqual(jiubVars[2], 42)
     jiubVars[3] = 1.25;
     testing.expectEqual(jiubVars.timer, 1.25)
-    end},
+end)
-}

scripts/data/morrowind_tests/player.lua

@@ -80,5 +80,5 @@ return {
     engineHandlers = {
         onFrame = testing.updateLocal,
     },
-    eventHandlers = testing.eventHandlers
+    eventHandlers = testing.localEventHandlers,
 }

scripts/integration_tests.py

@@ -1,6 +1,13 @@
 #!/usr/bin/env python3
 
-import argparse, datetime, os, subprocess, sys, shutil
+import argparse
+import datetime
+import os
+import shutil
+import subprocess
+import sys
+import time
+
 from pathlib import Path
 
 parser = argparse.ArgumentParser(description="OpenMW integration tests.")
@@ -40,12 +47,13 @@ testing_util_dir = tests_dir / "testing_util"
 time_str = datetime.datetime.now().strftime("%Y-%m-%d-%H.%M.%S")
 
 
-def runTest(name):
+def run_test(test_name):
-    print(f"Start {name}")
+    start = time.time()
+    print(f'[----------] Running tests from {test_name}')
     shutil.rmtree(config_dir, ignore_errors=True)
     config_dir.mkdir()
     shutil.copyfile(example_suite_dir / "settings.cfg", config_dir / "settings.cfg")
-    test_dir = tests_dir / name
+    test_dir = tests_dir / test_name
     with open(config_dir / "openmw.cfg", "w", encoding="utf-8") as omw_cfg:
         for path in content_paths:
             omw_cfg.write(f'data="{path.parent}"\n')
@@ -58,10 +66,8 @@ def runTest(name):
         )
         for path in content_paths:
             omw_cfg.write(f'content={path.name}\n')
-        if (test_dir / "openmw.cfg").exists():
+        with open(test_dir / "openmw.cfg") as stream:
-            omw_cfg.write(open(test_dir / "openmw.cfg").read())
+            omw_cfg.write(stream.read())
-        elif (test_dir / "test.omwscripts").exists():
-            omw_cfg.write("content=test.omwscripts\n")
     with open(config_dir / "settings.cfg", "a", encoding="utf-8") as settings_cfg:
         settings_cfg.write(
             "[Video]\n"
@@ -74,61 +80,76 @@ def runTest(name):
             f"memory limit = {1024 * 1024 * 256}\n"
         )
     stdout_lines = list()
-    exit_ok = True
     test_success = True
+    fatal_errors = list()
     with subprocess.Popen(
-        [openmw_binary, "--replace=config", "--config", config_dir, "--skip-menu", "--no-grab", "--no-sound"],
+        [openmw_binary, "--replace=config", "--config", config_dir, "--no-grab"],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         encoding="utf-8",
         env={
-            "OPENMW_OSG_STATS_FILE": str(work_dir / f"{name}.{time_str}.osg_stats.log"),
+            "OPENMW_OSG_STATS_FILE": str(work_dir / f"{test_name}.{time_str}.osg_stats.log"),
             "OPENMW_OSG_STATS_LIST": "times",
             **os.environ,
         },
     ) as process:
         quit_requested = False
+        running_test_number = None
+        running_test_name = None
+        count = 0
+        failed_tests = list()
+        test_start = None
         for line in process.stdout:
             if args.verbose:
                 sys.stdout.write(line)
             else:
                 stdout_lines.append(line)
-            words = line.split(" ")
+            if "Quit requested by a Lua script" in line:
-            if len(words) > 1 and words[1] == "E]":
-                print(line, end="")
-            elif "Quit requested by a Lua script" in line:
                 quit_requested = True
             elif "TEST_START" in line:
-                w = line.split("TEST_START")[1].split("\t")
+                test_start = time.time()
-                print(f"TEST {w[2].strip()}\t\t", end="")
+                number, name = line.split("TEST_START")[1].strip().split("\t", maxsplit=1)
+                running_test_number = int(number)
+                running_test_name = name
+                count += 1
+                print(f"[ RUN      ] {running_test_name}")
             elif "TEST_OK" in line:
-                print(f"OK")
+                duration = (time.time() - test_start) * 1000
+                number, name = line.split("TEST_OK")[1].strip().split("\t", maxsplit=1)
+                assert running_test_number == int(number)
+                print(f"[       OK ] {running_test_name} ({duration:.3f} ms)")
             elif "TEST_FAILED" in line:
-                w = line.split("TEST_FAILED")[1].split("\t")
+                duration = (time.time() - test_start) * 1000
-                print(f"FAILED {w[3]}\t\t")
+                number, name, error = line.split("TEST_FAILED")[1].strip().split("\t", maxsplit=2)
-                test_success = False
+                assert running_test_number == int(number)
+                print(error)
+                print(f"[  FAILED  ] {running_test_name} ({duration:.3f} ms)")
+                failed_tests.append(running_test_name)
         process.wait(5)
         if not quit_requested:
-            print("ERROR: Unexpected termination")
+            fatal_errors.append("unexpected termination")
-            exit_ok = False
         if process.returncode != 0:
-            print(f"ERROR: openmw exited with code {process.returncode}")
+            fatal_errors.append(f"openmw exited with code {process.returncode}")
-            exit_ok = False
     if os.path.exists(config_dir / "openmw.log"):
-        shutil.copyfile(config_dir / "openmw.log", work_dir / f"{name}.{time_str}.log")
+        shutil.copyfile(config_dir / "openmw.log", work_dir / f"{test_name}.{time_str}.log")
-    if not exit_ok and not args.verbose:
+    if fatal_errors and not args.verbose:
         sys.stdout.writelines(stdout_lines)
-    if test_success and exit_ok:
+    total_duration = (time.time() - start) * 1000
-        print(f"{name} succeeded")
+    print(f'\n[----------] {count} tests from {test_name} ({total_duration:.3f} ms total)')
-    else:
+    print(f"[  PASSED  ] {count - len(failed_tests)} tests.")
-        print(f"{name} failed")
+    if fatal_errors:
-    return test_success and exit_ok
+        print(f"[  FAILED  ] fatal error: {'; '.join(fatal_errors)}")
+    if failed_tests:
+        print(f"[  FAILED  ] {len(failed_tests)} tests, listed below:")
+        for failed_test in failed_tests:
+            print(f"[  FAILED  ] {failed_test}")
+    return len(failed_tests) == 0 and not fatal_errors
 
 
 status = 0
 for entry in tests_dir.glob("test_*"):
     if entry.is_dir():
-        if not runTest(entry.name):
+        if not run_test(entry.name):
             status = -1
 if status == 0:
     shutil.rmtree(config_dir, ignore_errors=True)