From c07304bccba3037d6f032edf6c7006d1b5cc5d2f Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Sun, 21 Dec 2025 01:51:13 +0000 Subject: [PATCH 01/17] See what happens if we split the Windows job again --- .gitlab-ci.yml | 99 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 92 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ea831feee4..47ee08b3a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -650,6 +650,61 @@ macOS15_Xcode16_arm64: aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp --recursive --exclude '*' --include '*.ex_' --include '*.dl_' --include '*.pd_' sym_store s3://openmw-sym fi +.Merge_Artifacts_Base: + extends: .Ubuntu_Image + stage: build + variables: + GIT_STRATEGY: none + script: + - apt-get update + - apt-get install -y curl unzip zip + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o awscli-exe-linux-x86_64.zip + - unzip -d awscli-exe-linux-x86_64 awscli-exe-linux-x86_64.zip + - pushd awscli-exe-linux-x86_64 + - ./aws/install + - popd + - aws --version + - mkdir -p incoming_artifacts + - mv *.zip incoming_artifacts/ + - destinations=() + - | + # todo: this will get upset by certain characters + for merge_list in *_to-be-merged.txt; do + while IFS=: read -r partial destination; do + destinations+=("$destination") + echo "Unzipping '$partial' to '$destination'" + unzip -d "$destination" "incoming_artifacts/$partial" + done < merge_list + done + - | + for destination in ${destinations[@]}; do + pushd "$destination" + for ci_id in CI-ID_*.txt; do + cat "$ci_id" >> CI-ID.txt + rm "$ci_id" + done + echo "Creating $destination.zip" + zip -r "../$destination.zip" . + popd + done + - | + if [[ -v AWS_ACCESS_KEY_ID ]]; then + artifactDirectory="$CI_PROJECT_NAMESPACE/$CI_COMMIT_REF_NAME/$CI_COMMIT_SHORT_SHA-$CI_JOB_ID/" + aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp *.zip s3://openmw-artifacts/$artifactDirectory + fi + artifacts: + when: always + paths: + - "*.zip" + +variables: &target-group-one + targets: "bsatool components-tests esmtool niftest openmw openmw_detournavigator_navmeshtilescache_benchmark openmw_esm_refid_benchmark openmw_settings_access_benchmark openmw_bulletobjecttool openmw-essimporter openmw-iniimporter openmw-navmeshtool openmw-tests" + group_name: "group-one" + +variables: &target-group-two + targets: "openmw-cs openmw-launcher openmw-wizard" + group_name: "group-two" + .Windows_Ninja_Base: tags: - saas-windows-medium-amd64 @@ -862,26 +917,28 @@ macOS15_Xcode16_arm64: - aws --version - | if (Get-ChildItem -Recurse *.pdb) { - 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" '*.pdb' CI-ID.txt + 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" '*.pdb' CI-ID.txt if(!$?) { Exit $LASTEXITCODE } if (Test-Path env:AWS_ACCESS_KEY_ID) { - aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" s3://openmw-artifacts/${artifactDirectory} + aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" s3://openmw-artifacts/${artifactDirectory} if(!$?) { Exit $LASTEXITCODE } } Push-Location .. ..\CI\Store-Symbols.ps1 -SkipCompress if(!$?) { Exit $LASTEXITCODE } - 7z a -tzip "..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_sym_store.zip"))" '.\SymStore\*' $config\CI-ID.txt + 7z a -tzip "..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_sym_store.zip"))" '.\SymStore\*' $config\CI-ID.txt if(!$?) { Exit $LASTEXITCODE } Pop-Location Get-ChildItem -Recurse *.pdb | Remove-Item } - - 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}.zip"))" '*' + - Rename-Item CI-ID.txt CI-ID_${group_name}.txt + - 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip"))" '*' - | if (Test-Path env:AWS_ACCESS_KEY_ID) { - aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}.zip"))" s3://openmw-artifacts/${artifactDirectory} + aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip"))" s3://openmw-artifacts/${artifactDirectory} if(!$?) { Exit $LASTEXITCODE } } + - echo "$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip")):$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}"))" | Out-File -Encoding UTF8 "..\..\${group_name}_to-be-merged.txt" - | if ($executables) { foreach ($exe in $executables.Split(',')) { @@ -904,6 +961,7 @@ macOS15_Xcode16_arm64: - "*.log" - MSVC2022_64/*.log - MSVC2022_64/**/*.log + - "*_to-be-merged.txt" variables: targets: ALL_BUILD # When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks. @@ -936,7 +994,7 @@ macOS15_Xcode16_arm64: - job: "Windows_MSBuild_Debug" artifacts: true -Windows_MSBuild_RelWithDebInfo: +.Windows_MSBuild_RelWithDebInfo: extends: - .Windows_MSBuild_Base variables: @@ -948,11 +1006,38 @@ Windows_MSBuild_RelWithDebInfo: # run this for both pushes and schedules so 'latest successful pipeline for branch' always includes it - if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "schedule" +Windows_MSBuild_RelWithDebInfo_GroupOne: + extends: + - .Windows_MSBuild_RelWithDebInfo + variables: + <<: *target-group-one + +Windows_MSBuild_RelWithDebInfo_GroupTwo: + extends: + - .Windows_MSBuild_RelWithDebInfo + variables: + <<: *target-group-two + Windows_Compress_And_Upload_Symbols_MSBuild_RelWithDebInfo: extends: - .Compress_And_Upload_Symbols_Base needs: - - job: "Windows_MSBuild_RelWithDebInfo" + - job: "Windows_MSBuild_RelWithDebInfo_GroupOne" + artifacts: true + - job: "Windows_MSBuild_RelWithDebInfo_GroupTwo" + artifacts: true + # temporarily enabled while we're linking the above on the downloads page + rules: + # run this for both pushes and schedules so 'latest successful pipeline for branch' always includes it + - if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "schedule" + +Windows_Merge_Artifacts_MSBuild_RelWithDebInfo: + extends: + - .Merge_Artifacts_Base + needs: + - job: "Windows_MSBuild_RelWithDebInfo_GroupOne" + artifacts: true + - job: "Windows_MSBuild_RelWithDebInfo_GroupTwo" artifacts: true # temporarily enabled while we're linking the above on the downloads page rules: From b1bc22d24a23bd409a66abf2466d36a40a51df24 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Sun, 21 Dec 2025 02:15:29 +0000 Subject: [PATCH 02/17] --targets needs splat operator --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 47ee08b3a5..ca217ba79a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -749,7 +749,7 @@ variables: &target-group-two - Get-Volume - cd MSVC2022_64_Ninja - .\ActivateMSVC.ps1 - - cmake --build . --config $config --target $targets + - cmake --build . --config $config --target @targets - ccache --show-stats -v - cd $config - echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`nCI_JOB_ID ${CI_JOB_ID}`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt @@ -909,7 +909,7 @@ variables: &target-group-two - sh CI/before_script.msvc.sh -c $config -p Win64 -v 2022 -k -V -b -t -C $multiview -E - cd MSVC2022_64 - Get-Volume - - cmake --build . --config $config --target $targets + - cmake --build . --config $config --target @targets - cd $config - echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`nCI_JOB_ID ${CI_JOB_ID}`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt - $artifactDirectory = "$(Make-SafeFileName("${CI_PROJECT_NAMESPACE}"))/$(Make-SafeFileName("${CI_COMMIT_REF_NAME}"))/$(Make-SafeFileName("${CI_COMMIT_SHORT_SHA}-${CI_JOB_ID}"))/" From bff53fb37c36406ec4d2f66430301e87bf6f5a8e Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Sun, 21 Dec 2025 02:41:03 +0000 Subject: [PATCH 03/17] Can't splat, as it's a string, not an array Go back to the approach we used before https://gitlab.com/OpenMW/openmw/-/merge_requests/2012 was merged. We know that worked. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca217ba79a..1ef595df8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -749,7 +749,7 @@ variables: &target-group-two - Get-Volume - cd MSVC2022_64_Ninja - .\ActivateMSVC.ps1 - - cmake --build . --config $config --target @targets + - cmake --build . --config $config --target ($targets.Split(' ')) - ccache --show-stats -v - cd $config - echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`nCI_JOB_ID ${CI_JOB_ID}`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt @@ -909,7 +909,7 @@ variables: &target-group-two - sh CI/before_script.msvc.sh -c $config -p Win64 -v 2022 -k -V -b -t -C $multiview -E - cd MSVC2022_64 - Get-Volume - - cmake --build . --config $config --target @targets + - cmake --build . --config $config --target ($targets.Split(' ')) - cd $config - echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`nCI_JOB_ID ${CI_JOB_ID}`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt - $artifactDirectory = "$(Make-SafeFileName("${CI_PROJECT_NAMESPACE}"))/$(Make-SafeFileName("${CI_COMMIT_REF_NAME}"))/$(Make-SafeFileName("${CI_COMMIT_SHORT_SHA}-${CI_JOB_ID}"))/" From a79e5fcee6d5062a839a5e98469bf8ec8416d85d Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Sun, 21 Dec 2025 16:30:43 +0000 Subject: [PATCH 04/17] Missing targets --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ef595df8e..4de23959e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -698,11 +698,11 @@ macOS15_Xcode16_arm64: - "*.zip" variables: &target-group-one - targets: "bsatool components-tests esmtool niftest openmw openmw_detournavigator_navmeshtilescache_benchmark openmw_esm_refid_benchmark openmw_settings_access_benchmark openmw_bulletobjecttool openmw-essimporter openmw-iniimporter openmw-navmeshtool openmw-tests" + targets: "bsatool components-tests esmtool niftest openmw openmw_detournavigator_navmeshtilescache_benchmark openmw_esm_refid_benchmark openmw_settings_access_benchmark openmw-bulletobjecttool openmw-essimporter openmw-iniimporter openmw-navmeshtool openmw-tests" group_name: "group-one" variables: &target-group-two - targets: "openmw-cs openmw-launcher openmw-wizard" + targets: "openmw-cs openmw-cs-tests openmw-launcher openmw-wizard" group_name: "group-two" .Windows_Ninja_Base: From 93c3618121f4c87fc99716368cf279723a484572 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Mon, 22 Dec 2025 01:54:38 +0000 Subject: [PATCH 05/17] Stop docker Maybe this will give us more memory to work with. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4de23959e8..b314e967f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -712,6 +712,7 @@ variables: &target-group-two - if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event" before_script: - Get-Volume + - Stop-Service docker - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - choco source add -n=openmw-proxy -s="https://repo.openmw.org/repository/Chocolatey/" --priority=1 - choco source disable -n=chocolatey @@ -878,6 +879,7 @@ variables: &target-group-two - if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event" before_script: - Get-Volume + - Stop-Service docker - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - choco source add -n=openmw-proxy -s="https://repo.openmw.org/repository/Chocolatey/" --priority=1 - choco source disable -n=chocolatey From f884c26631d5af3f3eeff9d9e7cd127730a35847 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Mon, 22 Dec 2025 17:42:50 +0000 Subject: [PATCH 06/17] Maybe an anonymous namespace reduces RAM requirements This file is the most likely to make the compiler run out of heap space --- apps/openmw/mwlua/stats.cpp | 811 ++++++++++++++++++------------------ 1 file changed, 410 insertions(+), 401 deletions(-) diff --git a/apps/openmw/mwlua/stats.cpp b/apps/openmw/mwlua/stats.cpp index e4ee31d411..93cd8c294e 100644 --- a/apps/openmw/mwlua/stats.cpp +++ b/apps/openmw/mwlua/stats.cpp @@ -62,428 +62,437 @@ namespace namespace MWLua { - static void addStatUpdateAction(MWLua::LuaManager* manager, const SelfObject& obj) + namespace { - if (!obj.mStatsCache.empty()) - return; // was already added before - manager->addAction( - [obj = Object(obj)] { - LocalScripts* scripts = obj.ptr().getRefData().getLuaScripts(); - if (scripts) - scripts->applyStatsCache(); - }, - "StatUpdateAction"); - } - - static void setCreatureValue(Index, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) - { - auto& stats = ptr.getClass().getCreatureStats(ptr); - if (prop == "current") - stats.setLevel(LuaUtil::cast(value)); - } - - static void setNpcValue(Index index, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) - { - auto& stats = ptr.getClass().getNpcStats(ptr); - if (prop == "progress") - stats.setLevelProgress(LuaUtil::cast(value)); - else if (prop == "skillIncreasesForAttribute") - stats.setSkillIncreasesForAttribute( - *std::get(index).getIf(), LuaUtil::cast(value)); - else if (prop == "skillIncreasesForSpecialization") - stats.setSkillIncreasesForSpecialization( - static_cast(std::get(index)), LuaUtil::cast(value)); - } - - class SkillIncreasesForAttributeStats - { - ObjectVariant mObject; - - public: - SkillIncreasesForAttributeStats(ObjectVariant object) - : mObject(std::move(object)) + static void addStatUpdateAction(MWLua::LuaManager* manager, const SelfObject& obj) { + if (!obj.mStatsCache.empty()) + return; // was already added before + manager->addAction( + [obj = Object(obj)] { + LocalScripts* scripts = obj.ptr().getRefData().getLuaScripts(); + if (scripts) + scripts->applyStatsCache(); + }, + "StatUpdateAction"); } - sol::object get(const Context& context, ESM::StringRefId attributeId) const + static void setCreatureValue(Index, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) { - if (!mObject.ptr().getClass().isNpc()) - return sol::nil; - - return getValue(context, mObject, &setNpcValue, attributeId, "skillIncreasesForAttribute", - [attributeId](const MWWorld::Ptr& ptr) { - return ptr.getClass().getNpcStats(ptr).getSkillIncreasesForAttribute(attributeId); - }); - } - - void set(const Context& context, ESM::StringRefId attributeId, const sol::object& value) const - { - const auto& ptr = mObject.ptr(); - if (!ptr.getClass().isNpc()) - return; - - SelfObject* obj = mObject.asSelfObject(); - addStatUpdateAction(context.mLuaManager, *obj); - obj->mStatsCache[SelfObject::CachedStat{ &setNpcValue, attributeId, "skillIncreasesForAttribute" }] - = sol::main_object(value); - } - }; - - class SkillIncreasesForSpecializationStats - { - ObjectVariant mObject; - - public: - SkillIncreasesForSpecializationStats(ObjectVariant object) - : mObject(std::move(object)) - { - } - - sol::object get(const Context& context, int specialization) const - { - if (!mObject.ptr().getClass().isNpc()) - return sol::nil; - - return getValue(context, mObject, &setNpcValue, specialization, "skillIncreasesForSpecialization", - [specialization](const MWWorld::Ptr& ptr) { - return ptr.getClass().getNpcStats(ptr).getSkillIncreasesForSpecialization( - static_cast(specialization)); - }); - } - - void set(const Context& context, int specialization, const sol::object& value) const - { - const auto& ptr = mObject.ptr(); - if (!ptr.getClass().isNpc()) - return; - - SelfObject* obj = mObject.asSelfObject(); - addStatUpdateAction(context.mLuaManager, *obj); - obj->mStatsCache[SelfObject::CachedStat{ &setNpcValue, specialization, "skillIncreasesForSpecialization" }] - = sol::main_object(value); - } - }; - - class LevelStat - { - ObjectVariant mObject; - - LevelStat(ObjectVariant object) - : mObject(std::move(object)) - { - } - - public: - sol::object getCurrent(const Context& context) const - { - return getValue(context, mObject, &setCreatureValue, std::monostate{}, "current", - [](const MWWorld::Ptr& ptr) { return ptr.getClass().getCreatureStats(ptr).getLevel(); }); - } - - void setCurrent(const Context& context, const sol::object& value) const - { - SelfObject* obj = mObject.asSelfObject(); - addStatUpdateAction(context.mLuaManager, *obj); - obj->mStatsCache[SelfObject::CachedStat{ &setCreatureValue, std::monostate{}, "current" }] - = sol::main_object(value); - } - - sol::object getProgress(const Context& context) const - { - if (!mObject.ptr().getClass().isNpc()) - return sol::nil; - - return getValue(context, mObject, &setNpcValue, std::monostate{}, "progress", - [](const MWWorld::Ptr& ptr) { return ptr.getClass().getNpcStats(ptr).getLevelProgress(); }); - } - - void setProgress(const Context& context, const sol::object& value) const - { - const auto& ptr = mObject.ptr(); - if (!ptr.getClass().isNpc()) - return; - - SelfObject* obj = mObject.asSelfObject(); - addStatUpdateAction(context.mLuaManager, *obj); - obj->mStatsCache[SelfObject::CachedStat{ &setNpcValue, std::monostate{}, "progress" }] - = sol::main_object(value); - } - - SkillIncreasesForAttributeStats getSkillIncreasesForAttributeStats() const - { - return SkillIncreasesForAttributeStats{ mObject }; - } - - SkillIncreasesForSpecializationStats getSkillIncreasesForSpecializationStats() const - { - return SkillIncreasesForSpecializationStats{ mObject }; - } - - static std::optional create(ObjectVariant object, Index) - { - if (!object.ptr().getClass().isActor()) - return {}; - return LevelStat{ std::move(object) }; - } - }; - - class DynamicStat - { - ObjectVariant mObject; - int mIndex; - - DynamicStat(ObjectVariant object, int index) - : mObject(std::move(object)) - , mIndex(index) - { - } - - public: - template - sol::object get(const Context& context, std::string_view prop, G getter) const - { - return getValue( - context, mObject, &DynamicStat::setValue, mIndex, prop, [this, getter](const MWWorld::Ptr& ptr) { - return (ptr.getClass().getCreatureStats(ptr).getDynamic(mIndex).*getter)(); - }); - } - - static std::optional create(ObjectVariant object, Index i) - { - if (!object.ptr().getClass().isActor()) - return {}; - int index = std::get(i); - return DynamicStat{ std::move(object), index }; - } - - void cache(const Context& context, std::string_view prop, const sol::object& value) const - { - SelfObject* obj = mObject.asSelfObject(); - addStatUpdateAction(context.mLuaManager, *obj); - obj->mStatsCache[SelfObject::CachedStat{ &DynamicStat::setValue, mIndex, prop }] = sol::main_object(value); - } - - static void setValue(Index i, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) - { - int index = std::get(i); auto& stats = ptr.getClass().getCreatureStats(ptr); - auto stat = stats.getDynamic(index); - float floatValue = LuaUtil::cast(value); - if (prop == "base") - stat.setBase(floatValue); - else if (prop == "current") - stat.setCurrent(floatValue, true, true); - else if (prop == "modifier") - stat.setModifier(floatValue); - stats.setDynamic(index, stat); - } - }; - - class AttributeStat - { - ObjectVariant mObject; - ESM::RefId mId; - - AttributeStat(ObjectVariant object, ESM::RefId id) - : mObject(std::move(object)) - , mId(id) - { + if (prop == "current") + stats.setLevel(LuaUtil::cast(value)); } - public: - template - sol::object get(const Context& context, std::string_view prop, G getter) const + static void setNpcValue(Index index, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) { - return getValue( - context, mObject, &AttributeStat::setValue, mId, prop, [this, getter](const MWWorld::Ptr& ptr) { - return (ptr.getClass().getCreatureStats(ptr).getAttribute(mId).*getter)(); - }); - } - - float getModified(const Context& context) const - { - auto base = LuaUtil::cast(get(context, "base", &MWMechanics::AttributeValue::getBase)); - auto damage = LuaUtil::cast(get(context, "damage", &MWMechanics::AttributeValue::getDamage)); - auto modifier = LuaUtil::cast(get(context, "modifier", &MWMechanics::AttributeValue::getModifier)); - return std::max(0.f, base - damage + modifier); // Should match AttributeValue::getModified - } - - static std::optional create(ObjectVariant object, Index i) - { - if (!object.ptr().getClass().isActor()) - return {}; - ESM::RefId id = std::get(i); - return AttributeStat{ std::move(object), id }; - } - - void cache(const Context& context, std::string_view prop, const sol::object& value) const - { - SelfObject* obj = mObject.asSelfObject(); - addStatUpdateAction(context.mLuaManager, *obj); - obj->mStatsCache[SelfObject::CachedStat{ &AttributeStat::setValue, mId, prop }] = sol::main_object(value); - } - - static void setValue(Index i, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) - { - ESM::RefId id = std::get(i); - auto& stats = ptr.getClass().getCreatureStats(ptr); - auto stat = stats.getAttribute(id); - float floatValue = LuaUtil::cast(value); - if (prop == "base") - stat.setBase(floatValue); - else if (prop == "damage") - { - stat.restore(stat.getDamage()); - stat.damage(floatValue); - } - else if (prop == "modifier") - stat.setModifier(floatValue); - stats.setAttribute(id, stat); - } - }; - - class SkillStat - { - ObjectVariant mObject; - ESM::RefId mId; - - SkillStat(ObjectVariant object, ESM::RefId id) - : mObject(std::move(object)) - , mId(id) - { - } - - static float getProgress(const MWWorld::Ptr& ptr, ESM::RefId id, const MWMechanics::SkillValue& stat) - { - float progress = stat.getProgress(); - if (progress != 0.f) - progress /= getMaxProgress(ptr, id, stat); - return progress; - } - - static float getMaxProgress(const MWWorld::Ptr& ptr, ESM::RefId id, const MWMechanics::SkillValue& stat) - { - const auto& store = *MWBase::Environment::get().getESMStore(); - const auto cl = store.get().find(ptr.get()->mBase->mClass); - return ptr.getClass().getNpcStats(ptr).getSkillProgressRequirement(id, *cl); - } - - public: - template - sol::object get(const Context& context, std::string_view prop, G getter) const - { - return getValue(context, mObject, &SkillStat::setValue, mId, prop, [this, getter](const MWWorld::Ptr& ptr) { - return (ptr.getClass().getNpcStats(ptr).getSkill(mId).*getter)(); - }); - } - - float getModified(const Context& context) const - { - auto base = LuaUtil::cast(get(context, "base", &MWMechanics::SkillValue::getBase)); - auto damage = LuaUtil::cast(get(context, "damage", &MWMechanics::SkillValue::getDamage)); - auto modifier = LuaUtil::cast(get(context, "modifier", &MWMechanics::SkillValue::getModifier)); - return std::max(0.f, base - damage + modifier); // Should match SkillValue::getModified - } - - sol::object getProgress(const Context& context) const - { - return getValue(context, mObject, &SkillStat::setValue, mId, "progress", [this](const MWWorld::Ptr& ptr) { - return getProgress(ptr, mId, ptr.getClass().getNpcStats(ptr).getSkill(mId)); - }); - } - - static std::optional create(ObjectVariant object, Index index) - { - if (!object.ptr().getClass().isNpc()) - return {}; - ESM::RefId id = std::get(index); - return SkillStat{ std::move(object), id }; - } - - void cache(const Context& context, std::string_view prop, const sol::object& value) const - { - SelfObject* obj = mObject.asSelfObject(); - addStatUpdateAction(context.mLuaManager, *obj); - obj->mStatsCache[SelfObject::CachedStat{ &SkillStat::setValue, mId, prop }] = sol::main_object(value); - } - - static void setValue(Index index, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) - { - ESM::RefId id = std::get(index); auto& stats = ptr.getClass().getNpcStats(ptr); - auto stat = stats.getSkill(id); - float floatValue = LuaUtil::cast(value); - if (prop == "base") - stat.setBase(floatValue); - else if (prop == "damage") + if (prop == "progress") + stats.setLevelProgress(LuaUtil::cast(value)); + else if (prop == "skillIncreasesForAttribute") + stats.setSkillIncreasesForAttribute( + *std::get(index).getIf(), LuaUtil::cast(value)); + else if (prop == "skillIncreasesForSpecialization") + stats.setSkillIncreasesForSpecialization( + static_cast(std::get(index)), LuaUtil::cast(value)); + } + + class SkillIncreasesForAttributeStats + { + ObjectVariant mObject; + + public: + SkillIncreasesForAttributeStats(ObjectVariant object) + : mObject(std::move(object)) { - stat.restore(stat.getDamage()); - stat.damage(floatValue); } - else if (prop == "modifier") - stat.setModifier(floatValue); - else if (prop == "progress") - stat.setProgress(floatValue * getMaxProgress(ptr, id, stat)); - stats.setSkill(id, stat); - } - }; - class AIStat - { - ObjectVariant mObject; - MWMechanics::AiSetting mIndex; + sol::object get(const Context& context, ESM::StringRefId attributeId) const + { + if (!mObject.ptr().getClass().isNpc()) + return sol::nil; - AIStat(ObjectVariant object, MWMechanics::AiSetting index) - : mObject(std::move(object)) - , mIndex(index) + return getValue(context, mObject, &setNpcValue, attributeId, "skillIncreasesForAttribute", + [attributeId](const MWWorld::Ptr& ptr) { + return ptr.getClass().getNpcStats(ptr).getSkillIncreasesForAttribute(attributeId); + }); + } + + void set(const Context& context, ESM::StringRefId attributeId, const sol::object& value) const + { + const auto& ptr = mObject.ptr(); + if (!ptr.getClass().isNpc()) + return; + + SelfObject* obj = mObject.asSelfObject(); + addStatUpdateAction(context.mLuaManager, *obj); + obj->mStatsCache[SelfObject::CachedStat{ &setNpcValue, attributeId, "skillIncreasesForAttribute" }] + = sol::main_object(value); + } + }; + + class SkillIncreasesForSpecializationStats { - } + ObjectVariant mObject; - public: - template - sol::object get(const Context& context, std::string_view prop, G getter) const - { - return getValue(context, mObject, &AIStat::setValue, static_cast(mIndex), prop, - [this, getter](const MWWorld::Ptr& ptr) { - return (ptr.getClass().getCreatureStats(ptr).getAiSetting(mIndex).*getter)(); - }); - } + public: + SkillIncreasesForSpecializationStats(ObjectVariant object) + : mObject(std::move(object)) + { + } - int getModified(const Context& context) const - { - auto base = LuaUtil::cast(get(context, "base", &MWMechanics::Stat::getBase)); - auto modifier = LuaUtil::cast(get(context, "modifier", &MWMechanics::Stat::getModifier)); - return std::max(0, base + modifier); - } + sol::object get(const Context& context, int specialization) const + { + if (!mObject.ptr().getClass().isNpc()) + return sol::nil; - static std::optional create(ObjectVariant object, MWMechanics::AiSetting index) - { - if (!object.ptr().getClass().isActor()) - return {}; - return AIStat{ std::move(object), index }; - } + return getValue(context, mObject, &setNpcValue, specialization, "skillIncreasesForSpecialization", + [specialization](const MWWorld::Ptr& ptr) { + return ptr.getClass().getNpcStats(ptr).getSkillIncreasesForSpecialization( + static_cast(specialization)); + }); + } - void cache(const Context& context, std::string_view prop, const sol::object& value) const - { - SelfObject* obj = mObject.asSelfObject(); - addStatUpdateAction(context.mLuaManager, *obj); - obj->mStatsCache[SelfObject::CachedStat{ &AIStat::setValue, static_cast(mIndex), prop }] - = sol::main_object(value); - } + void set(const Context& context, int specialization, const sol::object& value) const + { + const auto& ptr = mObject.ptr(); + if (!ptr.getClass().isNpc()) + return; - static void setValue(Index i, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) + SelfObject* obj = mObject.asSelfObject(); + addStatUpdateAction(context.mLuaManager, *obj); + obj->mStatsCache[SelfObject::CachedStat{ + &setNpcValue, specialization, "skillIncreasesForSpecialization" }] + = sol::main_object(value); + } + }; + + class LevelStat { - auto index = static_cast(std::get(i)); - auto& stats = ptr.getClass().getCreatureStats(ptr); - auto stat = stats.getAiSetting(index); - int intValue = LuaUtil::cast(value); - if (prop == "base") - stat.setBase(intValue); - else if (prop == "modifier") - stat.setModifier(intValue); - stats.setAiSetting(index, stat); - } - }; + ObjectVariant mObject; + + LevelStat(ObjectVariant object) + : mObject(std::move(object)) + { + } + + public: + sol::object getCurrent(const Context& context) const + { + return getValue(context, mObject, &setCreatureValue, std::monostate{}, "current", + [](const MWWorld::Ptr& ptr) { return ptr.getClass().getCreatureStats(ptr).getLevel(); }); + } + + void setCurrent(const Context& context, const sol::object& value) const + { + SelfObject* obj = mObject.asSelfObject(); + addStatUpdateAction(context.mLuaManager, *obj); + obj->mStatsCache[SelfObject::CachedStat{ &setCreatureValue, std::monostate{}, "current" }] + = sol::main_object(value); + } + + sol::object getProgress(const Context& context) const + { + if (!mObject.ptr().getClass().isNpc()) + return sol::nil; + + return getValue(context, mObject, &setNpcValue, std::monostate{}, "progress", + [](const MWWorld::Ptr& ptr) { return ptr.getClass().getNpcStats(ptr).getLevelProgress(); }); + } + + void setProgress(const Context& context, const sol::object& value) const + { + const auto& ptr = mObject.ptr(); + if (!ptr.getClass().isNpc()) + return; + + SelfObject* obj = mObject.asSelfObject(); + addStatUpdateAction(context.mLuaManager, *obj); + obj->mStatsCache[SelfObject::CachedStat{ &setNpcValue, std::monostate{}, "progress" }] + = sol::main_object(value); + } + + SkillIncreasesForAttributeStats getSkillIncreasesForAttributeStats() const + { + return SkillIncreasesForAttributeStats{ mObject }; + } + + SkillIncreasesForSpecializationStats getSkillIncreasesForSpecializationStats() const + { + return SkillIncreasesForSpecializationStats{ mObject }; + } + + static std::optional create(ObjectVariant object, Index) + { + if (!object.ptr().getClass().isActor()) + return {}; + return LevelStat{ std::move(object) }; + } + }; + + class DynamicStat + { + ObjectVariant mObject; + int mIndex; + + DynamicStat(ObjectVariant object, int index) + : mObject(std::move(object)) + , mIndex(index) + { + } + + public: + template + sol::object get(const Context& context, std::string_view prop, G getter) const + { + return getValue( + context, mObject, &DynamicStat::setValue, mIndex, prop, [this, getter](const MWWorld::Ptr& ptr) { + return (ptr.getClass().getCreatureStats(ptr).getDynamic(mIndex).*getter)(); + }); + } + + static std::optional create(ObjectVariant object, Index i) + { + if (!object.ptr().getClass().isActor()) + return {}; + int index = std::get(i); + return DynamicStat{ std::move(object), index }; + } + + void cache(const Context& context, std::string_view prop, const sol::object& value) const + { + SelfObject* obj = mObject.asSelfObject(); + addStatUpdateAction(context.mLuaManager, *obj); + obj->mStatsCache[SelfObject::CachedStat{ &DynamicStat::setValue, mIndex, prop }] + = sol::main_object(value); + } + + static void setValue(Index i, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) + { + int index = std::get(i); + auto& stats = ptr.getClass().getCreatureStats(ptr); + auto stat = stats.getDynamic(index); + float floatValue = LuaUtil::cast(value); + if (prop == "base") + stat.setBase(floatValue); + else if (prop == "current") + stat.setCurrent(floatValue, true, true); + else if (prop == "modifier") + stat.setModifier(floatValue); + stats.setDynamic(index, stat); + } + }; + + class AttributeStat + { + ObjectVariant mObject; + ESM::RefId mId; + + AttributeStat(ObjectVariant object, ESM::RefId id) + : mObject(std::move(object)) + , mId(id) + { + } + + public: + template + sol::object get(const Context& context, std::string_view prop, G getter) const + { + return getValue( + context, mObject, &AttributeStat::setValue, mId, prop, [this, getter](const MWWorld::Ptr& ptr) { + return (ptr.getClass().getCreatureStats(ptr).getAttribute(mId).*getter)(); + }); + } + + float getModified(const Context& context) const + { + auto base = LuaUtil::cast(get(context, "base", &MWMechanics::AttributeValue::getBase)); + auto damage = LuaUtil::cast(get(context, "damage", &MWMechanics::AttributeValue::getDamage)); + auto modifier + = LuaUtil::cast(get(context, "modifier", &MWMechanics::AttributeValue::getModifier)); + return std::max(0.f, base - damage + modifier); // Should match AttributeValue::getModified + } + + static std::optional create(ObjectVariant object, Index i) + { + if (!object.ptr().getClass().isActor()) + return {}; + ESM::RefId id = std::get(i); + return AttributeStat{ std::move(object), id }; + } + + void cache(const Context& context, std::string_view prop, const sol::object& value) const + { + SelfObject* obj = mObject.asSelfObject(); + addStatUpdateAction(context.mLuaManager, *obj); + obj->mStatsCache[SelfObject::CachedStat{ &AttributeStat::setValue, mId, prop }] + = sol::main_object(value); + } + + static void setValue(Index i, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) + { + ESM::RefId id = std::get(i); + auto& stats = ptr.getClass().getCreatureStats(ptr); + auto stat = stats.getAttribute(id); + float floatValue = LuaUtil::cast(value); + if (prop == "base") + stat.setBase(floatValue); + else if (prop == "damage") + { + stat.restore(stat.getDamage()); + stat.damage(floatValue); + } + else if (prop == "modifier") + stat.setModifier(floatValue); + stats.setAttribute(id, stat); + } + }; + + class SkillStat + { + ObjectVariant mObject; + ESM::RefId mId; + + SkillStat(ObjectVariant object, ESM::RefId id) + : mObject(std::move(object)) + , mId(id) + { + } + + static float getProgress(const MWWorld::Ptr& ptr, ESM::RefId id, const MWMechanics::SkillValue& stat) + { + float progress = stat.getProgress(); + if (progress != 0.f) + progress /= getMaxProgress(ptr, id, stat); + return progress; + } + + static float getMaxProgress(const MWWorld::Ptr& ptr, ESM::RefId id, const MWMechanics::SkillValue& stat) + { + const auto& store = *MWBase::Environment::get().getESMStore(); + const auto cl = store.get().find(ptr.get()->mBase->mClass); + return ptr.getClass().getNpcStats(ptr).getSkillProgressRequirement(id, *cl); + } + + public: + template + sol::object get(const Context& context, std::string_view prop, G getter) const + { + return getValue( + context, mObject, &SkillStat::setValue, mId, prop, [this, getter](const MWWorld::Ptr& ptr) { + return (ptr.getClass().getNpcStats(ptr).getSkill(mId).*getter)(); + }); + } + + float getModified(const Context& context) const + { + auto base = LuaUtil::cast(get(context, "base", &MWMechanics::SkillValue::getBase)); + auto damage = LuaUtil::cast(get(context, "damage", &MWMechanics::SkillValue::getDamage)); + auto modifier = LuaUtil::cast(get(context, "modifier", &MWMechanics::SkillValue::getModifier)); + return std::max(0.f, base - damage + modifier); // Should match SkillValue::getModified + } + + sol::object getProgress(const Context& context) const + { + return getValue( + context, mObject, &SkillStat::setValue, mId, "progress", [this](const MWWorld::Ptr& ptr) { + return getProgress(ptr, mId, ptr.getClass().getNpcStats(ptr).getSkill(mId)); + }); + } + + static std::optional create(ObjectVariant object, Index index) + { + if (!object.ptr().getClass().isNpc()) + return {}; + ESM::RefId id = std::get(index); + return SkillStat{ std::move(object), id }; + } + + void cache(const Context& context, std::string_view prop, const sol::object& value) const + { + SelfObject* obj = mObject.asSelfObject(); + addStatUpdateAction(context.mLuaManager, *obj); + obj->mStatsCache[SelfObject::CachedStat{ &SkillStat::setValue, mId, prop }] = sol::main_object(value); + } + + static void setValue(Index index, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) + { + ESM::RefId id = std::get(index); + auto& stats = ptr.getClass().getNpcStats(ptr); + auto stat = stats.getSkill(id); + float floatValue = LuaUtil::cast(value); + if (prop == "base") + stat.setBase(floatValue); + else if (prop == "damage") + { + stat.restore(stat.getDamage()); + stat.damage(floatValue); + } + else if (prop == "modifier") + stat.setModifier(floatValue); + else if (prop == "progress") + stat.setProgress(floatValue * getMaxProgress(ptr, id, stat)); + stats.setSkill(id, stat); + } + }; + + class AIStat + { + ObjectVariant mObject; + MWMechanics::AiSetting mIndex; + + AIStat(ObjectVariant object, MWMechanics::AiSetting index) + : mObject(std::move(object)) + , mIndex(index) + { + } + + public: + template + sol::object get(const Context& context, std::string_view prop, G getter) const + { + return getValue(context, mObject, &AIStat::setValue, static_cast(mIndex), prop, + [this, getter](const MWWorld::Ptr& ptr) { + return (ptr.getClass().getCreatureStats(ptr).getAiSetting(mIndex).*getter)(); + }); + } + + int getModified(const Context& context) const + { + auto base = LuaUtil::cast(get(context, "base", &MWMechanics::Stat::getBase)); + auto modifier = LuaUtil::cast(get(context, "modifier", &MWMechanics::Stat::getModifier)); + return std::max(0, base + modifier); + } + + static std::optional create(ObjectVariant object, MWMechanics::AiSetting index) + { + if (!object.ptr().getClass().isActor()) + return {}; + return AIStat{ std::move(object), index }; + } + + void cache(const Context& context, std::string_view prop, const sol::object& value) const + { + SelfObject* obj = mObject.asSelfObject(); + addStatUpdateAction(context.mLuaManager, *obj); + obj->mStatsCache[SelfObject::CachedStat{ &AIStat::setValue, static_cast(mIndex), prop }] + = sol::main_object(value); + } + + static void setValue(Index i, std::string_view prop, const MWWorld::Ptr& ptr, const sol::object& value) + { + auto index = static_cast(std::get(i)); + auto& stats = ptr.getClass().getCreatureStats(ptr); + auto stat = stats.getAiSetting(index); + int intValue = LuaUtil::cast(value); + if (prop == "base") + stat.setBase(intValue); + else if (prop == "modifier") + stat.setModifier(intValue); + stats.setAiSetting(index, stat); + } + }; + } } namespace sol From 136a4d02fc159719594889f41c802ddd9ce56418 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Mon, 22 Dec 2025 20:28:40 +0000 Subject: [PATCH 07/17] Missing $ operator --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b314e967f5..6e8dcc317b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -674,7 +674,7 @@ macOS15_Xcode16_arm64: destinations+=("$destination") echo "Unzipping '$partial' to '$destination'" unzip -d "$destination" "incoming_artifacts/$partial" - done < merge_list + done < $merge_list done - | for destination in ${destinations[@]}; do From f9f29c7a575dde6638bbbb15cd38e70089571e19 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Mon, 22 Dec 2025 20:32:19 +0000 Subject: [PATCH 08/17] unzip handles wildcards itself Otherwise, multiple paths are treated as one zip and several zip-relative paths to extract from it. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e8dcc317b..dd44e0e46d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -625,7 +625,7 @@ macOS15_Xcode16_arm64: - ./aws/install - popd - aws --version - - unzip -d sym_store *sym_store.zip + - unzip -d sym_store '*sym_store.zip' - shopt -s globstar - | for file in sym_store/**/*.exe; do From c23d8f64bc39bc55e2abd4e0bef0e4d7c747a547 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Wed, 24 Dec 2025 01:34:56 +0000 Subject: [PATCH 09/17] Disable multithreaded codegen for hungry TUs --- apps/openmw/CMakeLists.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/openmw/CMakeLists.txt b/apps/openmw/CMakeLists.txt index 8570d9df60..22956bf00f 100644 --- a/apps/openmw/CMakeLists.txt +++ b/apps/openmw/CMakeLists.txt @@ -123,6 +123,19 @@ add_library(openmw-lib STATIC ${OPENMW_SOURCES} ) +if (MSVC) + # these TUs take a lot of memory for codegen + # in CI, they often go over the 8GB memory limit if another cl.exe is running at the same time + # this option (make sure you're not looking at the docs for the identically-named linker option) stops four codegen threads being spun off + # it hurts build time ~33% to enable this unnecessarily, though, so target it to the hungriest boys + set_source_files_properties( + options.cpp + mwlua/stats.cpp + PROPERTIES + COMPILE_OPTIONS /cgthreads1 + ) +endif() + if(BUILD_OPENMW) if (ANDROID) add_library(openmw SHARED From fe9f0bbb460bd4f6aaab0244f01a36f62b93b755 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Wed, 24 Dec 2025 11:53:40 +0000 Subject: [PATCH 10/17] Skip duplicates without prompting Any file we want to keep has a unique name based on its contents. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dd44e0e46d..d953213aee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -625,7 +625,7 @@ macOS15_Xcode16_arm64: - ./aws/install - popd - aws --version - - unzip -d sym_store '*sym_store.zip' + - unzip -n -d sym_store '*sym_store.zip' - shopt -s globstar - | for file in sym_store/**/*.exe; do From 87dfcf302fb1d002d7266d5cb45cc5932c586761 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Wed, 24 Dec 2025 14:35:40 +0000 Subject: [PATCH 11/17] Fix artifact merging --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d953213aee..d8d66c5509 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -657,7 +657,7 @@ macOS15_Xcode16_arm64: GIT_STRATEGY: none script: - apt-get update - - apt-get install -y curl unzip zip + - apt-get install -y curl dos2unix unzip zip - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o awscli-exe-linux-x86_64.zip - unzip -d awscli-exe-linux-x86_64 awscli-exe-linux-x86_64.zip - pushd awscli-exe-linux-x86_64 @@ -670,11 +670,11 @@ macOS15_Xcode16_arm64: - | # todo: this will get upset by certain characters for merge_list in *_to-be-merged.txt; do - while IFS=: read -r partial destination; do + dos2unix < $merge_list | while IFS=: read -r partial destination; do destinations+=("$destination") echo "Unzipping '$partial' to '$destination'" - unzip -d "$destination" "incoming_artifacts/$partial" - done < $merge_list + unzip -n -d "$destination" "incoming_artifacts/$partial" + done done - | for destination in ${destinations[@]}; do From 3849f36b46e3bd645a87e54cc097cfc98da15938 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Wed, 24 Dec 2025 17:32:34 +0000 Subject: [PATCH 12/17] Don't run while loop in subshell We need the variable writes to stick around. Also switch to an associative array so duplicates are removed. --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d8d66c5509..3fe8cc4b7c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -666,18 +666,18 @@ macOS15_Xcode16_arm64: - aws --version - mkdir -p incoming_artifacts - mv *.zip incoming_artifacts/ - - destinations=() + - declare -A destinations - | # todo: this will get upset by certain characters for merge_list in *_to-be-merged.txt; do - dos2unix < $merge_list | while IFS=: read -r partial destination; do - destinations+=("$destination") + while IFS=: read -r partial destination; do + destinations["$destination"]="" echo "Unzipping '$partial' to '$destination'" unzip -n -d "$destination" "incoming_artifacts/$partial" - done + done < <(dos2unix < $merge_list) done - | - for destination in ${destinations[@]}; do + for destination in ${!destinations[@]}; do pushd "$destination" for ci_id in CI-ID_*.txt; do cat "$ci_id" >> CI-ID.txt From 1932258db979f10b07c414b4d967ca6b38913dc2 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Wed, 24 Dec 2025 17:42:04 +0000 Subject: [PATCH 13/17] Don't mix line endings This doesn't fix the BOM in the middle of the merged CI-ID.txt, but that shouldn't be happening in the first place. GitLab's docs say the default is pwsh rather than powershell (i.e. PowerShell >= 6, not 5.1), which defaults to non-BOM UTF-8, so there should be no BOM to contend with. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3fe8cc4b7c..4fb5851c15 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -753,7 +753,7 @@ variables: &target-group-two - cmake --build . --config $config --target ($targets.Split(' ')) - ccache --show-stats -v - cd $config - - echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`nCI_JOB_ID ${CI_JOB_ID}`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt + - echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`r`nCI_JOB_ID ${CI_JOB_ID}`r`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt - $artifactDirectory = "$(Make-SafeFileName("${CI_PROJECT_NAMESPACE}"))/$(Make-SafeFileName("${CI_COMMIT_REF_NAME}"))/$(Make-SafeFileName("${CI_COMMIT_SHORT_SHA}-${CI_JOB_ID}"))/" - Get-ChildItem -Recurse *.ilk | Remove-Item - aws --version @@ -913,7 +913,7 @@ variables: &target-group-two - Get-Volume - cmake --build . --config $config --target ($targets.Split(' ')) - cd $config - - echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`nCI_JOB_ID ${CI_JOB_ID}`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt + - echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`r`nCI_JOB_ID ${CI_JOB_ID}`r`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt - $artifactDirectory = "$(Make-SafeFileName("${CI_PROJECT_NAMESPACE}"))/$(Make-SafeFileName("${CI_COMMIT_REF_NAME}"))/$(Make-SafeFileName("${CI_COMMIT_SHORT_SHA}-${CI_JOB_ID}"))/" - Get-ChildItem -Recurse *.ilk | Remove-Item - aws --version From 6478e8c3e8efa945ff50459fb7c6fb11ed84957a Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Wed, 24 Dec 2025 20:05:44 +0000 Subject: [PATCH 14/17] Fix log timestamps Don't call the block variables because that hides the default variables block we set at the top. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4fb5851c15..f060b7613f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -697,11 +697,11 @@ macOS15_Xcode16_arm64: paths: - "*.zip" -variables: &target-group-one +.variables-for-split-jobs: &target-group-one targets: "bsatool components-tests esmtool niftest openmw openmw_detournavigator_navmeshtilescache_benchmark openmw_esm_refid_benchmark openmw_settings_access_benchmark openmw-bulletobjecttool openmw-essimporter openmw-iniimporter openmw-navmeshtool openmw-tests" group_name: "group-one" -variables: &target-group-two +.variables-for-split-jobs: &target-group-two targets: "openmw-cs openmw-cs-tests openmw-launcher openmw-wizard" group_name: "group-two" From 04443191a1d74345bfaac14009ab7f0f0fbf2330 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Wed, 24 Dec 2025 20:12:09 +0000 Subject: [PATCH 15/17] Sync MSBuild and Ninja jobs --- .gitlab-ci.yml | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f060b7613f..35014fc071 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -747,8 +747,8 @@ macOS15_Xcode16_arm64: - New-Item -Type Directory -Force -Path $env:CCACHE_DIR - New-Item -Type File -Force -Path MSVC2022_64_Ninja\.cmake\api\v1\query\codemodel-v2 - sh CI/before_script.msvc.sh -c $config -p Win64 -v 2022 -k -V -N -b -t -C $multiview -E - - Get-Volume - cd MSVC2022_64_Ninja + - Get-Volume - .\ActivateMSVC.ps1 - cmake --build . --config $config --target ($targets.Split(' ')) - ccache --show-stats -v @@ -759,26 +759,28 @@ macOS15_Xcode16_arm64: - aws --version - | if (Get-ChildItem -Recurse *.pdb) { - 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" '*.pdb' CI-ID.txt + 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" '*.pdb' CI-ID.txt if(!$?) { Exit $LASTEXITCODE } if (Test-Path env:AWS_ACCESS_KEY_ID) { - aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" s3://openmw-artifacts/${artifactDirectory} + aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" s3://openmw-artifacts/${artifactDirectory} if(!$?) { Exit $LASTEXITCODE } } Push-Location .. ..\CI\Store-Symbols.ps1 -SkipCompress if(!$?) { Exit $LASTEXITCODE } - 7z a -tzip "..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_sym_store.zip"))" '.\SymStore\*' $config\CI-ID.txt + 7z a -tzip "..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_sym_store.zip"))" '.\SymStore\*' $config\CI-ID.txt if(!$?) { Exit $LASTEXITCODE } Pop-Location Get-ChildItem -Recurse *.pdb | Remove-Item } - - 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}.zip"))" '*' + - Rename-Item CI-ID.txt CI-ID_${group_name}.txt + - 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip"))" '*' - | if (Test-Path env:AWS_ACCESS_KEY_ID) { - aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}.zip"))" s3://openmw-artifacts/${artifactDirectory} + aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip"))" s3://openmw-artifacts/${artifactDirectory} if(!$?) { Exit $LASTEXITCODE } } + - echo "$(Make-SafeFileName("OpenMW_MSVC2022_64_${group_name}_${config}_${CI_COMMIT_REF_NAME}.zip")):$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}"))" | Out-File -Encoding UTF8 "..\..\${group_name}_to-be-merged.txt" - | if ($executables) { foreach ($exe in $executables.Split(',')) { @@ -802,6 +804,7 @@ macOS15_Xcode16_arm64: - "*.log" - MSVC2022_64_Ninja/*.log - MSVC2022_64_Ninja/**/*.log + - "*_to-be-merged.txt" variables: targets: all # When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks. @@ -856,11 +859,34 @@ macOS15_Xcode16_arm64: # Gitlab can't successfully execute following binaries due to unknown reason # executables: "components-tests.exe,openmw-tests.exe,openmw-cs-tests.exe,openmw_detournavigator_navmeshtilescache_benchmark.exe" +.Windows_Ninja_RelWithDebInfo_GroupOne: + extends: + - .Windows_Ninja_RelWithDebInfo + variables: + <<: *target-group-one + +.Windows_Ninja_RelWithDebInfo_GroupTwo: + extends: + - .Windows_Ninja_RelWithDebInfo + variables: + <<: *target-group-two + .Windows_Compress_And_Upload_Symbols_Ninja_RelWithDebInfo: extends: - .Compress_And_Upload_Symbols_Base needs: - - job: "Windows_Ninja_RelWithDebInfo" + - job: "Windows_Ninja_RelWithDebInfo_GroupOne" + artifacts: true + - job: "Windows_Ninja_RelWithDebInfo_GroupTwo" + artifacts: true + +.Windows_Merge_Artifacts_Ninja_RelWithDebInfo: + extends: + - .Merge_Artifacts_Base + needs: + - job: "Windows_Ninja_RelWithDebInfo_GroupOne" + artifacts: true + - job: "Windows_Ninja_RelWithDebInfo_GroupTwo" artifacts: true .Windows_Ninja_CacheInit: @@ -1010,13 +1036,13 @@ macOS15_Xcode16_arm64: Windows_MSBuild_RelWithDebInfo_GroupOne: extends: - - .Windows_MSBuild_RelWithDebInfo + - .Windows_MSBuild_RelWithDebInfo variables: <<: *target-group-one Windows_MSBuild_RelWithDebInfo_GroupTwo: extends: - - .Windows_MSBuild_RelWithDebInfo + - .Windows_MSBuild_RelWithDebInfo variables: <<: *target-group-two From a783431df79a3d5b4a9c993a17fbde90b5d25b1c Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Thu, 25 Dec 2025 00:50:49 +0000 Subject: [PATCH 16/17] Balance build times between jobs Before this, there was a difference of about nineteen minutes. All the things that aren't the engine (except its dependencies) that were in group one take less than nineteen minutes to build in total, so moving everything isn't enough to give total balance. openmw-tests depends on openmw-lib, so you have to build basically the whole engine to build it, so it should stay in the same group. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 35014fc071..a425626fd7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -698,11 +698,11 @@ macOS15_Xcode16_arm64: - "*.zip" .variables-for-split-jobs: &target-group-one - targets: "bsatool components-tests esmtool niftest openmw openmw_detournavigator_navmeshtilescache_benchmark openmw_esm_refid_benchmark openmw_settings_access_benchmark openmw-bulletobjecttool openmw-essimporter openmw-iniimporter openmw-navmeshtool openmw-tests" + targets: "openmw openmw-tests" group_name: "group-one" .variables-for-split-jobs: &target-group-two - targets: "openmw-cs openmw-cs-tests openmw-launcher openmw-wizard" + targets: "bsatool components-tests esmtool niftest openmw-cs openmw-cs-tests openmw_detournavigator_navmeshtilescache_benchmark openmw_esm_refid_benchmark openmw_settings_access_benchmark openmw-bulletobjecttool openmw-essimporter openmw-iniimporter openmw-launcher openmw-navmeshtool openmw-wizard" group_name: "group-two" .Windows_Ninja_Base: From 25543090af96c37c106e054dae48b41e3075b549 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Tue, 30 Dec 2025 17:44:44 +0000 Subject: [PATCH 17/17] Get rid of TODO that doesn't need doing --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a425626fd7..95b2a517a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -668,7 +668,6 @@ macOS15_Xcode16_arm64: - mv *.zip incoming_artifacts/ - declare -A destinations - | - # todo: this will get upset by certain characters for merge_list in *_to-be-merged.txt; do while IFS=: read -r partial destination; do destinations["$destination"]=""