From e90c4187ff035475eaddc55333d92daae4698eac Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Mon, 20 Jan 2025 22:50:38 +0000 Subject: [PATCH 1/7] Pin awscli to 2.22.35 Because of https://github.com/aws/aws-cli/issues/9214, 2.23.0 and later won't work with our non-Amazon-hosted S3 buckets. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f1da3eb43c..7499ab2287 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -540,7 +540,7 @@ macOS14_Xcode15_arm64: script: - apt-get update - apt-get install -y curl gcab unzip - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o awscli-exe-linux-x86_64.zip + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o awscli-exe-linux-x86_64.zip - unzip -d awscli-exe-linux-x86_64 awscli-exe-linux-x86_64.zip - pushd awscli-exe-linux-x86_64 - ./aws/install From 2d0f45ea41c2df2210bd06f06b93b8b89cf8468f Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Tue, 21 Jan 2025 17:15:10 +0000 Subject: [PATCH 2/7] Log awscli version 2.23.0 had breaking changes, so we need to know if we're using it, and be able to diagnose anything else caused by breaking changes in the future now they're a possibility. --- .github/workflows/windows.yml | 4 +++- .gitlab-ci.yml | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 82386e5292..b1c7c41b15 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -202,7 +202,9 @@ jobs: AWS_DEFAULT_REGION: eu-west-3 if: ${{ env.AWS_ACCESS_KEY_ID != '' && env.AWS_SECRET_ACCESS_KEY != '' && inputs.package }} working-directory: ${{ github.workspace }}/SymStore - run: aws --endpoint-url https://rgw.ctrl-c.liu.se s3 sync --size-only --exclude * --include *.ex_ --include *.dl_ --include *.pd_ . s3://openmw-sym + run: | + aws --version + aws --endpoint-url https://rgw.ctrl-c.liu.se s3 sync --size-only --exclude * --include *.ex_ --include *.dl_ --include *.pd_ . s3://openmw-sym - name: Add install directory to PATH shell: bash diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7499ab2287..4a8f0f96b9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -510,6 +510,7 @@ Ubuntu_GCC_integration_tests_asan: - CI/before_script.osx.sh - cd build; make -j $(sysctl -n hw.logicalcpu) package - for dmg in *.dmg; do mv "$dmg" "${dmg%.dmg}_${CI_COMMIT_REF_NAME##*/}.dmg"; done + - aws --version - | if [[ -n "${AWS_ACCESS_KEY_ID}" ]]; then artifactDirectory="${CI_PROJECT_NAMESPACE//[\"<>|$'\t'\/\\?*]/_}/${CI_COMMIT_REF_NAME//[\"<>|$'\t'\/\\?*]/_}/${CI_COMMIT_SHORT_SHA//[\"<>|$'\t'\/\\?*]/_}-${CI_JOB_ID//[\"<>|$'\t'\/\\?*]/_}/" @@ -621,6 +622,7 @@ macOS14_Xcode15_arm64: - echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`nCI_JOB_ID ${CI_JOB_ID}`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt - $artifactDirectory = "$(Make-SafeFileName("${CI_PROJECT_NAMESPACE}"))/$(Make-SafeFileName("${CI_COMMIT_REF_NAME}"))/$(Make-SafeFileName("${CI_COMMIT_SHORT_SHA}-${CI_JOB_ID}"))/" - Get-ChildItem -Recurse *.ilk | Remove-Item + - aws --version - | if (Get-ChildItem -Recurse *.pdb) { 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" '*.pdb' CI-ID.txt @@ -767,6 +769,7 @@ macOS14_Xcode15_arm64: - echo "CI_COMMIT_REF_NAME ${CI_COMMIT_REF_NAME}`nCI_JOB_ID ${CI_JOB_ID}`nCI_COMMIT_SHA ${CI_COMMIT_SHA}" | Out-File -Encoding UTF8 CI-ID.txt - $artifactDirectory = "$(Make-SafeFileName("${CI_PROJECT_NAMESPACE}"))/$(Make-SafeFileName("${CI_COMMIT_REF_NAME}"))/$(Make-SafeFileName("${CI_COMMIT_SHORT_SHA}-${CI_JOB_ID}"))/" - Get-ChildItem -Recurse *.ilk | Remove-Item + - aws --version - | if (Get-ChildItem -Recurse *.pdb) { 7z a -tzip "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" '*.pdb' CI-ID.txt From 57ffc11fbaef391fe92ed532bc9c3b86d2d30dc9 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Tue, 21 Jan 2025 18:47:54 +0000 Subject: [PATCH 3/7] Try installing specific version of awscli on MacOS --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a8f0f96b9..45d8d9ed30 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -503,6 +503,7 @@ Ubuntu_GCC_integration_tests_asan: - ccache/ script: - CI/before_install.osx.sh + - brew install awscli@2.22.35 - export CCACHE_BASEDIR="$(pwd)" - export CCACHE_DIR="$(pwd)/ccache" - mkdir -pv "${CCACHE_DIR}" From 9ae12baee1c8e568831023dd74144c8c52203572 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Tue, 21 Jan 2025 20:44:08 +0000 Subject: [PATCH 4/7] Pin it on Windows, too --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 45d8d9ed30..63c650945c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -589,7 +589,7 @@ macOS14_Xcode15_arm64: - choco install vswhere -y - choco install ninja -y - choco install python -y - - choco install awscli -y + - choco install awscli -y --version=2.22.35 - refreshenv - | function Make-SafeFileName { @@ -741,7 +741,7 @@ macOS14_Xcode15_arm64: - choco install 7zip -y - choco install vswhere -y - choco install python -y - - choco install awscli -y + - choco install awscli -y --version=2.22.35 - refreshenv - | function Make-SafeFileName { From 4c95e91a8d9581f6deb679e433ceea8593f8f5a0 Mon Sep 17 00:00:00 2001 From: Alexei Kotov Date: Fri, 31 Jan 2025 00:59:50 +0300 Subject: [PATCH 5/7] Replace awscli with s3cmd for macOS Homebrew doesn't let us downgrade, we have to use an alternative client (for now) --- .gitlab-ci.yml | 11 ++++++++--- CI/before_install.osx.sh | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 63c650945c..c910a0c4ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -503,7 +503,6 @@ Ubuntu_GCC_integration_tests_asan: - ccache/ script: - CI/before_install.osx.sh - - brew install awscli@2.22.35 - export CCACHE_BASEDIR="$(pwd)" - export CCACHE_DIR="$(pwd)/ccache" - mkdir -pv "${CCACHE_DIR}" @@ -511,12 +510,18 @@ Ubuntu_GCC_integration_tests_asan: - CI/before_script.osx.sh - cd build; make -j $(sysctl -n hw.logicalcpu) package - for dmg in *.dmg; do mv "$dmg" "${dmg%.dmg}_${CI_COMMIT_REF_NAME##*/}.dmg"; done - - aws --version - | if [[ -n "${AWS_ACCESS_KEY_ID}" ]]; then + echo "[default]" > ~/.s3cfg + echo "access_key = ${AWS_ACCESS_KEY_ID}" >> ~/.s3cfg + echo "secret_key = ${AWS_SECRET_ACCESS_KEY}" >> ~/.s3cfg + echo "host_base = rgw.ctrl-c.liu.se" >> ~/.s3cfg + echo "host_bucket = %(bucket)s.rgw.ctrl-c.liu.se" >> ~/.s3cfg + echo "use_https = True" >> ~/.s3cfg + artifactDirectory="${CI_PROJECT_NAMESPACE//[\"<>|$'\t'\/\\?*]/_}/${CI_COMMIT_REF_NAME//[\"<>|$'\t'\/\\?*]/_}/${CI_COMMIT_SHORT_SHA//[\"<>|$'\t'\/\\?*]/_}-${CI_JOB_ID//[\"<>|$'\t'\/\\?*]/_}/" for dmg in *.dmg; do - aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "${dmg}" s3://openmw-artifacts/${artifactDirectory} + s3cmd put "${dmg}" s3://openmw-artifacts/${artifactDirectory} done fi - ccache -s diff --git a/CI/before_install.osx.sh b/CI/before_install.osx.sh index 0120c55202..d73399c102 100755 --- a/CI/before_install.osx.sh +++ b/CI/before_install.osx.sh @@ -7,7 +7,7 @@ export HOMEBREW_AUTOREMOVE=1 brew tap --repair brew update --quiet -brew install curl xquartz gd fontconfig freetype harfbuzz brotli +brew install curl xquartz gd fontconfig freetype harfbuzz brotli s3cmd command -v ccache >/dev/null 2>&1 || brew install ccache command -v cmake >/dev/null 2>&1 || brew install cmake From e583e64380368a128d70faf6f871cc6d552a9453 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Fri, 31 Jan 2025 00:00:29 +0000 Subject: [PATCH 6/7] Downgrade preinstalled awscli to a version that works --- .github/workflows/windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index b1c7c41b15..014542ed22 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -203,6 +203,7 @@ jobs: if: ${{ env.AWS_ACCESS_KEY_ID != '' && env.AWS_SECRET_ACCESS_KEY != '' && inputs.package }} working-directory: ${{ github.workspace }}/SymStore run: | + choco upgrade awscli -y --version=2.22.35 --allow-downgrade aws --version aws --endpoint-url https://rgw.ctrl-c.liu.se s3 sync --size-only --exclude * --include *.ex_ --include *.dl_ --include *.pd_ . s3://openmw-sym From a3531fe954bbb3327feeb5ede1e58b33cab660a2 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Fri, 31 Jan 2025 00:55:17 +0000 Subject: [PATCH 7/7] Direct downgrade failed, try uninstalling first --- .github/workflows/windows.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 014542ed22..d2a7d37990 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -203,7 +203,8 @@ jobs: if: ${{ env.AWS_ACCESS_KEY_ID != '' && env.AWS_SECRET_ACCESS_KEY != '' && inputs.package }} working-directory: ${{ github.workspace }}/SymStore run: | - choco upgrade awscli -y --version=2.22.35 --allow-downgrade + choco uninstall awscli -y + choco install awscli -y --version=2.22.35 aws --version aws --endpoint-url https://rgw.ctrl-c.liu.se s3 sync --size-only --exclude * --include *.ex_ --include *.dl_ --include *.pd_ . s3://openmw-sym