diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 726995e520..5d37548bc8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,22 @@ variables: rules: - if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event" +Ubuntu_GCC_preprocess: + extends: .Ubuntu_Image + cache: + key: Ubuntu_GCC_preprocess.ubuntu_22.04.v1 + paths: + - apt-cache/ + - .cache/pip/ + stage: build + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + before_script: + - CI/install_debian_deps.sh openmw-deps openmw-deps-dynamic gcc_preprocess + - pip3 install --user click termtables + script: + - CI/ubuntu_gcc_preprocess.sh + .Ubuntu: extends: .Ubuntu_Image cache: diff --git a/CI/install_debian_deps.sh b/CI/install_debian_deps.sh index b05a01c4dd..48b28d25e0 100755 --- a/CI/install_debian_deps.sh +++ b/CI/install_debian_deps.sh @@ -12,6 +12,20 @@ declare -rA GROUPED_DEPS=( [gcc]="binutils gcc build-essential cmake ccache curl unzip git pkg-config mold" [clang]="binutils clang make cmake ccache curl unzip git pkg-config mold" [coverity]="binutils clang-11 make cmake ccache curl unzip git pkg-config" + [gcc_preprocess]=" + binutils + build-essential + clang + cmake + curl + gcc + git + libclang-dev + ninja-build + python3-clang + python3-pip + unzip + " # Common dependencies for building OpenMW. [openmw-deps]=" diff --git a/CI/ubuntu_gcc_preprocess.sh b/CI/ubuntu_gcc_preprocess.sh new file mode 100755 index 0000000000..1eefa13ead --- /dev/null +++ b/CI/ubuntu_gcc_preprocess.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -xeo pipefail + +SRC="${PWD:?}" +VERSION=$(git rev-parse HEAD) + +mkdir -p build +cd build + +cmake \ + -G Ninja \ + -D CMAKE_C_COMPILER=gcc \ + -D CMAKE_CXX_COMPILER=g++ \ + -D USE_SYSTEM_TINYXML=ON \ + -D OPENMW_USE_SYSTEM_RECASTNAVIGATION=ON \ + -D CMAKE_BUILD_TYPE=Release \ + -D CMAKE_C_FLAGS_RELEASE='-DNDEBUG -E -w' \ + -D CMAKE_CXX_FLAGS_RELEASE='-DNDEBUG -E -w' \ + -D CMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -D BUILD_BENCHMARKS=ON \ + -D BUILD_BSATOOL=ON \ + -D BUILD_BULLETOBJECTTOOL=ON \ + -D BUILD_ESMTOOL=ON \ + -D BUILD_ESSIMPORTER=ON \ + -D BUILD_LAUNCHER=ON \ + -D BUILD_LAUNCHER_TESTS=ON \ + -D BUILD_MWINIIMPORTER=ON \ + -D BUILD_NAVMESHTOOL=ON \ + -D BUILD_NIFTEST=ON \ + -D BUILD_OPENCS=ON \ + -D BUILD_OPENCS_TESTS=ON \ + -D BUILD_OPENMW=ON \ + -D BUILD_UNITTESTS=ON \ + -D BUILD_WIZARD=ON \ + "${SRC}" +cmake --build . --parallel + +cd .. + +scripts/preprocessed_file_size_stats.py --remove_prefix "${SRC}/" build > "${VERSION:?}.json" +ls -al "${VERSION:?}.json" + +if [[ "${GENERATE_ONLY}" ]]; then + exit 0 +fi + +git remote add target "${CI_MERGE_REQUEST_SOURCE_PROJECT_URL:-https://gitlab.com/OpenMW/openmw.git}" + +TARGET_BRANCH="${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-master}" + +git fetch target "${TARGET_BRANCH:?}" + +BASE_VERSION=$(git merge-base "target/${TARGET_BRANCH:?}" HEAD) + +# Save and use scripts from this commit because they may be absent or different in the base version +cp scripts/preprocessed_file_size_stats.py scripts/preprocessed_file_size_stats.bak.py +cp CI/ubuntu_gcc_preprocess.sh CI/ubuntu_gcc_preprocess.bak.sh +git checkout "${BASE_VERSION:?}" +mv scripts/preprocessed_file_size_stats.bak.py scripts/preprocessed_file_size_stats.py +mv CI/ubuntu_gcc_preprocess.bak.sh CI/ubuntu_gcc_preprocess.sh +env GENERATE_ONLY=1 CI/ubuntu_gcc_preprocess.sh +git checkout --force "${VERSION:?}" + +scripts/preprocessed_file_size_stats_diff.py "${BASE_VERSION:?}.json" "${VERSION:?}.json" diff --git a/scripts/preprocessed_file_size_stats.py b/scripts/preprocessed_file_size_stats.py new file mode 100755 index 0000000000..d1589bf05c --- /dev/null +++ b/scripts/preprocessed_file_size_stats.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import clang.cindex +import click +import json +import os +import os.path +import sys + + +@click.command() +@click.option('--remove_prefix', type=str, default='') +@click.argument('build_dir', type=click.Path(exists=True)) +def main(build_dir, remove_prefix): + libclang = os.environ.get('LIBCLANG') + if libclang is not None: + clang.cindex.Config.set_library_file(libclang) + db = clang.cindex.CompilationDatabase.fromDirectory(build_dir) + files = dict() + total = 0 + for command in db.getAllCompileCommands(): + try: + size = os.stat(os.path.join(command.directory, get_output_path(command.arguments))).st_size + files[command.filename.removeprefix(remove_prefix)] = size + total += size + except Exception as e: + print(f'Failed to process command for {command.filename}: {e}', file=sys.stderr) + pass + files['total'] = total + json.dump(files, sys.stdout) + + +def get_output_path(arguments): + return_next = False + for v in arguments: + if return_next: + return v + elif v == '-o': + return_next = True + +if __name__ == '__main__': + main() diff --git a/scripts/preprocessed_file_size_stats_diff.py b/scripts/preprocessed_file_size_stats_diff.py new file mode 100755 index 0000000000..f158a81058 --- /dev/null +++ b/scripts/preprocessed_file_size_stats_diff.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import click +import json +import termtables + + +@click.command() +@click.argument('first', type=click.Path(exists=True)) +@click.argument('second', type=click.Path(exists=True)) +def main(first, second): + stats = tuple(read_stats(v) for v in (first, second)) + keys = set() + for v in stats: + keys.update(v.keys()) + keys = sorted(keys - {'total'}) + + result = list() + for k in keys: + first_size = stats[0].get(k, 0) + second_size = stats[1].get(k, 0) + diff = second_size - first_size + if diff != 0: + result.append((k, first_size, diff, (second_size / first_size - 1) * 100 if first_size != 0 else 100)) + + result.sort(key=lambda v: tuple(reversed(v))) + + diff = stats[1]['total'] - stats[0]['total'] + first_total = stats[0]['total'] + result.append(('total', first_total, diff, (stats[1]['total'] / first_total - 1) * 100) if first_total != 0 else 100) + + print(f'Preprocessed code size diff between {first} and {second}:\n') + + termtables.print( + [(v[0], v[1], f'{v[2]:+}', f'{v[3]:+}') for v in result], + header=['file', 'size, bytes', 'diff, bytes', 'diff, %'], + style=termtables.styles.markdown, + ) + + +def read_stats(path): + with open(path) as stream: + return json.load(stream) + +if __name__ == '__main__': + main()