Add CI job to find missing MRs

.gitlab-ci.yml

@@ -493,3 +493,18 @@ Debian_AndroidNDK_arm64-v8a:
   # When CCache doesn't exist (e.g. first build on a fork), build takes more than 1h, which is the default for forks.
   timeout: 1h30m
 
+FindMissingMergeRequests:
+  image: python:latest
+  stage: build
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  cache:
+    key: FindMissingMergeRequests.v1
+    paths:
+      - .cache/pip
+  before_script:
+    - pip3 install --user requests click discord_webhook
+  script:
+    - scripts/find_missing_merge_requests.py --project_id=$CI_PROJECT_ID --ignored_mrs_path=$CI_PROJECT_DIR/.resubmitted_merge_requests.txt

.resubmitted_merge_requests.txt

@@ -0,0 +1,5 @@
+1314
+1216
+1172
+1160
+1051

scripts/find_missing_merge_requests.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
 
 import click
+import discord_webhook
 import multiprocessing
+import os
 import pathlib
 import requests
 import urllib.parse
@@ -24,18 +26,23 @@ import urllib.parse
               help='End before given /merge_requests page.')
 @click.option('--per_page', type=int, default=100,
               help='Number of merge requests per page.')
-def main(token_path, project_id, host, workers, target_branch, begin_page, end_page, per_page):
+@click.option('--ignored_mrs_path', type=str,
-    token = read_token(token_path)
+              help='Path to a list of ignored MRs.')
+def main(token_path, project_id, host, workers, target_branch, begin_page, end_page, per_page, ignored_mrs_path):
+    headers = make_headers(token_path)
     base_url = f'https://{host}/api/v4/projects/{project_id}/'
+    discord_webhook_url = os.getenv('DISCORD_WEBHOOK_URL')
+    ignored_mrs = frozenset(read_ignored_mrs(ignored_mrs_path))
     checked = 0
     filtered = 0
     missing = 0
+    missing_mrs = list()
     for page in range(begin_page, end_page):
-        merge_requests = requests.get(
+        merge_requests = parse_gitlab_response(requests.get(
             url=urllib.parse.urljoin(base_url, 'merge_requests'),
-            headers={'PRIVATE-TOKEN': token},
+            headers=headers,
             params=dict(state='merged', per_page=per_page, page=page),
-        ).json()
+        ))
         if not merge_requests:
             break
         checked += len(merge_requests)
@@ -44,24 +51,63 @@ def main(token_path, project_id, host, workers, target_branch, begin_page, end_p
             continue
         filtered += len(merge_requests)
         with multiprocessing.Pool(workers) as pool:
-            missing_merge_requests = pool.map(FilterMissingMergeRequest(token, base_url), merge_requests)
+            missing_merge_requests = pool.map(FilterMissingMergeRequest(headers, base_url), merge_requests)
             for mr in missing_merge_requests:
-                if mr is not None:
+                if mr is None:
-                    missing += 1
+                    continue
-                    print(f"MR {mr['reference']} ({mr['id']}) is missing from branch {mr['target_branch']},"
+                if mr['reference'] in ignored_mrs or mr['reference'].strip('!') in ignored_mrs:
+                    print(f"Ignored MR {mr['reference']} ({mr['id']}) is missing from branch {mr['target_branch']},"
                           f" previously was merged as {mr['merge_commit_sha']}")
+                    continue
+                missing += 1
+                missing_mrs.append(mr)
+                print(f"MR {mr['reference']} ({mr['id']}) is missing from branch {mr['target_branch']},"
+                      f" previously was merged as {mr['merge_commit_sha']}")
     print(f'Checked {checked} MRs ({filtered} with {target_branch} target branch), {missing} are missing')
+    if discord_webhook_url is not None and missing_mrs:
+        project_web_url = parse_gitlab_response(requests.get(url=base_url, headers=headers))['web_url'] + '/'
+        discord_message = format_discord_message(missing=missing, filtered=filtered, target_branch=target_branch,
+                                                 project_web_url=project_web_url, missing_mrs=missing_mrs)
+        print('Sending Discord notification...')
+        print(discord_message)
+        discord_webhook.DiscordWebhook(url=discord_webhook_url, content=discord_message, rate_limit_retry=True).execute()
+    if missing_mrs:
+        exit(-1)
+
+
+def format_discord_message(missing, filtered, target_branch, project_web_url, missing_mrs):
+    target_branch = format_link(target_branch, urllib.parse.urljoin(project_web_url, f'-/tree/{target_branch}'))
+    return (
+        f'Found {missing} missing MRs out of {filtered} from {target_branch} target branch:\n'
+        + '\n'.join(format_missing_mr_message(v, project_web_url) for v in missing_mrs)
+    )
+
+
+def format_missing_mr_message(mr, project_web_url):
+    web_url = mr.get('web_url')
+    reference = mr['reference']
+    target_branch = mr['target_branch']
+    commit = mr['merge_commit_sha']
+    if web_url is not None:
+        reference = format_link(reference, web_url)
+        target_branch = format_link(target_branch, urllib.parse.urljoin(project_web_url, f'-/tree/{target_branch}'))
+        commit = format_link(commit, urllib.parse.urljoin(project_web_url, f'-/commit/{commit}'))
+    return f"MR {reference} is missing from branch {target_branch}, previously was merged as {commit}"
+
+
+def format_link(name, url):
+    return f'[{name}]({url})'
 
 
 class FilterMissingMergeRequest:
-    def __init__(self, token, base_url):
+    def __init__(self, headers, base_url):
-        self.token = token
+        self.headers = headers
         self.base_url = base_url
 
     def __call__(self, merge_request):
         commit_refs = requests.get(
             url=urllib.parse.urljoin(self.base_url, f"repository/commits/{merge_request['merge_commit_sha']}/refs"),
-            headers={'PRIVATE-TOKEN': self.token},
+            headers=self.headers,
         ).json()
         if 'message' in commit_refs and commit_refs['message'] == '404 Commit Not Found':
             return merge_request
@@ -73,10 +119,39 @@ def present_in_branch(commit_refs, branch):
     return bool(next((v for v in commit_refs if v['type'] == 'branch' and v['name'] == branch), None))
 
 
+def make_headers(token_path):
+    job_token = os.environ.get('CI_JOB_TOKEN')
+    if job_token is not None:
+        print('Using auth token from CI_JOB_TOKEN env')
+        return {'JOB_TOKEN': job_token}
+    if not os.path.exists(token_path):
+        print(f'Ignore absent token path: {token_path}')
+        return dict()
+    print(f'Using auth token from: {token_path}')
+    return {'PRIVATE-TOKEN': read_token(token_path)}
+
+
+def read_ignored_mrs(path):
+    if path is None:
+        return
+    with open(path) as stream:
+        for line in stream:
+            yield line.strip()
+
+
 def read_token(path):
     with open(path) as stream:
         return stream.readline().strip()
 
 
+def parse_gitlab_response(response):
+    response = response.json()
+    if isinstance(response, dict):
+        message = response.get('message')
+        if message is not None:
+            raise RuntimeError(f'Gitlab request has failed: {message}')
+    return response
+
+
 if __name__ == '__main__':
     main()