report-pr-errors

#!/usr/bin/env python
from __future__ import print_function

from argparse import ArgumentParser, Namespace
from subprocess import getstatusoutput
from collections import deque
from fnmatch import fnmatch
from glob import glob
from os.path import dirname, join, getmtime
import re
import os
import platform
import shutil
import sys
import datetime

from alibot_helpers.github_utilities import (
    calculateMessageHash, setGithubStatus, parseGithubRef, GithubCachedClient,
    PickledCache,
)
from alibot_helpers.utilities import to_unicode

# Allow uploading logs to S3
try:
    from boto3.s3.transfer import S3Transfer
    import boto3
except ImportError:
    pass


ERRORS_RE = re.compile(
    ': (\x1b\\[[0-9;]+m)?(internal compiler |fatal )?error:|^Error(:| in )|'
    '^ERROR: | !!! \x1b\\[[0-9;]+mError - | -> FAILED|'
    r'ninja: build stopped: |make.*: \*\*\*|\[(ERROR|FATAL)\]')
WARNINGS_RE = re.compile(
    ': warning:|^Warning: (?!Unused direct dependencies:$)|'
    '  !  \x1b\\[[0-9;]+mWarning - ')
FAILED_UNIT_TEST_RE = re.compile(
    r'Test *#[0-9]*: .*\*\*\*(Failed|Timeout|Exception)|% tests passed')
KILLED_RE = re.compile(r'fatal error: Killed signal terminated program')
CMAKE_ERROR_RE = re.compile(r'^CMake Error')
O2CHECKCODE_RE = re.compile(r'^=+ List of errors found =+$')
# Messages from the full system test
FST_TASK_TIMEOUT_RE = re.compile(r'task timeout reached')
FST_LOGFILE_RE = re.compile(r'Detected critical problem in logfile')
FST_FAILED_CMD_RE = re.compile(r'^command .* had nonzero exit code [0-9]+$')
# For the comment previewing error messages, if any
ALL_ERROR_MSG_RE = re.compile('|'.join(regex.pattern for regex in (
    ERRORS_RE,
    FAILED_UNIT_TEST_RE,
    KILLED_RE,
    CMAKE_ERROR_RE,
    FST_TASK_TIMEOUT_RE,
    FST_LOGFILE_RE,
    FST_FAILED_CMD_RE,
    # Excluding WARNINGS_RE (unwanted in preview comment) and O2CHECKCODE_RE
    # (not useful without context, so it's handled specially).
)))


def parse_args():
    parser = ArgumentParser()
    parser.add_argument("--work-dir", "-w", default="sw", dest="workDir")

    parser.add_argument("--default", default="release")

    parser.add_argument("--devel-prefix", "-z",
                        dest="develPrefix",
                        default="")

    parser.add_argument("--pr",
                        required=True,
                        help=("Pull request which was checked in "
                              "<org>/<project>#<nr>@ref format"))

    parser.add_argument("--no-comments",
                        action="store_true",
                        dest="noComments",
                        default=False,
                        help="Use Details button, do not post a comment")

    parser.add_argument("--main-package",
                        default=[], action="append", dest="main_packages",
                        help=("Only this package's build logs are searched for"
                              " warnings to be shown in the HTML log. If not"
                              " given, all packages' logs are included. Can"
                              " be given multiple times."))

    status_gp = parser.add_mutually_exclusive_group()

    status_gp.add_argument("--success",
                           action="store_true",
                           dest="success",
                           default=False,
                           help="Signal a green status, not error")

    status_gp.add_argument("--pending",
                           action="store_true",
                           help="Signal only that the build has started")

    parser.add_argument("--status", "-s",
                        required=True,
                        help="Check which had the error")

    parser.add_argument("--dry-run", "-n",
                        action="store_true",
                        default=False,
                        help="Do not actually comment")

    parser.add_argument("--limit", "-l",
                        default=50,
                        help="Max number of lines from the report")

    parser.add_argument("--message", "-m",
                        dest="message",
                        help="Message to be posted")

    parser.add_argument("--logs-dest",
                        dest="logsDest",
                        default="s3://alice-build-logs.s3.cern.ch",
                        help="Destination store for logs. Either rsync://<rsync server enpoint> or s3://<bucket>.<server>")

    parser.add_argument("--log-url",
                        dest="logsUrl",
                        default="https://ali-ci.cern.ch/repo/logs",
                        help="Destination path for logs")

    parser.add_argument("--github-cache-file", default=PickledCache.default_cache_location(),
                        help="Where to cache GitHub API responses (default %(default)s)")

    parser.add_argument("--debug", "-d",
                        action="store_true",
                        default=False,
                        help="Turn on debug output")

    args = parser.parse_args()
    if "#" not in args.pr:
        parser.error("You need to specify a pull request")
    if "@" not in args.pr:
        parser.error("You need to specify a commit this error refers to")
    return args


class Logs(object):
    def __init__(self, args, is_branch):
        self.work_dir = args.workDir
        self.develPrefix = args.develPrefix
        self.limit = args.limit
        self.norm_status = re.sub('[^a-zA-Z0-9_-]', '_', args.status)
        self.full_log = self.constructFullLogName(args.pr)
        self.is_branch = is_branch
        self.full_log_latest = self.constructFullLogName(args.pr, latest=True)
        self.pretty_log = self.constructFullLogName(args.pr, pretty=True)
        self.dest = args.logsDest
        self.url = join(args.logsUrl, self.pretty_log)
        self.log_url = join(args.logsUrl, self.full_log)
        self.build_successful = args.success
        self.pr_built = parse_pr(args.pr)
        self.main_packages = args.main_packages
        self.alibuild_version = None
        self.alidist_version = None

    def parse(self):
        self.find()
        self.grep()
        self.get_versions()
        self.cat(self.full_log)
        self.generate_pretty_log()
        if self.is_branch:
            self.cat(self.full_log_latest, no_delete=True)
        if self.dest.startswith("rsync:"):
            self.rsync(self.dest)
        elif self.dest.startswith("s3"):
            self.s3Upload(self.dest)
        else:
            print("Unknown destination url %s" % self.dest)

    def constructFullLogName(self, pr, latest=False, pretty=False):
        # file to which we cat all the individual logs
        pr = parse_pr(pr)
        return join(pr.repo_name, pr.id, "latest" if latest else pr.commit, self.norm_status,
                    "pretty.html" if pretty else "fullLog.txt")

    def find(self):
        self.fetch_log = join(self.work_dir, "MIRROR", "fetch-log.txt")
        search_path = join(self.work_dir, "BUILD", "*latest*", "log")
        print("Searching all logs matching:", search_path, file=sys.stderr)
        suffix = "latest-" + self.develPrefix if self.develPrefix else "latest"
        self.all_logs = [x for x in glob(search_path)
                         if dirname(x).endswith(suffix)]
        self.all_logs.sort(key=getmtime)
        print("Found:", *self.all_logs, sep="\n", file=sys.stderr)
        if not self.main_packages:
            print("No main package given, using warnings from all logs", file=sys.stderr)
            self.important_logs = self.all_logs
            return
        self.important_logs = []
        for package in self.main_packages:
            important_search_path = join(self.work_dir, "BUILD",
                                         package + "-latest*", "log")
            print("Important logs for package", package, "match:",
                  important_search_path, file=sys.stderr)
            self.important_logs.extend(x for x in glob(important_search_path)
                                       if dirname(x).endswith(suffix))
        self.important_logs.sort(key=getmtime)
        print("Important:", *self.important_logs, sep="\n", file=sys.stderr)

    def grep_logs(self, regex, context_before=3, context_after=3,
                  main_packages_only=False, ignore_log_files=()):
        '''Grep logfiles for regex, keeping context lines around matches.

        Each logfile is searched for lines matching regex. If a line matches,
        it is returned, together with the preceding context_before and
        following context_after lines. Overlapping context lines are only
        returned once. File names matching a glob pattern in ignore_log_files
        are not searched.

        Matching lines and context lines from all files are returned
        concatenated into a single string.
        '''
        context_sep = '--\n' if context_before > 0 or context_after > 0 else ''
        out_lines = []
        for log in self.important_logs if main_packages_only else self.all_logs:
            if any(fnmatch(log, ignore) for ignore in ignore_log_files):
                continue
            # This deque will discard old lines when new ones are appended.
            context_lines = deque(maxlen=context_before)
            future_context = 0
            first_match = True
            try:
                with open(log, encoding='utf-8', errors='replace') as logf:
                    for line in logf:
                        if re.search(regex, line):
                            # If this is the first match in this file, print
                            # the file name header. This means we get no header
                            # if nothing in this file matches, as intended.
                            if first_match:
                                out_lines.append('## %s\n' % log)
                                first_match = False
                            if future_context <= 0:
                                # If we're not currently in context, start a
                                # new block and output the last few lines.
                                out_lines.append(context_sep)
                                out_lines.extend(context_lines)
                                # The current line is output below.
                            future_context = context_after + 1
                        context_lines.append(line)
                        if future_context > 0:
                            out_lines.append(line)
                            future_context -= 1
                # If we matched at least once, we must've output at least one
                # block, so close it here by outputting a block separator line,
                # and separate files from each other using newlines.
                if not first_match:
                    out_lines.append(context_sep + '\n\n')
            except Exception as err:
                out_lines.append('\n!!! {} parsing {}: {}\n\n'
                                 .format(type(err), log, err))
        return ''.join(out_lines)

    def grep(self):
        '''Grep for errors in the build logs, or, if none are found,
        return the last N lines where N is the limit argument.

        Also extract errors from failed unit tests and o2checkcode, and various
        other helpful messages.
        '''
        # Messages from the general error/warning logs are reported in
        # o2checkcode_messages as well, so don't report them twice. The general
        # logs also contain false positives, so o2checkcode_messages is better.
        self.errors_log = self.grep_logs(
            ERRORS_RE, ignore_log_files=['*/o2checkcode-latest*/log'])
        self.warnings_log = self.grep_logs(
            WARNINGS_RE, main_packages_only=True)
        self.o2checkcode_messages = self.grep_logs(
            O2CHECKCODE_RE, context_before=0, context_after=float('inf'))
        self.failed_unit_tests = self.grep_logs(
            FAILED_UNIT_TEST_RE, context_before=0, context_after=0)
        self.compiler_killed = self.grep_logs(KILLED_RE)
        self.cmake_errors = self.grep_logs(
            CMAKE_ERROR_RE, context_before=0, context_after=10)
        # These two sections are for the O2 full system test.
        self.fst_task_timeout = self.grep_logs(
            FST_TASK_TIMEOUT_RE, context_before=3, context_after=0)
        self.full_system_test = self.grep_logs(
            FST_LOGFILE_RE, context_before=0, context_after=20)
        self.fst_failed_command = self.grep_logs(
            FST_FAILED_CMD_RE, context_before=0, context_after=float('inf'))

        # The o2checkcode log can contain spurious errors before the
        # "=== List of errors found ===" line, so treat it specially.
        error_log = self.grep_logs(
            ALL_ERROR_MSG_RE, context_before=0, context_after=0,
            ignore_log_files=['*/o2checkcode-latest*/log'])
        error_log += self.o2checkcode_messages
        if error_log:
            # Get the first recognised error messages from the error log.
            error_log_lines = error_log.split('\n')
            if len(error_log_lines) > self.limit:
                # If there are more errors than we can display, show a hint.
                error_log_lines = error_log_lines[:self.limit - 1]
                error_log_lines.append('[{} more errors; see full log]'.format(
                    len(error_log_lines) - self.limit + 1
                ))
        else:
            # Get the last lines from the log for the package built last.
            try:
                with open(self.all_logs[-1], encoding="utf-8", errors="replace") as logf:
                    error_log_lines = logf.read().splitlines()[-self.limit:]
            except IndexError:
                error_log_lines = ['No log files found']
            except OSError as err:
                error_log_lines = ['Error opening log {}: {!r}'
                                   .format(self.all_logs[-1], err)]
        self.preview_error_log = '\n'.join(error_log_lines).strip()

    def get_versions(self):
        # The versions are defined differently for both job runner setups
        # Jenkins (daily builds):
        #   - aliBuild: ALIBUILD_SLUG
        #   - alidist: ALIDIST_SLUG (if not set it defaults to alidist@master)
        # Nomad (CI checks):
        #   - aliBuild: INSTALL_ALIBUILD
        #   - alidist: (defaults to alidist@master)
        self.alibuild_version = os.getenv('ALIBUILD_SLUG') or os.getenv('INSTALL_ALIBUILD') or 'Not pinned (most likely alibuild@master)'
        self.alidist_version = os.getenv('ALIDIST_SLUG') or 'Not pinned (most likely alidist@master)'

    def generate_pretty_log(self):
        '''Extract error messages from logs.

        The errors are presented on a user-friendly HTML page, to be uploaded
        alongside the full message log.
        '''
        def htmlescape(string):
            return string.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')

        def display(string):
            return 'block' if string else 'none'

        try:
            with open(self.fetch_log, encoding='utf-8', errors='replace') as fetch_logf:
                fetch_log = fetch_logf.read()
        except OSError:
            fetch_log = ''

        with open(join('copy-logs', self.pretty_log), 'w', encoding='utf-8') as out_f:
            out_f.write(PRETTY_LOG_TEMPLATE % {
                'status': 'succeeded' if self.build_successful else 'failed',
                'log_url': self.log_url,
                'fetch': htmlescape(fetch_log),
                'fetch_display': display(fetch_log),
                'o2checkcode': htmlescape(self.o2checkcode_messages),
                'o2c_display': display(self.o2checkcode_messages),
                'unittests': htmlescape(self.failed_unit_tests),
                'unit_display': display(self.failed_unit_tests),
                'errors': htmlescape(self.errors_log),
                'err_display': display(self.errors_log),
                'warnings': htmlescape(self.warnings_log),
                'warn_display': display(self.warnings_log),
                'cmake': htmlescape(self.cmake_errors),
                'cmake_display': display(self.cmake_errors),
                'killed_display': display(self.compiler_killed),
                'noerrors_display': display(not any((
                    self.o2checkcode_messages, self.failed_unit_tests,
                    self.errors_log, self.warnings_log, self.compiler_killed,
                    self.cmake_errors, fetch_log,
                ))),
                'fst_logfile': htmlescape(self.full_system_test),
                'fst_timeout': htmlescape(self.fst_task_timeout),
                'fst_failedcmd': htmlescape(self.fst_failed_command),
                'fst_display': display(self.full_system_test or
                                       self.fst_task_timeout or
                                       self.fst_failed_command),
                'hostname': htmlescape(platform.node()),
                'pr_start_time': os.getenv('PR_START_TIME', ''),
                'finish_time': datetime.datetime.now().strftime('%a %-d %b %Y, %H:%M:%S %Z'),
                'repo': self.pr_built.repo_name,
                'pr_id': self.pr_built.id,
                'commit': self.pr_built.commit,
                'nomad_alloc_id': os.getenv('NOMAD_ALLOC_ID', ''),
                'nomad_short_alloc_id': os.getenv('NOMAD_SHORT_ALLOC_ID',
                                                  os.getenv('NOMAD_ALLOC_ID', '(no Nomad ID)')),
                'alibuild_version': self.alibuild_version,
                'alidist_version': self.alidist_version,
            })

    def cat(self, target_file, no_delete=False):
        if not no_delete:
            def print_delete_error(func, path, exc):
                print(func.__name__, "could not delete", path,
                      sep=": ", file=sys.stderr)
            shutil.rmtree("copy-logs", onerror=print_delete_error)
        try:
            os.makedirs(dirname(join("copy-logs", target_file)))
        except OSError as err:
            # Directory already exists. OSError is raised on python2 and
            # FileExistsError on python3; the latter is a subclass of OSError,
            # so this clause handles both.
            print("cannot create target dir:", err, file=sys.stderr)

        with open(join("copy-logs", target_file), "w", encoding="utf-8") as logf:
            print("Finished building on", platform.node(), "at",
                  datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S"),
                  file=logf)
            print("Built commit", self.pr_built.commit, "from",
                  "https://github.com/%s/pull/%s" % (self.pr_built.repo_name,
                                                     self.pr_built.id),
                  file=logf)
            if "NOMAD_ALLOC_ID" in os.environ:
                print("Nomad allocation:",
                      "    https://alinomad.cern.ch/ui/allocations/" + os.environ["NOMAD_ALLOC_ID"],
                      "To log into the build machine, use:",
                      "    nomad alloc exec %s bash" %
                      os.getenv("NOMAD_SHORT_ALLOC_ID", os.environ["NOMAD_ALLOC_ID"]),
                      "To stream logs from the CI process, use:",
                      "    nomad alloc logs -stderr -tail -f %s" %
                      os.getenv("NOMAD_SHORT_ALLOC_ID", os.environ["NOMAD_ALLOC_ID"]),
                      sep="\n", file=logf)
            if self.all_logs:
                print("The following files (oldest first) are present in the log:", file=logf)
                for log in self.all_logs:
                    print("-", log, file=logf)
            else:
                print("No logs found. Please check the aurora log.",
                      "See http://alisw.github.io/infrastructure-pr-testing for more instructions.",
                      sep="\n", file=logf)
            for log in self.all_logs:
                print("## Begin", log, file=logf)
                with open(log, encoding="utf-8", errors="replace") as sublogf:
                    logf.writelines(sublogf)
                print("## End", log, file=logf)

    def rsync(self, dest):
        err, out = getstatusoutput("cd copy-logs && rsync -av ./ %s" % dest)
        if err:
            print("Error while copying logs to store.", file=sys.stderr)
            print(out, file=sys.stderr)

    def s3Upload(self, dest):
        m = re.compile("^s3://([^.]+)[.](.*)").match(dest)
        bucket_name = m.group(1)
        server = m.group(2)
        s3_client = boto3.client('s3',
                                 aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
                                 aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
                                 endpoint_url="https://%s" % (server))
        s3_client.list_buckets()
        transfer = S3Transfer(s3_client)
        matches = []
        for root, dirnames, filenames in os.walk('copy-logs'):
            for filename in filenames:
                matches.append(join(root, filename))
        for src in matches:
            try:
                dst = re.compile('^copy-logs/').sub('', src)
                transfer.upload_file(src, bucket_name, dst, extra_args={
                    'ContentType': 'text/html' if src.endswith('.html') else 'text/plain',
                    'ContentDisposition': 'inline'})
            except Exception as e:
                print("Failed to upload %s to %s in bucket %s.%s" % (src, dst, bucket_name, server))


def get_pending_namespace(args):
    '''Return a Namespace to set status to pending with an appropriate message.'''
    return Namespace(commit=args.pr,
                     status=args.status + "/pending",
                     message=args.message,
                     url="")


def handle_branch(cgh: GithubCachedClient, pr, logs, args):
    ns = get_pending_namespace(args) if args.pending else \
        Namespace(commit=args.pr,
                  status=args.status + "/error",
                  message="",
                  url="")
    setGithubStatus(cgh, ns)
    sys.exit(0)


def handle_pr_id(cgh: GithubCachedClient, pr, logs, args):
    commit = cgh.get("/repos/{repo_name}/commits/{ref}",
                     repo_name=pr.repo_name,
                     ref=pr.commit)
    sha = commit["sha"]

    message = ""
    if not args.pending:
        message = "Error while checking %s for %s at %s:\n" % (args.status, sha, datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
        if args.message:
            message += args.message
        else:
            message += "```\n%s\n```\nFull log [here](%s).\n" % (to_unicode(logs.preview_error_log), to_unicode(logs.url))

    if args.dry_run:
        # commit does not exist...
        print("Will annotate %s" % commit["sha"])
        print(message)
        sys.exit(0)

    # Set status
    ns = get_pending_namespace(args) if args.pending else \
        Namespace(commit=args.pr,
                  status=args.status + ("/success" if args.success else "/error"),
                  message="",
                  url=logs.url)
    setGithubStatus(cgh, ns)

    # Comment if appropriate
    if args.noComments or args.success or args.pending:
        return

    prIssueComments = cgh.get("/repos/{repo_name}/issues/{pr_id}/comments",
                              repo_name=pr.repo_name,
                              pr_id=pr.id)

    messageHash = calculateMessageHash(message)
    for comment in prIssueComments:
        if comment["body"].startswith("Error while checking %s for %s" % (args.status, sha)):
            if calculateMessageHash(comment["body"]) != messageHash:
                print("Comment was different. Updating", file=sys.stderr)
                cgh.patch(
                    "/repos/{repo_name}/issues/comments/{commentID}",
                    {"body": message},
                    repo_name=pr.repo_name,
                    commentID=comment["id"]
                )
                sys.exit(0)

            print("Found same comment for the same commit", file=sys.stderr)
            sys.exit(0)


    cgh.post(
        "repos/{repo_name}/issues/{pr_id}/comments",
        {"body": message},
        repo_name=pr.repo_name,
        pr_id=pr.id
    )


def parse_pr(pr):
    repo_name, pr_id, pr_commit = parseGithubRef(pr)
    return Namespace(repo_name=repo_name,
                     id=pr_id,
                     commit=pr_commit)


def main():
    args = parse_args()
    pr = parse_pr(args.pr)
    logs = Logs(args, is_branch=not pr.id.isdigit())
    if not args.message and not args.pending:
        logs.parse()

    with GithubCachedClient(cache=PickledCache(args.github_cache_file)) as cgh:
        # If the branch is not a PR, we should look for open issues
        # for the branch. This should really folded as a special case
        # of the PR case.
        func = handle_branch if not pr.id.isdigit() else handle_pr_id
        func(cgh, pr, logs, args)

    cgh.printStats()


PRETTY_LOG_TEMPLATE = '''\
<!DOCTYPE html>
<head>
  <title>Build results</title>
  <style>
   body { border-width: 0.5rem; border-style: solid; margin: 0; padding: 1rem; min-height: 100vh; box-sizing: border-box; }
   pre { overflow-x: auto; }
   .succeeded { border-color: #1b5e20; }
   .succeeded h1 { color: #1b5e20; }
   .failed { border-color: #b71c1c; }
   .failed h1 { color: #b71c1c; }
   #noerrors, #noerrors-toc { display: %(noerrors_display)s; }
   #fetch, #fetch-toc { display: %(fetch_display)s; }
   #compiler-killed, #compiler-killed-toc { display: %(killed_display)s; }
   #o2checkcode, #o2checkcode-toc { display: %(o2c_display)s; }
   #tests, #tests-toc { display: %(unit_display)s; }
   #errors, #errors-toc { display: %(err_display)s; }
   #warnings, #warnings-toc { display: %(warn_display)s; }
   #cmake, #cmake-toc { display: %(cmake_display)s; }
   #fullsystest, #fullsystest-toc { display: %(fst_display)s; }
  </style>
</head>
<body class="%(status)s">
  <h1>The build %(status)s</h1>
  <table>
    <tr><th>Build host</th><td>%(hostname)s</td></tr>
    <tr><th>Started at</th><td>%(pr_start_time)s</td></tr>
    <tr><th>Finished at</th><td>%(finish_time)s</td></tr>
    <tr><th>Built PR</th>
        <td><a href="https://github.com/%(repo)s/pull/%(pr_id)s">%(repo)s#%(pr_id)s</a></td></tr>
    <tr><th>Built commit</th>
        <td><a href="https://github.com/%(repo)s/pull/%(pr_id)s/commits/%(commit)s"><code>%(commit)s</code></a></td></tr>
    <tr><th>Nomad allocation</th>
        <td><a href="https://alinomad.cern.ch/ui/allocations/%(nomad_alloc_id)s">Nomad allocation information</a></td></tr>
    <tr><th>SSH command</th>
        <td><code>nomad alloc exec %(nomad_short_alloc_id)s bash</code></td></tr>
    <tr><th>Stream logs</th>
        <td><code>nomad alloc logs -stderr -tail -f %(nomad_short_alloc_id)s</code></td></tr>
    <tr><th>Alibuild version</th>
        <td>%(alibuild_version)s</td></tr>
    <tr><th>Alidist version</th>
        <td>%(alidist_version)s</td></tr>
  </table>
  <p>The code that finds and extracts error messages may have missed some.
     Check the <a href="%(log_url)s">full build log</a> if you suspect the build failed for reasons not listed below.</p>
  <h3>Table of contents</h3>
  <p><nav><ol>
    <li id="noerrors-toc"><a href="#noerrors">No errors found</a></li>
    <li id="fetch-toc"><a href="#fetch">Git fetch failed</a></li>
    <li id="compiler-killed-toc"><a href="#compiler-killed">Error: the compiler process was killed</a></li>
    <li id="cmake-toc"><a href="#cmake">CMake errors and warnings</a></li>
    <li id="o2checkcode-toc"><a href="#o2checkcode"><code>o2checkcode</code> results</a></li>
    <li id="tests-toc"><a href="#tests">Unit test results</a></li>
    <li id="fullsystest-toc"><a href="#fullsystest">O2 full system test</a></li>
    <li id="errors-toc"><a href="#errors">Error messages</a></li>
    <li id="warnings-toc"><a href="#warnings">Compiler warnings</a></li>
  </ol></nav></p>
  <section id="noerrors">
    <h2>No errors found</h2>
    <p>Check the <a href="%(log_url)s">full build log</a> to see all compilation messages.</p>
  </section>
  <section id="fetch">
    <h2>Git fetch failed</h2>
    <p>This error may indicate an outage of a git host, such as GitHub or CERN GitLab.</p>
    <p>This build will be retried automatically;
       this error will be resolved once the git host comes back.</p>
    <p><pre><code>%(fetch)s</code></pre></p>
    <p>As the output from git may contain secrets, it is not shown here.</p>
  </section>
  <section id="compiler-killed">
    <h2>Error: the compiler process was killed</h2>
    <p>This can happen in case of memory pressure.
       A rebuild may solve this; these happen automatically.</p>
  </section>
  <section id="o2checkcode">
    <h2><code>o2checkcode</code> results</h2>
    <p><pre><code>%(o2checkcode)s</code></pre></p>
  </section>
  <section id="tests">
    <h2>Unit test results</h2>
    <p><pre><code>%(unittests)s</code></pre></p>
  </section>
  <section id="fullsystest">
    <h2>O2 full system test</h2>
    <p><pre><code>%(fst_timeout)s</code></pre></p>
    <p><pre><code>%(fst_logfile)s</code></pre></p>
    <p><pre><code>%(fst_failedcmd)s</code></pre></p>
  </section>
  <section id="cmake">
    <h2>CMake errors</h2>
    <p><pre><code>%(cmake)s</code></pre></p>
  </section>
  <section id="errors">
    <h2>Error messages</h2>
    <p>Note that the following list may include false positives! Check the sections above first.</p>
    <p><pre><code>%(errors)s</code></pre></p>
  </section>
  <section id="warnings">
    <h2>Compiler warnings</h2>
    <p>Note that the following list may include false positives! Check the sections above first.</p>
    <p><pre><code>%(warnings)s</code></pre></p>
  </section>
</body>
'''

if __name__ == "__main__":
    main()