WIP: Repackage .deb builds as mozillavpn-beta

taskcluster/kinds/beetmover-apt/kind.yml

@@ -26,7 +26,7 @@ task-defaults:
 
 tasks:
     promote-linux64-deb:
-        worker-type: beetmover-apt-stage
+        worker-type: beetmover-apt
         attributes:
             shipping-phase: promote-client
 

taskcluster/kinds/beetmover-promote/kind.yml

@@ -9,12 +9,14 @@ transforms:
     - mozillavpn_taskgraph.transforms.beetmover_mac_upstream:transforms
     - mozilla_taskgraph.transforms.scriptworker.release_artifacts:transforms
     - mozillavpn_taskgraph.transforms.beetmover:transforms
+    - mozillavpn_taskgraph.transforms.add_version:transforms
     - taskgraph.transforms.task:transforms
 
 kind-dependencies:
     - build
     - signing
     - mac-notarization
+    - repackage
     - repackage-signing
 
 task-defaults:

taskcluster/kinds/build/linux.yml

@@ -82,6 +82,7 @@ linux64/release-deb:
     fetches:
         toolchain:
             - qt-linux
+    worker-type: b-linux-large
     worker:
         docker-image: {in-tree: linux-qt6-build}
     add-index-routes:

taskcluster/kinds/repackage/kind.yml

@@ -6,11 +6,13 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - mozilla_taskgraph.transforms.scriptworker.release_artifacts:transforms
+    - mozillavpn_taskgraph.transforms.add_version:transforms
     - taskgraph.transforms.run:transforms
     - taskgraph.transforms.task:transforms
 
 kind-dependencies:
     - signing
+    - build
 
 tasks:
     msi:
@@ -43,3 +45,67 @@ tasks:
             kind: build
             tier: 1
             platform: windows/x86_64
+
+    deb-beta:
+        description: "Debian repackage for beta release"
+        # disable for testing
+        # run-on-tasks-for: [github-push, github-release]
+        fetches:
+            build:
+                - artifact: mozillavpn.deb
+        dependencies:
+            build: build-linux64/release-deb
+        attributes:
+            build-type: linux64/release-deb
+        worker-type: b-linux
+        worker:
+            docker-image: {in-tree: build}
+            max-run-time: 3600
+            chain-of-trust: true
+        release-artifacts:
+            - mozillavpn-beta.deb
+        treeherder:
+            symbol: rpk(deb-beta)
+            kind: build
+            tier: 1
+            platform: linux/x86_64
+        run:
+            using: run-task
+            use-caches: true
+            cwd: '{checkout}'
+            command: >-
+                ./taskcluster/scripts/build/dpkg-repack.py ${MOZ_FETCHES_DIR}/mozillavpn.deb \
+                    --set provides=mozillavpn --set replaces=mozillavpn --set package=mozillavpn-beta \
+                    -o /builds/worker/artifacts/mozillavpn-beta.deb
+
+    deb-release:
+        description: "Debian repackage for public release"
+        # disable for testing
+        # run-on-tasks-for: [github-push, github-release]
+        fetches:
+            build:
+                - artifact: mozillavpn.deb
+        dependencies:
+            build: build-linux64/release-deb
+        attributes:
+            build-type: linux64/release-deb
+        worker-type: b-linux
+        worker:
+            docker-image: {in-tree: build}
+            max-run-time: 3600
+            chain-of-trust: true
+        release-artifacts:
+            - mozillavpn.deb
+        treeherder:
+            symbol: rpk(deb-release)
+            kind: build
+            tier: 1
+            platform: linux/x86_64
+        run:
+            using: run-task
+            use-caches: true
+            cwd: '{checkout}'
+            command: >-
+                ./taskcluster/scripts/build/dpkg-repack.py ${MOZ_FETCHES_DIR}/mozillavpn.deb \
+                    --set provides=mozillavpn --set version=$(cat version.txt) \
+                    -o /builds/worker/artifacts/mozillavpn-$(cat version.txt).deb

taskcluster/mozillavpn_taskgraph/transforms/add_version.py

@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os.path
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+# Append the VPN client version to the filenames of the release-artifacts.
+@transforms.add
+def add_version(config, tasks):
+    # Get the version number from version.txt
+    cwd = os.path.dirname(os.path.realpath(__file__))
+    with open(os.path.join(cwd, '..', '..', '..', 'version.txt'), 'r') as fp:
+        version = fp.readline().strip()
+
+    # If this is a beetmover promotion job, change the version to "beta" instead
+    if config.kind == "beetmover-promote":
+        version = "beta"
+
+    for task in tasks:
+        if task["name"] not in ("deb-release", "linux64-deb"):
+            yield task
+            continue
+
+        # Get the release artifacts and append -<version> to their filenames.
+        for artifact in task["attributes"]["release-artifacts"]:  
+            if "name" in artifact:
+                root, ext = os.path.splitext(artifact["name"])
+                artifact["name"] = root + "-" + version + ext
+            if "path" in artifact:
+                root, ext = os.path.splitext(artifact["path"])
+                artifact["path"] = root + "-" + version + ext
+
+        yield task

taskcluster/mozillavpn_taskgraph/transforms/beetmover.py

@@ -67,6 +67,24 @@ def add_addons_release_artifacts(config, tasks):
                 )
         yield task
 
+@transforms.add
+def add_linux_repackage_dependency(config, tasks):
+    short_phase = config.kind[len("beetmover-") :]
+
+    for task in tasks:
+        if task["attributes"]["build-type"] != "linux64/release-deb":
+            yield task
+            continue
+        if task["name"] != "linux64-deb":
+            yield task
+            continue
+
+        if short_phase == "ship":
+            task["dependencies"]["repackage"] = "repackage-deb-release"
+        else:
+            task["dependencies"]["repackage"] = "repackage-deb-beta"
+
+        yield task
 
 @transforms.add
 def add_beetmover_worker_config(config, tasks):
@@ -105,12 +123,12 @@ def add_beetmover_worker_config(config, tasks):
 
         upstream_artifacts = []
         for dep in task["dependencies"]:
-            if dep not in ("build", "mac-notarization", "repackage-signing", "signing"):
+            if dep not in ("build", "mac-notarization", "repackage", "repackage-signing", "signing"):
                 continue
             upstream_artifacts.append(
                 {
                     "taskId": {"task-reference": f"<{dep}>"},
-                    "taskType": dep if dep == "build" else "scriptworker",
+                    "taskType": dep if (dep == "build" or dep == "repackage") else "scriptworker",
                     "paths": [
                         release_artifact["name"]
                         for release_artifact in task["attributes"]["release-artifacts"]

taskcluster/mozillavpn_taskgraph/transforms/beetmover_apt.py

@@ -43,6 +43,11 @@ def get_gcs_sources(dependent_task):
 
 @transforms.add
 def beetmover_apt(config, tasks):
+    is_production = (
+        config.params["level"] == "3" and config.params["tasks_for"] == "action"
+    )
+    bucket = "release" if is_production else "dep"
+
     for task in tasks:
         dep = get_primary_dependency(config, task)
         assert dep
@@ -52,11 +57,6 @@ def beetmover_apt(config, tasks):
             continue
         task["worker"]["gcs-sources"] = gcs_sources
 
-        if task["attributes"]["shipping-phase"] == "ship-client":
-            bucket = "release"
-        else:
-            bucket = "dep"
-
         scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
         task["scopes"] = [
             f"{scope_prefix}:beetmover:apt-repo:{bucket}",

taskcluster/scripts/build/dpkg-repack.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import subprocess
+import os
+import tempfile
+import re
+import sys
+from collections import OrderedDict
+
+def convert_case(name):
+    """Convert strings to Deb822-style case"""
+    start = True
+    result = ''
+    for char in name:
+        if not char.isalnum():
+            result += '-'
+            start = True
+        elif start:
+            result += char.upper()
+            start = False
+        else:
+            result += char.lower()
+            start = False
+    return result
+
+def parse_control(filename):
+    """Parse a Debian package control file into an OrderedDict"""
+    contents = OrderedDict()
+    restart = re.compile('^[A-Z][A-Za-z0-9-]+:')
+
+    with open(filename, 'r') as fp:
+        section = None
+        for line in fp.readlines():
+            line = line.rstrip()
+            if restart.match(line):
+                section, text = line.split(':', 1)
+                section = convert_case(section)
+                contents[section] = text.lstrip()
+            elif section is not None:
+                contents[section] = contents[section] + '\n' + line
+
+    return contents
+
+def write_control(contents, file=sys.stdout):
+    """Print an OrderedDict as a Debian package control file"""
+    for key in contents:
+        print(f"{key}: {contents[key]}", file=file)
+
+if __name__ == "__main__":
+    argparser = argparse.ArgumentParser(
+        description='Repack a Debian binary package while modifying its control file')
+
+    argparser.add_argument('filename', metavar='INPUT', type=str,
+        help='Debian package to repack')
+    argparser.add_argument('-o', '--output', metavar='OUTPUT', type=str,
+        help='Output debian package file')
+    argparser.add_argument('-s', '--set', metavar='KEY=VALUE', type=str, action='append', default=[],
+        help='Set or update values in the package control file')
+    args = argparser.parse_args()
+
+    # Default output behavior modifies the package in place.
+    if not args.output:
+        args.output = args.filename
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        # Unpack the Debian package.
+        archivedir = os.path.join(tempdir, 'archive')
+        subprocess.check_call(['dpkg-deb', '-R', args.filename, archivedir])
+
+        # Parse the control file
+        controlfile = os.path.join(archivedir, 'DEBIAN', 'control')
+        contents = parse_control(controlfile)
+
+        # Set/update extra values in the control file
+        for keyval in args.set:
+            key, value = keyval.split('=', 1)
+            contents[convert_case(key)] = value
+
+        # Ensure the 'Description' always goes last and then write the updated control file.
+        contents.move_to_end('Description')
+        write_control(contents, file=sys.stderr)
+        with open(controlfile, 'w') as fp:
+            write_control(contents, file=fp)
+
+        # Repack the Debian package files.
+        print(f"Writing updated package to {os.path.abspath(args.output)}", file=sys.stderr)
+        subprocess.check_call(['dpkg-deb', '-Zgzip', '-b', archivedir, args.output])