Neon OS Initial Implementation (#1)

* Add scripts and workflows for image builds * Remove Completed TODO * Fix error in `trigger_release` action * Fix error in `trigger_release` action * Fix cache error to actually use cache when available * Troubleshoot file path errors in GHA * Troubleshoot build output directory error * Fix envvar references in workflow * Fix typo in release workflow * Refactor workflows and events for consistency * Troubleshoot logic for core/node build * Fix syntax error in node/core check * Troubleshoot conditional builds * Troubleshoot conditional builds * Troubleshoot conditional builds * Troubleshoot conditional builds * Fix python package installation Improve build cancel/failure handling * Update relative paths in automation to use absolute paths * Fix typo in filename * Fix path error in image outputs Add missing base URL to script call Add missing call to update changelog * Fix output path errors and ensure they exist * Troubleshoot lfs file checkout * Fix path error in LFS cache * Fix incomplete path in directory creation * Fix path error in cache save * Update pi4 build automation for mark_2 compat. * Fix logical error in metadata update script * Fix typo in device/platform translation * Fix metadata handling in build and metadata updates More precisely reference output files to catch errors earlier * Add script to archive old builds on the server * Remove rpi4 platform for testing * Update kernel handling for OPi support Update release tags to ensure uniqueness * Remove failing `sudo` call Fix release notes formatting * Fix list reference error * Replace rpi4 automation * Fix image_id variable set * Fix handling of first daily beta release * Refactor to nest updates within platform directories to match existing structure * Refactor to separate "core" images into a subdirectory for consistency * Minor metadata formatting changes * Add README with content from docs --------- Co-authored-by: Daniel McKnight <[email protected]>
NeonGeckoCom · Feb 27, 2024 · 66a1782 · 66a1782
1 parent 4900d02
commit 66a1782
Show file tree

Hide file tree

Showing 6 changed files with 492 additions and 0 deletions.
diff --git a/.github/workflows/manual_release.yaml b/.github/workflows/manual_release.yaml
@@ -0,0 +1,24 @@
+name: Manual Release
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        type: string
+        default: "dev"
+      repo:
+        type: string
+        default: "neon-debos"
+jobs:
+  trigger_build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Call Release Action
+      uses: peter-evans/repository-dispatch@v3
+      with:
+        repository: neongeckocom/neon-os
+        event-type: Publish Release
+        client-payload: |-
+          {
+            "ref": "${{ inputs.branch }}",
+            "repo": "${{ inputs.repo }}"
+          }
diff --git a/.github/workflows/publish_release.yaml b/.github/workflows/publish_release.yaml
@@ -0,0 +1,91 @@
+name: Publish Release
+on:
+  repository_dispatch:
+    types:
+      - Publish Release
+
+env:
+  REF: ${{ github.event.client_payload.ref }}
+  PLATFORMS: "rpi4 opi5"
+  UPLOAD_URL: "https://2222.us"
+  OUTPUT_DIR: /var/www/html/app/files/neon_images
+  DO_CORE: ${{ contains(fromJson('["neon-debos", "neon-core"]'), github.event.client_payload.repo) }}
+  DO_NODE: ${{ contains(fromJson('["neon-debos", "neon-nodes"]'), github.event.client_payload.repo) }}
+
+jobs:
+  build_images:
+    runs-on: 2222.us
+    steps:
+      - name: Checkout Debos Repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.REF }}
+          repository: NeonGeckoCom/neon_debos
+          path: action/neon_debos
+          lfs: False
+      - name: Create LFS file list
+        run: |
+          cd action/neon_debos
+          git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id
+          cd ../..
+      - name: Restore LFS cache
+        uses: actions/cache@v3
+        id: lfs-cache
+        with:
+          path: action/neon_debos/.git/lfs
+          key: ${{ runner.os }}-lfs-${{ hashFiles('action/neon_debos/.lfs-assets-id') }}-v1
+      - name: Ensure LFS files are pulled
+        if: steps.lfs-cache.outputs.cache-hit != 'true'
+        run: |
+          cd action/neon_debos
+          git lfs pull
+          cd ../..
+      - name: Save LFS Cache
+        if: steps.lfs-cache.outputs.cache-hit != 'true'
+        id: lfs-cache-save
+        uses: actions/cache/save@v3
+        with:
+          path: action/neon_debos/.git/lfs
+          key: ${{ runner.os }}-lfs-${{ hashFiles('action/neon_debos/.lfs-assets-id') }}-v1
+      - name: Export keys for image build
+        run: |
+          mkdir -p action/neon_debos/overlays/80-google-json-overlay/home/neon/.local/share/neon
+          echo ${GOOGLE_KEY}>action/neon_debos/overlays/80-google-json-overlay/home/neon/.local/share/neon/google.json
+        env:
+          GOOGLE_KEY: ${{secrets.google_api_key}}
+      - name: Checkout NeonOS Repository
+        uses: actions/checkout@v3
+        with:
+          repository: NeonGeckoCom/neon-os
+          path: action/neon-os
+      - name: Run Neon OS Core Build
+        if: env.DO_CORE == 'true'
+        run: |
+          bash "${{ github.workspace }}/action/neon-os/scripts/build_image.sh" "${{ github.workspace }}/action/neon_debos" "${{env.REF}}" "debian-neon-image.yml" "${{env.PLATFORMS}}" "${{env.OUTPUT_DIR}}" "${{env.UPLOAD_URL}}"
+      - name: Run Neon OS Node Build
+        if: env.DO_NODE == 'true'
+        run: |
+          bash "${{ github.workspace }}/action/neon-os/scripts/build_image.sh" "${{ github.workspace }}/action/neon_debos" "${{env.REF}}" "debian-node-image.yml" "${{env.PLATFORMS}}" "${{env.OUTPUT_DIR}}" "${{env.UPLOAD_URL}}"
+      - name: Stop Docker Build Containers
+        if: failure() || cancelled()
+        run: |
+          docker kill neon_debos_ghaction
+      - name: Update Metadata
+        id: metadata
+        run: |
+          python3 "${{ github.workspace }}/action/neon-os/scripts/update_metadata.py" ${{env.REF}}
+          echo ::set-output name=version::$(grep "tag" action/neon-os/release_notes.md | cut -d'=' -f2)
+      - name: Commit Metadata Updates
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Release ${{ steps.metadata.outputs.version }}
+          repository: "${{ github.workspace }}/action/neon-os/"
+      - name: Create Pre-release
+        uses: ncipollo/release-action@v1
+        with:
+          token: ${{secrets.GITHUB_TOKEN}}
+          tag: ${{steps.metadata.outputs.version}}
+          commit: ${{ github.ref }}
+          prerelease: true
+          bodyFile: "${{ github.workspace }}/action/neon-os/release_notes.md"
+
diff --git a/README.md b/README.md
@@ -0,0 +1,29 @@
+# Neon OS Releases
+Neon OS releases can be primarily differentiated by a `core` version and
+an `image` version, where `core` refers to the repository providing the primary
+functionality (i.e. `neon-core` or `neon-nodes`) and `image` refers to code from
+the `neon_debos` repository. A particular OS release is identified by a version 
+string based on the time at which the release was compiled.
+
+## Identifying Updates
+Updates are identified as OS releases with the same `core` and a newer timestamp.
+
+An OS release is identified as a beta if EITHER the `core` or the `image` ref
+used is a beta. A device on a beta update track will only update to a newer
+beta version; a device on a stable update track will only update to a newer
+stable version. If a device changes tracks, it will update to a NEWER release on
+the new track, but it will not install an older version by default.
+> Note that in practice, any stable update will have first been released to the
+> beta track.
+
+## Version Management
+Released images are identified by GitHub releases in this repository. The `yaml`
+index files may also be used to view release history per-image.
+
+### Versioning Scheme
+Releases will follow [CalVer](https://calver.org/), so a release version may be
+`24.02.14` or `24.02.14b1`. Note that the GitHub beta tags will *not* match the 
+associated images' versions for beta versions since each release may relate to a
+different `core`.
+> i.e. Neon OS tag `24.02.27.beta1` may contain `debian-neon-image-24.02.27b1` 
+> and Neon OS tag `24.02.27.beta2` may contain `debian-node-image-24.02.27b4`.
diff --git a/scripts/archive_old_files.py b/scripts/archive_old_files.py
@@ -0,0 +1,95 @@
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework
+# All trademark and other rights reserved by their respective owners
+# Copyright 2008-2022 Neongecko.com Inc.
+# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds,
+# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo
+# BSD-3 License
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from this
+#    software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS  BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS;  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from os.path import join
+from sys import argv
+from os import walk, listdir, makedirs
+from pprint import pprint
+from shutil import move
+
+
+def get_image_dirs(base_dir: str) -> dict:
+    """
+    Get a dict of stable images, beta images, and update file directories
+    :param base_dir: Root directory to search for Neon OS files
+    :return: dict of stable, beta, and updates (beta and stable)
+    """
+    directories = {"stable": [],
+                   "beta": [],
+                   "updates": {"stable": [],
+                               "beta": []}}
+    for root, dirs, files in walk(base_dir):
+        for d in dirs:
+            if "updates" in root:
+                if "dev" in d:
+                    directories['updates']["beta"].append(join(root, d))
+                elif "master" in d:
+                    directories['updates']["stable"].append(join(root, d))
+            elif "dev" in d:
+                directories['beta'].append(join(root, d))
+            elif "master" in d:
+                directories['stable'].append(join(root, d))
+    return directories
+
+
+def prune_directory(directory: str, releases_to_keep: int,
+                    archive: str):
+    print(directory)
+    if "/updates/" in directory:
+        # Updates directories contain json and squashfs files
+        releases_to_keep = 2 * releases_to_keep
+
+    releases = listdir(directory)
+    releases.sort()
+    files_to_keep = releases[-releases_to_keep:]
+    for file in releases:
+        if file in files_to_keep:
+            print(f"Keep: {file}")
+        else:
+            print(f"Archive: {file}")
+            move(join(directory, file), join(archive, file))
+
+
+def prune_uploaded_images(base_dir: str, archive: str):
+    dirs = get_image_dirs(base_dir)
+    pprint(dirs)
+    makedirs(archive_dir, exist_ok=True)
+    for group in (dirs['beta'], dirs['stable'],
+                  dirs['updates']['beta'], dirs['updates']['stable']):
+        for d in group:
+            if "dev" in d:
+                releases_to_keep = 3
+            else:
+                releases_to_keep = 5
+            prune_directory(d, releases_to_keep, archive)
+
+
+if __name__ == "__main__":
+    root_dir = argv[1]  # /var/www/html/app/files/neon_images
+    archive_dir = argv[2]  # /home/d_mcknight/image_archives
+    prune_uploaded_images(root_dir, archive_dir)
diff --git a/scripts/build_image.sh b/scripts/build_image.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework
+# All trademark and other rights reserved by their respective owners
+# Copyright 2008-2022 Neongecko.com Inc.
+# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds,
+# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo
+# BSD-3 License
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from this
+#    software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS  BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS;  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+debos_dir=${1}  # Absolute path to already cloned neon_debos
+repo_ref=${2}  # dev, master
+recipe=${3}  # debian-neon-image.yml, debian-node-image.yml
+platforms=${4}  # "rpi4 opi5"
+output_dir=${5}  # /var/www/html/app/files/neon_images
+base_url=${6}  # https://2222.us
+os_dir="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."
+timestamp=$(date '+%Y-%m-%d_%H_%M')
+mem_limit=${MEM_LIMIT:-"64G"}
+core_limit=${CORE_LIMIT:-32}
+
+debos_version="$(python3 "${debos_dir}/version.py")"
+
+[ -d "${debos_dir}/output" ] || mkdir "${debos_dir}/output"
+echo "Building recipe with core=${repo_ref} recipe=${debos_version}"
+chmod ugo+x "${debos_dir}/scripts/"*
+
+for platform in ${platforms}; do
+  image_id="${recipe%.*}-${platform}_${timestamp}"
+  # TODO: Refactor builds to be platform-specific and not device-specific
+  if [ "${platform}" == "rpi4" ]; then
+    device="mark_2"
+    kernel_version="6.1.77-gecko+"
+  else
+    device="${platform}"
+    kernel_version="5.10.110-gecko+"
+  fi
+  docker run --rm \
+  --device /dev/kvm \
+  --workdir /image_build \
+  --mount type=bind,source="${debos_dir}",destination=/image_build \
+  --group-add=108 \
+  --security-opt label=disable \
+  --name neon_debos_ghaction \
+  godebos/debos "${recipe}" \
+  -t platform:"${platform}" \
+  -t device:"${device}" \
+  -t kernel_version:"${kernel_version}" \
+  -t architecture:arm64 \
+  -t image:"${image_id}" \
+  -t neon_core:"${repo_ref}" \
+  -t neon_debos:"${debos_version}" \
+  -t build_cores:"${core_limit}" -m "${mem_limit}" -c "${core_limit}" || exit 2
+  echo "Completed build: ${platform}"
+
+  # Determine Server Path for outputs
+  output_path="${output_dir}/${platform}/"
+  update_path="${output_dir}/${platform}/updates/"
+  if [[ "${recipe}" == *node* ]]; then
+    output_path="${output_dir}/node/${platform}/"
+    update_path="${output_dir}/node/${platform}/updates/"
+  elif [[ "${recipe}" == *neon* ]]; then
+    output_path="${output_dir}/core/${platform}/"
+    update_path="${output_dir}/core/${platform}/updates/"
+  fi
+
+  # Ensure directories exist
+  [ -d "${output_path}${repo_ref}" ] || mkdir -p "${output_path}${repo_ref}"
+  [ -d "${update_path}${repo_ref}" ] || mkdir -p "${update_path}${repo_ref}"
+
+  # Add `download_url` metadata to json output
+  url="${base_url}$(sed -e "s|^/var/www/html||g" <<< "${output_path}")${repo_ref}/${image_id}.img.xz"
+  sed -i -e "s|^{|{\n  \"download_url\": \"${url}\",|g" "${debos_dir}/output/"*.json
+  cp "${debos_dir}/output/${image_id}.json" "${os_dir}"  # Copy metadata for upload/parse
+  mv "${debos_dir}/output/${image_id}.img.xz" "${output_path}${repo_ref}/"  # Image File
+  mv "${debos_dir}/output/${image_id}.squashfs" "${update_path}${repo_ref}/"  # Update File
+  mv "${debos_dir}/output/${image_id}.json" "${update_path}${repo_ref}/"  # Update Metadata
+done
+
+echo "completed ${timestamp}"