Added ci scripts and github workflow to use them

With the idea that this can be re-used by other projects as well as locally for developers.

Ticket: ENT-9617
Changelog: none

.github/workflows/build-using-buildscripts.yml

@@ -0,0 +1,108 @@
+name: Build and package
+
+on:
+  workflow_call:
+    secrets:
+      GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO:
+        required: true
+      GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO:
+        required: true
+      GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO:
+        required: true
+
+jobs:
+  build_using_buildscripts:
+    name: Build and package using buildscripts
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout Together Action
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/together-javascript-action
+          ref: v1.7
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_TOGETHER_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Action step
+        uses: ./
+        id: together
+        with:
+          myToken: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout Core
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/core
+          path: core
+          ref: ${{steps.together.outputs.core || github.base_ref || github.ref}}
+          submodules: recursive
+
+      - name: Checkout Masterfiles
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/masterfiles
+          path: masterfiles
+          ref: ${{steps.together.outputs.masterfiles || github.base_ref || github.ref}}
+
+      - name: Checkout Buildscripts (current repo)
+        uses: actions/checkout@v3
+        with:
+          path: buildscripts
+
+      - name: Checkout Nova
+        uses: actions/checkout@v3
+        with:
+          path: nova
+          repository: cfengine/nova
+          ref: ${{steps.together.outputs.nova || github.base_ref || github.ref}}
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_NOVA_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Checkout Enterprise
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/enterprise
+          path: enterprise
+          ref: ${{steps.together.outputs.enterprise || github.base_ref || github.ref}}
+          submodules: recursive
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_ENTERPRISE_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: Checkout Mission Portal
+        uses: actions/checkout@v3
+        with:
+          repository: cfengine/mission-portal
+          path: mission-portal
+          ref: ${{steps.together.outputs.mission-portal || github.base_ref || github.ref}}
+          submodules: recursive
+          ssh-key: ${{ secrets.GH_ACTIONS_SSH_DEPLOY_KEY_MISSION_PORTAL_REPO }}
+          ssh-known-hosts: github.com
+
+      - name: get SHA of buildscripts/deps-packaging last commit
+        run: echo "DEPS_SHA=$(git log --pretty='format:%h' -1 -- .)" | tee -a ${GITHUB_ENV}
+        working-directory: buildscripts/deps-packaging
+
+      - name: get HOME dir
+        run: echo "HOME=$HOME" | tee -a ${GITHUB_ENV}
+
+      - name: Cache Dependencies
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.HOME }}/.cache/
+          key: deps-${{ github.base_ref }}-${{ env.DEPS_SHA }}
+          restore-keys: |
+            deps-${{ github.base_ref }}
+            deps-master
+            deps
+
+      - name: Setup Build Host
+        run: buildscripts/ci/setup.sh
+
+      - name: Build and package
+        run: buildscripts/ci/build.sh
+
+      - name: Upload packages as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: packages
+          path: packages

.gitignore

@@ -1,3 +1,5 @@
 workdir*
 revision
 *.html
+ci/cache
+ci/packages

ci/Dockerfile-cfengine-build-package

@@ -0,0 +1,5 @@
+FROM ubuntu:20.04
+RUN apt-get update -y && apt-get install -y systemd wget sudo
+ADD setup-build-host.sh /
+RUN /setup-build-host.sh
+CMD [ "/lib/systemd/systemd" ]

ci/README

@@ -0,0 +1,45 @@
+In this 'buildscripts/ci' directory you will find the means to build cfengine packages.
+
+Two options: containerize build or "normal" machine (such as virtual machine or actual hardware).
+
+The build is designed for ubuntu-20.04 but could be adjusted in various shell scripts for other platforms. (TODO: do this!)
+
+# containerized build
+./clean.sh # cleans any leftover docker bits
+./run.sh # builds a docker image, sets it up, runs it in the background, copies local cache into container, runs the build and saves the results
+./clean.sh # runs bash in the container for debugging
+
+See /data/buildscripts/ci/build.sh for required environment variables and steps to try manually
+
+# virtual or real machine
+
+For virtual machine such as with vagrant, at $NTECH_ROOT (aka top-level directory containing all CFEngine repositories), init an ubuntu-20.04 vagrant machine so it has access to all your repositories.
+
+vagrant init ubuntu/focal64
+vagrant ssh
+# note however, that currently you will need to copy /vagrant to a non shared filesystem most likely as vboxsf type doesn't support required hard and soft links for packaging :(
+cp -R /vagrant ~/workspace
+
+And then do the following as you would on a real machine:
+./setup.sh # install needed dependencies/packages/etc
+./build.sh # run the build, should generate a package
+
+don't use ./clean.sh ./run.sh or ./shell.sh those are specifically for containerized/docker build
+
+# manual debugging on-host
+
+cat buildscripts/ci/build.sh
+# now export the environment variables there (and adjust as needed)
+export BUILD_TYPE=DEBUG
+export ESCAPETEST=yes
+export TEST_MACHINE=chroot
+
+then take each step one-at-a-time or re-run as needed
+
+./buildscripts/build-scripts/install-dependencies
+
+Note that you can provide an argument to install-dependencies to just build ONE dependency, such as
+
+./buildscripts/build-scripts/install-dependencies lmdb
+
+If you change the version of a dependency the cache for that should be skipped and the dep rebuilt from scratch.

ci/build.sh

@@ -0,0 +1,21 @@
+# build cfengine hub package
+set -ex
+export BUILD_TYPE=DEBUG
+export ESCAPETEST=yes
+export TEST_MACHINE=chroot
+# TODO maybe seed the cache? cp -R buildscripts/ci/cache ~/.cache
+./buildscripts/build-scripts/install-dependencies
+./buildscripts/build-scripts/configure
+./buildscripts/build-scripts/generate-source-tarballs
+./buildscripts/build-scripts/compile
+sudo apt remove -y 'cfbuild*' || true
+sudo apt remove -y 'cfengine-*' || true
+sudo rm -rf /var/cfengine
+sudo rm -rf /opt/cfengine
+./buildscripts/build-scripts/install-dependencies
+./buildscripts/build-scripts/package
+sudo mkdir -p packages
+sudo cp cfengine-nova-hub/*.deb packages/ || true
+sudo cp cfengine-nova-hub/*.rpm packages/ || true
+
+# todo maybe save the cache cp -R ~/.cache buildscripts/ci/cache

ci/clean.sh

@@ -0,0 +1,6 @@
+# clean up docker stuff
+name=cfengine-build-package
+# TODO: a softer clean might get into the container and run ./buildscripts/build-scripts/clean-buildmachine
+docker stop $name
+docker rm $name
+#docker rmi $name

ci/run.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# run the build in a docker container
+set -ex
+
+# find the dir two levels up from here, home of all the repositories
+COMPUTED_ROOT=$(readlink -e $(dirname "$0")/../../)
+# NTECH_ROOT should be the same, but if available use it so user can do their own thing.
+NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+
+name=cfengine-build-package
+# todo, check the image against the Dockerfile for up-to-date ness?
+docker build -t $name -f ./Dockerfile-$name . || true
+# todo, check if already running and up-to-date?
+docker run -d --privileged -v ${NTECH_ROOT}:/data --name $name $name || true
+docker exec -i $name bash -c 'mkdir -p /root/.cache'
+docker cp cache $name:/root/.cache
+docker exec -i $name bash -c 'cd /data; ./buildscripts/ci/build.sh'
+docker cp $name:/root/.cache cache
+docker cp $name:/data/packages .

ci/setup.sh

@@ -0,0 +1,86 @@
+# setup build host on ubuntu 20
+PREFIX=/var/cfengine
+
+# Github Actions provides machines with various packages installed,
+# what confuses our build system into thinking that it's an RPM distro.
+sudo rm -f /bin/rpm
+export EXPLICIT_ROLE=hub
+
+# Install dependencies
+sudo apt-get update -y
+
+# Install Python2 and psycopg2
+sudo apt-get -qy install python2
+wget https://bootstrap.pypa.io/pip/2.7/get-pip.py -O get-pip.py
+sudo python2 get-pip.py
+sudo pip install psycopg2-binary
+
+# Remove libltdl
+sudo apt-get -qy purge 'libltdl*'
+
+# remove unwanted packages
+sudo apt-get -qq purge apache* "postgresql*" redis*
+
+(
+if test -f "mission-portal/public/scripts/package.json"; then
+	# packages needed for installing Mission portal dependencies
+	sudo apt-get -qq -y install npm
+	cd mission-portal/public/scripts
+	# install dependencies from npmjs
+	npm i
+fi
+)
+
+# install composer and friends
+sudo apt-get -qq -y install curl php php-curl php-zip php-mbstring php-xml php-gd composer
+
+(
+if test -f "mission-portal/composer.json"; then
+	cd mission-portal
+	# install PHP dependencies from composer
+	composer install
+fi
+)
+
+(
+if test -f "nova/api/http/composer.json"; then
+	cd nova/api/http/
+	# install PHP dependencies from composer
+	composer install --ignore-platform-reqs
+fi
+)
+
+(
+if test -f "mission-portal/public/themes/default/bootstrap/cfengine_theme.less"; then
+	sudo apt-get -qq -y install npm
+	cd mission-portal/public/themes/default/bootstrap
+	npx -p less lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css
+fi
+)
+
+(
+if test -f "mission-portal/ldap/composer.json"; then
+	sudo apt-get -qq -y install php-ldap
+	cd mission-portal/ldap
+	# install PHP dependencies from composer
+	composer install
+fi
+)
+
+# packages needed for autogen
+sudo apt-get -qy install git autoconf automake m4 make bison flex \
+	binutils libtool gcc g++ libc-dev libpam0g-dev python2 python3 psmisc \
+	libtokyocabinet-dev libssl-dev libpcre3-dev default-jre-headless
+
+NO_CONFIGURE=1 PROJECT=nova ./buildscripts/build-scripts/autogen
+
+# packages needed for building
+sudo apt-get -qy install bison flex binutils build-essential fakeroot ntp \
+	dpkg-dev libpam0g-dev python2 python3 debhelper pkg-config psmisc nfs-common \
+	dpkg-dev debhelper g++ libncurses5 pkg-config build-essential libpam0g-dev fakeroot rsync gcc make sudo wget
+
+# remove unwanted dependencies
+sudo apt-get -qy purge libltdl-dev libltdl7 #libtool
+
+# needed for cfengine-nova-hub.deb packaging
+sudo apt-get install -qy python3-pip

ci/shell.sh

@@ -0,0 +1,2 @@
+# shell into the docker container
+docker exec -it cfengine-build-package bash