Merge pull request #509 from infrawatch/import/stf153

Import master to stable-1.5

.github/workflows/main.yml

@@ -2,8 +2,8 @@ name: CI
 on: push
 
 jobs:
-  linting:
-    name: Linting
+  operator-linting:
+    name: Operator Linting
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout code
@@ -16,11 +16,53 @@ jobs:
         run: ansible-galaxy collection install operator_sdk.util
 
       - name: Install ansible-lint
-        run: pip install 'ansible-lint < 6.0.0'
+        run: python -m pip install 'ansible-lint < 6.0.0'
 
       - name: Lint Ansible roles/servicetelemetry/ directory
         run: ${HOME}/.local/bin/ansible-lint roles/servicetelemetry
 
+  stf-run-ci-linting:
+    name: stf-run-ci Linting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install Ansible
+        run: python -m pip install 'ansible'
+
+      - name: Install ansible-lint
+        run: python -m pip install 'ansible-lint'
+
+      - name: Lint Ansible build/stf-run-ci  directory
+        run: ${HOME}/.local/bin/ansible-lint .
+        working-directory: ./build/stf-run-ci
+
+  generate-bundle-diff-check:
+    name: Check if generating the bundle would result in local changes
+    runs-on: ubuntu-latest
+    env:
+      RELEASE_VERSION: v0.19.4
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Get operator-sdk image 0.19.4
+        run: curl --output operator-sdk -JL https://github.com/operator-framework/operator-sdk/releases/download/$RELEASE_VERSION/operator-sdk-$RELEASE_VERSION-x86_64-linux-gnu
+
+      - name: Make operator-sdk executable
+        run: chmod +x operator-sdk
+
+      - name: Move operator-sdk binary
+        run: sudo mv operator-sdk /usr/local/bin
+
+      - name: Generate bundle locally
+        run: operator-sdk generate bundle --manifests --metadata --default-channel stable-1.5 --channels stable-1.5
+
+      - name: Check if bundle generation results in local changes
+        run: git diff --exit-code
+
   build-operator-check:
     name: Build Operator check
     runs-on: ubuntu-20.04

.gitleaks.toml

@@ -0,0 +1,8 @@
+[allowlist]
+  description = "Global Allowlist"
+
+  # Ignore based on any subset of the file path
+  paths = [
+    # Ignore all example certs
+    '''roles\/servicetelemetry\/vars\/dummy_user_certs\.yml'''
+  ]

.jenkins/Dockerfile

@@ -10,6 +10,7 @@ RUN jenkins-plugin-cli -p ant:latest \
   email-ext:latest \
   git:latest \
   github-branch-source:latest \
+  github-scm-trait-notification-context:latest \
   gradle:latest \
   ldap:latest \
   mailer:latest \

.jenkins/README.md

@@ -39,6 +39,8 @@ oc apply -f deploy/service-route.yaml
 export SMEE_CHANNEL=<YOUR_SMEE_CHANNEL>  #(just the slug, not the whole URL)
 export GH_ORG=<YOUR_GITHUB_ORGANIZATION>
 export JENKINS_URL=$(oc get route jenkins -ojsonpath='{.spec.host}')
+# This is for labelling the status that is returned to github
+export OCP_VERSION=<ocp version>  # e.g. 4.14
 
 for f in deploy/*; do
   envsubst < "${f}" | oc apply -f -

.jenkins/agent/Dockerfile

@@ -1,7 +1,7 @@
 FROM quay.io/openshift/origin-jenkins-agent-base:latest
 
 # pass --build-arg OC_CLIENT_VERSION=<version> to build stage to change client version
-ARG OC_CLIENT_VERSION="4.12"
+ARG OC_CLIENT_VERSION="4.13"
 
 RUN curl -LO "https://github.com/operator-framework/operator-sdk/releases/download/v0.19.4/operator-sdk-v0.19.4-x86_64-linux-gnu" && \
     chmod +x operator-sdk-v0.19.4-x86_64-linux-gnu && mv operator-sdk-v0.19.4-x86_64-linux-gnu /usr/local/bin/operator-sdk

.jenkins/deploy/casc-configmap.yaml

@@ -94,6 +94,11 @@ data:
                     // 1 : Forks in the same account
                     // 2 : Nobody
                 }
+                // Custom Github Notification Context; https://github.com/jenkinsci/github-scm-trait-notification-context-plugin
+                traits << 'org.jenkinsci.plugins.githubScmTraitNotificationContext.NotificationContextTrait' {
+                   contextLabel("continuous-integration/jenkins/ocp-${OCP_VERSION}")
+                   typeSuffix(true)
+                }
             }
 
               // "Project Recognizers"

Jenkinsfile

@@ -15,7 +15,7 @@ metadata:
   name: default
   namespace: ${namespace}
 spec:
-  observabilityStrategy: use_community
+  observabilityStrategy: use_redhat
   alerting:
     alertmanager:
       storage:
@@ -36,6 +36,7 @@ spec:
           strategy: ephemeral
   transports:
     qdr:
+      auth: none
       enabled: true
       deploymentSize: 1
       web:
@@ -177,7 +178,7 @@ pipeline {
 								openshift.withProject(namespace) {
 									timeout(time: 800, unit: 'SECONDS') {
 										openshift.create(stf_resource)
-										sh "OCP_PROJECT=${namespace} ./build/validate_deployment.sh"
+										sh "OCP_PROJECT=${namespace} VALIDATION_SCOPE=use_redhat ./build/validate_deployment.sh"
 									}
 								}
 							}

build/Dockerfile

@@ -1,5 +1,21 @@
-FROM quay.io/openshift/origin-ansible-operator:4.10
+FROM quay.io/openshift/origin-ansible-operator:4.12
 
+# temporarily switch to root user to adjust image layers
+USER 0
+# Upstream CI builds need the additional EPEL sources for python3-passlib and python3-bcrypt but have no working repos to install epel-release
+# NO_PROXY is undefined in upstream CI builds, but defined (usually blank) during openshift builds (a possibly brittle hack)
+RUN bash -c -- 'if [ "${NO_PROXY:-__ZZZZZ}" == "__ZZZZZ" ]; then echo "Applying upstream EPEL hacks" && echo -e "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFz3zvsBEADJOIIWllGudxnpvJnkxQz2CtoWI7godVnoclrdl83kVjqSQp+2\ndgxuG5mUiADUfYHaRQzxKw8efuQnwxzU9kZ70ngCxtmbQWGmUmfSThiapOz00018\n+eo5MFabd2vdiGo1y+51m2sRDpN8qdCaqXko65cyMuLXrojJHIuvRA/x7iqOrRfy\na8x3OxC4PEgl5pgDnP8pVK0lLYncDEQCN76D9ubhZQWhISF/zJI+e806V71hzfyL\n/Mt3mQm/li+lRKU25Usk9dWaf4NH/wZHMIPAkVJ4uD4H/uS49wqWnyiTYGT7hUbi\necF7crhLCmlRzvJR8mkRP6/4T/F3tNDPWZeDNEDVFUkTFHNU6/h2+O398MNY/fOh\nyKaNK3nnE0g6QJ1dOH31lXHARlpFOtWt3VmZU0JnWLeYdvap4Eff9qTWZJhI7Cq0\nWm8DgLUpXgNlkmquvE7P2W5EAr2E5AqKQoDbfw/GiWdRvHWKeNGMRLnGI3QuoX3U\npAlXD7v13VdZxNydvpeypbf/AfRyrHRKhkUj3cU1pYkM3DNZE77C5JUe6/0nxbt4\nETUZBTgLgYJGP8c7PbkVnO6I/KgL1jw+7MW6Az8Ox+RXZLyGMVmbW/TMc8haJfKL\nMoUo3TVk8nPiUhoOC0/kI7j9ilFrBxBU5dUtF4ITAWc8xnG6jJs/IsvRpQARAQAB\ntChGZWRvcmEgRVBFTCAoOCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQI4BBMB\nAgAiBQJc9877AhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAh6kWrL4bW\noWagD/4xnLWws34GByVDQkjprk0fX7Iyhpm/U7BsIHKspHLL+Y46vAAGY/9vMvdE\n0fcr9Ek2Zp7zE1RWmSCzzzUgTG6BFoTG1H4Fho/7Z8BXK/jybowXSZfqXnTOfhSF\nalwDdwlSJvfYNV9MbyvbxN8qZRU1z7PEWZrIzFDDToFRk0R71zHpnPTNIJ5/YXTw\nNqU9OxII8hMQj4ufF11040AJQZ7br3rzerlyBOB+Jd1zSPVrAPpeMyJppWFHSDAI\nWK6x+am13VIInXtqB/Cz4GBHLFK5d2/IYspVw47Solj8jiFEtnAq6+1Aq5WH3iB4\nbE2e6z00DSF93frwOyWN7WmPIoc2QsNRJhgfJC+isGQAwwq8xAbHEBeuyMG8GZjz\nxohg0H4bOSEujVLTjH1xbAG4DnhWO/1VXLX+LXELycO8ZQTcjj/4AQKuo4wvMPrv\n9A169oETG+VwQlNd74VBPGCvhnzwGXNbTK/KH1+WRH0YSb+41flB3NKhMSU6dGI0\nSGtIxDSHhVVNmx2/6XiT9U/znrZsG5Kw8nIbbFz+9MGUUWgJMsd1Zl9R8gz7V9fp\nn7L7y5LhJ8HOCMsY/Z7/7HUs+t/A1MI4g7Q5g5UuSZdgi0zxukiWuCkLeAiAP4y7\nzKK4OjJ644NDcWCHa36znwVmkz3ixL8Q0auR15Oqq2BjR/fyog==\n=84m8\n-----END PGP PUBLIC KEY BLOCK-----" > /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-8 && echo -e "[epel]\nname=Extra Packages for Enterprise Linux 8 - \$basearch\nmetalink=https://mirrors.fedoraproject.org/metalink?repo=epel-8&arch=\$basearch&infra=\$infra&content=\$contentdir\nenabled=1\ngpgcheck=1\ngpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-8" > /etc/yum.repos.d/epel.repo; fi'
+
+# update the base image to allow forward-looking optimistic updates during the testing phase, with the added benefit of helping move closer to passing security scans.
+# -- excludes ansible so it remains at 2.9 tag as shipped with the base image
+# -- installs python3-passlib and python3-bcrypt for oauth-proxy interface
+# -- cleans up the cached data from dnf to keep the image as small as possible
+RUN dnf update -y --exclude=ansible* && dnf install -y python3-passlib python3-bcrypt && dnf clean all && rm -rf /var/cache/dnf
+
+# switch back to user 1001 when running the base image (non-root)
+USER 1001
+
+# copy in required artifacts for the operator
 COPY watches.yaml ${HOME}/watches.yaml
 COPY roles/ ${HOME}/roles/
 COPY collections/ ${HOME}/.ansible/collections/

build/generate_bundle.sh

@@ -1,41 +1,52 @@
 #!/usr/bin/env bash
 set -e
-REL=$(dirname "$0")
+set -x
+
+LOGFILE=${LOGFILE:-/dev/null}
+
+# If LOGFILE is /dev/null, this command fails, so ignore that error
+truncate --size=0 ${LOGFILE} || true
+
+OPERATOR_SDK=${OPERATOR_SDK:-operator-sdk}
+
+REL=$( readlink -f $(dirname "$0"))
 
 # shellcheck source=build/metadata.sh
 . "${REL}/metadata.sh"
 
 generate_version() {
-    echo "-- Generating operator version"
     UNIXDATE=$(date '+%s')
     OPERATOR_BUNDLE_VERSION=${OPERATOR_CSV_MAJOR_VERSION}.${UNIXDATE}
-    echo "---- Operator Version: ${OPERATOR_BUNDLE_VERSION}"
 }
 
 create_working_dir() {
-    echo "-- Create working directory"
     WORKING_DIR=${WORKING_DIR:-"/tmp/${OPERATOR_NAME}-bundle-${OPERATOR_BUNDLE_VERSION}"}
     mkdir -p "${WORKING_DIR}"
-    echo "---- Created working directory: ${WORKING_DIR}"
 }
 
 generate_dockerfile() {
-    echo "-- Generate Dockerfile for bundle"
     sed -E "s#<<OPERATOR_BUNDLE_VERSION>>#${OPERATOR_BUNDLE_VERSION}#g;s#<<BUNDLE_CHANNELS>>#${BUNDLE_CHANNELS}#g;s#<<BUNDLE_DEFAULT_CHANNEL>>#${BUNDLE_DEFAULT_CHANNEL}#g" "${REL}/../${BUNDLE_PATH}/Dockerfile.in" > "${WORKING_DIR}/Dockerfile"
-    echo "---- Generated Dockerfile complete"
 }
 
 generate_bundle() {
-    echo "-- Generate bundle"
-    REPLACE_REGEX="s#<<CREATED_DATE>>#${CREATED_DATE}#g;s#<<OPERATOR_IMAGE>>#${OPERATOR_IMAGE}#g;s#<<OPERATOR_TAG>>#${OPERATOR_TAG}#g;s#<<RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP>>#${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP}#g;s#<<RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG>>#${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG}#g;s#<<OPERATOR_BUNDLE_VERSION>>#${OPERATOR_BUNDLE_VERSION}#g;s#1.99.0#${OPERATOR_BUNDLE_VERSION}#g;s#<<OPERATOR_DOCUMENTATION_URL>>#${OPERATOR_DOCUMENTATION_URL}#g;s#<<BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND>>#${BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND}#g"
+    REPLACE_REGEX="s#<<CREATED_DATE>>#${CREATED_DATE}#g;s#<<OPERATOR_IMAGE>>#${OPERATOR_IMAGE}#g;s#<<OPERATOR_TAG>>#${OPERATOR_TAG}#g;s#<<RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP>>#${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP}#g;s#<<RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG>>#${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG}#g;s#<<RELATED_IMAGE_OAUTH_PROXY>>#${RELATED_IMAGE_OAUTH_PROXY}#g;s#<<RELATED_IMAGE_OAUTH_PROXY_TAG>>#${RELATED_IMAGE_OAUTH_PROXY_TAG}#g;s#<<OPERATOR_BUNDLE_VERSION>>#${OPERATOR_BUNDLE_VERSION}#g;s#1.99.0#${OPERATOR_BUNDLE_VERSION}#g;s#<<OPERATOR_DOCUMENTATION_URL>>#${OPERATOR_DOCUMENTATION_URL}#g;s#<<BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND>>#${BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND}#g"
 
-    pushd "${REL}/../"
-    ${OPERATOR_SDK} generate bundle --channels ${BUNDLE_CHANNELS} --default-channel ${BUNDLE_DEFAULT_CHANNEL} --manifests --metadata --version "${OPERATOR_BUNDLE_VERSION}" --output-dir "${WORKING_DIR}"
-    popd
+    pushd "${REL}/../" > /dev/null 2>&1
+    ${OPERATOR_SDK} generate bundle --verbose --channels ${BUNDLE_CHANNELS} --default-channel ${BUNDLE_DEFAULT_CHANNEL} --manifests --metadata --version "${OPERATOR_BUNDLE_VERSION}" --output-dir "${WORKING_DIR}" >> ${LOGFILE} 2>&1
+    popd > /dev/null 2>&1
 
-    echo "---- Replacing variables in generated manifest"
     sed -i -E "${REPLACE_REGEX}" "${WORKING_DIR}/manifests/${OPERATOR_NAME}.clusterserviceversion.yaml"
-    echo "---- Generated bundle complete at ${WORKING_DIR}/manifests/${OPERATOR_NAME}.clusterserviceversion.yaml"
+}
+
+copy_extra_metadata() {
+    # We add this because our version of operator-sdk for building doesn't
+    # understand these files, but newer versions of operator-sdk (for testing
+    # purposes) does, and newer versions of opm (as used in both downstream and
+    # upstream index image builds) also understands these files. Just copy them
+    # into the bundle directory during building.
+    pushd "${REL}/../" > /dev/null 2>&1
+    cp -r ./deploy/olm-catalog/service-telemetry-operator/tests/ "${WORKING_DIR}"
+    cp ./deploy/olm-catalog/service-telemetry-operator/metadata/properties.yaml "${WORKING_DIR}/metadata/"
 }
 
 copy_extra_metadata() {
@@ -58,11 +69,15 @@ build_bundle_instructions() {
 
 
 # generate templates
-echo "## Begin bundle creation"
+#echo "## Begin bundle creation"
 generate_version
 create_working_dir
 generate_dockerfile
 generate_bundle
 copy_extra_metadata
-build_bundle_instructions
-echo "## End Bundle creation"
+#build_bundle_instructions
+#echo "## End Bundle creation"
+
+set +x
+JSON_OUTPUT='{"operator_bundle_image":"%s","operator_bundle_version":"%s","operator_image":"%s","bundle_channels":"%s","bundle_default_channel":"%s","operator_tag":"%s","working_dir":"%s"}'
+printf "$JSON_OUTPUT" "$OPERATOR_BUNDLE_IMAGE" "$OPERATOR_BUNDLE_VERSION" "$OPERATOR_IMAGE" "$BUNDLE_CHANNELS" "$BUNDLE_DEFAULT_CHANNEL" "$OPERATOR_TAG" "$WORKING_DIR"

build/get_operator_sdk.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -x
+
+REL=$(dirname "$0")
+ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac)
+OS=$(uname | awk '{print tolower($0)}')
+VERSION="${1:-v1.5.0}"
+OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/${VERSION}
+
+if [[ ! -f ${REL}/working/operator-sdk-${VERSION} ]]; then
+	mkdir ${REL}/working
+	if [[ "${VERSION}" =~ "v0" ]]; then
+		# naming scheme for v0.x is operator-sdk-$VERSION-$ARCH-$OS e.g. operator-sdk-v0.19.4-x86_64-linux-gnu
+		curl -L ${OPERATOR_SDK_DL_URL}/operator-sdk-${VERSION}-x86_64-linux-gnu -o ${REL}/working/operator-sdk-${VERSION}
+    else
+		# naming scheme for v1.x is operator-sdk_$OS-$ARCH e.g. operator-sdk_linux_amd64
+		curl -L ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} -o ${REL}/working/operator-sdk-${VERSION}
+	fi
+	chmod +x ${REL}/working/operator-sdk-${VERSION}
+	rm -f ${REL}/working/operator-sdk
+fi
+
+set +x

build/metadata.sh

@@ -19,6 +19,8 @@ BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND=${BUNDLE_OLM_SKIP_RANGE_LOWER_BOUND:-1.3.0}
 CREATED_DATE=${CREATED_DATE:-$(date +'%Y-%m-%dT%H:%M:%SZ')}
 RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP=${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP:-quay.io/infrawatch/prometheus-webhook-snmp}
 RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG=${RELATED_IMAGE_PROMETHEUS_WEBHOOK_SNMP_TAG:-stable-1.5}
+RELATED_IMAGE_OAUTH_PROXY=${RELATED_IMAGE_OAUTH_PROXY:-quay.io/openshift/origin-oauth-proxy}
+RELATED_IMAGE_OAUTH_PROXY_TAG=${RELATED_IMAGE_OAUTH_PROXY_TAG:-latest}
 BUNDLE_PATH=${BUNDLE_PATH:-deploy/olm-catalog/service-telemetry-operator}
 BUNDLE_CHANNELS=${BUNDLE_CHANNELS:-stable-1.5}
 BUNDLE_DEFAULT_CHANNEL=${BUNDLE_DEFAULT_CHANNEL:-stable-1.5}

build/run-ci.yaml

@@ -1,9 +1,15 @@
 ---
 # run STF CI setup in CRC (already provisioned)
 - hosts: localhost
-  gather_facts: no
+  gather_facts: yes
   connection: local
   tasks:
   - name: Run the STF CI system
     import_role:
-        name: stf-run-ci
+      name: stf-run-ci
+
+  - name: Collect the logs
+    import_role:
+      name: stf-collect-logs
+    vars:
+      logfile_dir: "{{ playbook_dir }}/"

build/stf-collect-logs/README.md

@@ -0,0 +1,38 @@
+stf-collect-logs
+================
+
+This role collects logs that are useful for debugging an STF deployment.
+
+Once the logs are collected, the user will need to fetch the logs themselves.
+
+Requirements
+------------
+
+
+Role Variables
+--------------
+
+* `logfile_dir` - The location that the logs will be created in on the remote host(s).
+
+Dependencies
+------------
+
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+Apache 2
+
+Author Information
+------------------
+
+Red Hat

build/stf-collect-logs/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for stf-collect-logs

build/stf-collect-logs/meta/main.yml

@@ -0,0 +1,15 @@
+galaxy_info:
+  role_name: stf-collect-logs
+  namespace: infrawatch
+
+  author: InfraWatch
+  description: Log collection role for Service Telemetry Framework
+  company: Red Hat
+
+  license: Apache-2.0
+
+  min_ansible_version: 2.1
+
+  galaxy_tags: []
+
+dependencies: []