Merge branch 'master' of https://github.com/synocommunity/spksrc into…

… lldpd

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -11,7 +11,7 @@ body:
   attributes:
     label: Is this a new Bug?
     options:
-      - label: "I checkd that the bug hasn't been reported before"
+      - label: "I checked that the bug hasn't been reported before"
         required: true
 
 - type: input
@@ -39,7 +39,7 @@ body:
   id: arch
   attributes:
     label: Device Architecture
-    description: Which CPU Architecture does your NAS have? [You can look up the Architectrue for your NAS here](https://github.com/SynoCommunity/spksrc/wiki/Synology-and-SynoCommunity-Package-Architectures)
+    description: Which CPU Architecture does your NAS have? [You can look up the Architecture for your NAS here](https://github.com/SynoCommunity/spksrc/wiki/Synology-and-SynoCommunity-Package-Architectures)
     options:
       - x86_64
       - AArch64 (ARMv8)
@@ -59,7 +59,7 @@ body:
     required: true
 
 - type: textarea
-  id: decription
+  id: description
   attributes:
     label: What happened?
     description: |
@@ -69,16 +69,15 @@ body:
   validations:
     required: true
 - type: textarea
-  id: repoduce
+  id: reproduce
   attributes:
     label: Reproduction steps
     description: "How do you trigger this bug? Please walk us through it step by step."
     value: |
-      1.
-      2.
-      3.
+      1. 
+      2. 
+      3. 
       ...
-    render: bash
   validations:
     required: true
 - type: textarea

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,7 +15,7 @@ Fixes # <!--Optionally, add links to existing issues or other PR's-->
 
 ### Type of change
 
-<!--Please use any relavent tags.-->
+<!--Please use any relevant tags.-->
 - [ ] Bug fix
 - [ ] New Package
 - [ ] Package update

.github/actions/build.sh

@@ -9,11 +9,14 @@
 # - Build for arch defined by ${GH_ARCH} (e.g. x64-6.1, noarch, ...).
 # - Successfully built packages are logged to $BUILD_SUCCESS_FILE.
 # - Failed builds are logged to ${BUILD_ERROR_FILE} and annotated as error.
+# - For failed builds, the make command and the latest 15 lines of the build output are written to ${BUILD_ERROR_LOGFILE}
 # - The build output is structured into log groups by package.
 # - As the disk space in the workflow environment is limitted, we clean the
 #   work folder of each package after build. At 2020.06 this limit is 14GB.
-# - ffmpeg is not cleaned to be available for dependents.
-# - Therefore ffmpeg is built first if triggered by its own or a dependent (see prepare.sh).
+# - synocli-videodriver and ffmpeg5-7 are not cleaned to be available for dependents.
+# - Therefore synocli-videodriver is built first if triggered by ffmpeg5-7
+# - Therefore ffmpeg5 and ffmpeg7 are built second if triggered by its
+#   own or a dependent (see prepare.sh).
 
 set -o pipefail
 
@@ -22,12 +25,20 @@ make setup-synocommunity
 sed -i -e "s|#PARALLEL_MAKE\s*=.*|PARALLEL_MAKE=max|" \
     -e "s|PUBLISH_API_KEY\s*=.*|PUBLISH_API_KEY=$API_KEY|" \
     local.mk
+# Git >= 2.35.2 stops directory traversals when ownership changes from the current user (in response to CVE-2022-24765). 
+# This prevents errors on nested repos that might have different file owner.
+git config --global --add safe.directory "/github/workspace"
 echo "::endgroup::"
 
 echo "===> TARGET: ${GH_ARCH}"
 echo "===> ARCH   packages: ${ARCH_PACKAGES}"
 echo "===> NOARCH packages: ${NOARCH_PACKAGES}"
 
+# Remove rust toolchain status file to enfore re-installing cargo/rust
+# This fixes isees on github-action where toolchain caching omits the
+# actual installation state of cargo/rust within the distrib folder
+rm -f toolchain/syno-${GH_ARCH}/work/.rustc_done
+
 if [ "${GH_ARCH%%-*}" = "noarch" ]; then
     build_packages=${NOARCH_PACKAGES}
 else
@@ -50,28 +61,39 @@ if [ -n "$API_KEY" ] && [ "$PUBLISH" == "true" ]; then
 fi
 
 # Build
-PACKAGES_TO_KEEP="ffmpeg"
+PACKAGES_TO_KEEP="synocli-videodriver ffmpeg5 ffmpeg7 python310 python311"
 for package in ${build_packages}
 do
     echo "::group:: ---- build ${package}"
     echo >build.log
 
     if [ "${GH_ARCH%%-*}" != "noarch" ]; then
-        echo "$ make ${MAKE_ARGS}arch-${GH_ARCH%%-*}-${GH_ARCH##*-} -C ./spk/${package}" >>build.log
-        make ${MAKE_ARGS}arch-${GH_ARCH%%-*}-${GH_ARCH##*-} -C ./spk/${package} |& tee >(tail -15 >>build.log)
+        if [ "${package}" == "${PACKAGE_TO_PUBLISH}" ]; then
+            echo "$ make ${MAKE_ARGS}arch-${GH_ARCH%%-*}-${GH_ARCH##*-} -C ./spk/${package}" >>build.log
+            make ${MAKE_ARGS}arch-${GH_ARCH%%-*}-${GH_ARCH##*-} -C ./spk/${package} |& tee >(tail -15 >>build.log)
+        else
+            echo "$ make arch-${GH_ARCH%%-*}-${GH_ARCH##*-} -C ./spk/${package}" >>build.log
+            make arch-${GH_ARCH%%-*}-${GH_ARCH##*-} -C ./spk/${package} |& tee >(tail -15 >>build.log)
+        fi
     else
         if [ "${GH_ARCH}" = "noarch" ]; then
             TCVERSION=
         else
             TCVERSION=${GH_ARCH##*-}
         fi
-        echo "$ make TCVERSION=${TCVERSION} ARCH= -C ./spk/${package} ${MAKE_ARGS%%-}" >>build.log
-        make TCVERSION=${TCVERSION} ARCH= -C ./spk/${package} ${MAKE_ARGS%%-} |& tee >(tail -15 >>build.log)
+        # noarch package must be first built then published
+        echo "$ make TCVERSION=${TCVERSION} ARCH= -C ./spk/${package}" >>build.log
+        make TCVERSION=${TCVERSION} ARCH= -C ./spk/${package} |& tee >(tail -15 >>build.log)
+
+        if [ "${package}" == "${PACKAGE_TO_PUBLISH}" ]; then
+            echo "$ make TCVERSION=${TCVERSION} ARCH= -C ./spk/${package} ${MAKE_ARGS%%-}" >>build.log
+            make TCVERSION=${TCVERSION} ARCH= -C ./spk/${package} ${MAKE_ARGS%%-} |& tee >(tail -15 >>build.log)
+        fi
     fi
     result=$?
 
     # For a build to succeed a <package>_<arch>-<version>.spk must also be generated
-    if [ ${result} -eq 0 -a "$(ls -1 ./packages/$(sed -n -e '/^SPK_NAME/ s/.*= *//p' spk/${package}/Makefile)_*.spk)" ]; then
+    if [ ${result} -eq 0 -a "$(ls -1 ./packages/$(sed -n -e '/^SPK_NAME/ s/.*= *//p' spk/${package}/Makefile)_*.spk 2> /dev/null)" ]; then
         echo "$(date --date=now +"%Y.%m.%d %H:%M:%S") - ${package}: (${GH_ARCH}) DONE"   >> ${BUILD_SUCCESS_FILE}
     # Ensure it's not a false-positive due to pre-check
     elif tail -15 build.log | grep -viq 'spksrc.pre-check.mk'; then
@@ -81,7 +103,10 @@ do
 
     if [ "$(echo ${PACKAGES_TO_KEEP} | grep -ow ${package})" = "" ]; then
         # free disk space (but not for packages to keep)
-        make -C ./spk/${package} clean
+        make -C ./spk/${package} clean |& tee >(tail -15 >>build.log)
+    else
+        # free disk space by removing source and staging directories (for packages to keep)
+        make arch-${GH_ARCH%%-*}-${GH_ARCH##*-} -C ./spk/${package} clean-source |& tee >(tail -15 >>build.log)
     fi
 
     echo "::endgroup::"

.github/actions/build_status.sh

@@ -15,6 +15,7 @@
 # BUILD_SUCCESS_FILE        defines the name of the file with built packages
 # BUILD_UNSUPPORTED_FILE    defines the name of the file with unsupported packages
 # BUILD_ERROR_FILE          defines the name of the file with build errors
+# BUILD_ERROR_LOGFILE       defines the name of the file with last 15 lines of build output for failed packages
 #
 
 echo ""
@@ -38,7 +39,7 @@ fi
 echo ""
 echo "UNSUPPORTED (skipped):"
 if [ -f "${BUILD_UNSUPPORTED_FILE}" ]; then
-    cat "${BUILD_UNSUPPORTED_FILE}"
+    cat "${BUILD_UNSUPPORTED_FILE}" | awk '!seen[$0]++'
     if [ -f "${BUILD_ERROR_FILE}" ]; then
         # remove unsupported packages from errors:
         unsupported_packages=$(cat "${BUILD_UNSUPPORTED_FILE}" | grep -Po "\- \K.*:" | sort -u | tr '\n' '|' | sed -e 's/|$//')

.github/actions/download.sh

@@ -6,16 +6,35 @@
 #
 # Functions:
 # - Download all referenced native and cross source files for packages to build.
+# - Download all referenced python wheels needed to build.
+# - use download-all target to get all files when package has multiple (arch specific) files.
 
 set -o pipefail
 
+# Download regular cross/* sources
 if [ -z "${DOWNLOAD_PACKAGES}" ]; then
     echo "===> No packages to download. <==="
 else
     echo "===> Download packages: ${DOWNLOAD_PACKAGES}"
     for download in ${DOWNLOAD_PACKAGES}
     do
-        echo "$ make -c ${download} download"
-        make -C ${download} download
+        echo "$ make -c ${download} download-all"
+        make -C ${download} download-all
     done
 fi
+
+echo ""
+
+# Download python wheel sources files
+build_packages="${NOARCH_PACKAGES} ${ARCH_PACKAGES}"
+
+if [ -z "${build_packages}" ]; then
+    echo "===> No wheels to download. <==="
+else
+    for package in ${build_packages}; do
+        echo "===> Download wheels: ${package}"
+        make -C spk/${package} wheeldownload
+    done
+fi
+
+echo ""

.github/actions/prepare.sh

@@ -6,7 +6,8 @@
 #
 # Functions:
 # - Evaluate all packages to build depending on files defined in ${GH_FILES}.
-# - ffmpeg is moved to head of packages to built first if triggered by its own or a dependent.
+# - synocli-videodriver is moved to head of packages to build first if triggered by its ffmpeg5-7
+# - python310-311 and ffmpeg5-7 are moved to head of remaining packages to build when triggered by its own or a dependent.
 # - Referenced native and cross packages of the packages to build are added to the download list.
 
 set -o pipefail
@@ -19,7 +20,7 @@ DEFAULT_TC=$(grep DEFAULT_TC local.mk | cut -f2 -d= | xargs)
 
 # filter for changes made in the spk directories and take unique package name (without spk folder)
 SPK_TO_BUILD+=" "
-SPK_TO_BUILD+=$(echo "${GH_FILES}" | tr ' ' '\n' | grep -oP "(spk)/\K[^\/]*" | sort -u | tr '\n' ' ')
+SPK_TO_BUILD+=$(echo "${GH_FILES}" | tr ' ' '\n' | grep -oP "^spk/\K[^\/]*" | sort -u | tr '\n' ' ')
 
 # filter for changes made in the cross and native directories and take unique package name (including cross or native folder)
 DEPENDENT_PACKAGES=$(echo "${GH_FILES}" | tr ' ' '\n' | grep -oP "(cross|native)/[^\/]*" | sort -u | tr '\n' ' ')
@@ -30,44 +31,76 @@ echo "Building dependency list..."
 DEPENDENCY_LIST=
 for package in $(find spk/ -maxdepth 1 -type d | cut -c 5- | sort)
 do
-    DEPENDENCY_LIST+=$(make -s -C spk/${package} dependency-list)$'\n'
+    if [ ! -f "./spk/${package}/BROKEN" ]; then
+        DEPENDENCY_LIST+=$(DEPENDENCY_WALK=1 make -s -C spk/${package} dependency-list 2> /dev/null)$'\n'
+    fi
 done
 
 # search for dependent spk packages
 for package in ${DEPENDENT_PACKAGES}
 do
     echo "===> Searching for dependent package: ${package}"
-    packages=$(echo "${DEPENDENCY_LIST}" | grep -w "${package}" | grep -o ".*:" | tr ':' ' ' | sort -u | tr '\n' ' ')
+    packages=$(echo "${DEPENDENCY_LIST}" | grep " ${package} " | grep -o ".*:" | tr ':' ' ' | sort -u | tr '\n' ' ')
     echo "===> Found: ${packages}"
     SPK_TO_BUILD+=${packages}
 done
 
 # fix for packages with different names
 if [ "$(echo ${SPK_TO_BUILD} | grep -ow nzbdrone)" != "" ]; then
-    SPK_TO_BUILD+=" sonarr3"
+    SPK_TO_BUILD=$(echo "${SPK_TO_BUILD}" | tr ' ' '\n' | grep -vw "nzbdrone" | tr '\n' ' ')" sonarr3"
 fi
 if [ "$(echo ${SPK_TO_BUILD} | grep -ow python)" != "" ]; then
-    SPK_TO_BUILD+=" python2"
+    SPK_TO_BUILD=$(echo "${SPK_TO_BUILD}" | tr ' ' '\n' | grep -vw "python" | tr '\n' ' ')" python2"
 fi
 
 # remove duplicate packages
 packages=$(printf %s "${SPK_TO_BUILD}" | tr ' ' '\n' | sort -u | tr '\n' ' ')
 
+# for ffmpeg v5-7 find all packages that depend on them
+for i in {5..7}; do
+    ffmpeg_dependent_packages=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile" -exec grep -Ho "FFMPEG_PACKAGE = ffmpeg${i}" {} \; | grep -Po ".*spk/\K[^/]*" | sort | tr '\n' ' ')
 
-# find all packages that depend on spk/ffmpeg is built before.
-all_ffmpeg_packages=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile" -exec grep -Ho "export FFMPEG_DIR" {} \; | grep -Po ".*spk/\K[^/]*" | sort | tr '\n' ' ')
+    # If packages contain a package that depends on ffmpeg (or is ffmpeg),
+    # then ensure relevant ffmpeg spk is first in list
+    for package in ${packages}
+    do
+        if [ "$(echo ffmpeg${i} ${ffmpeg_dependent_packages} | grep -ow ${package})" != "" ]; then
+            packages_without_ffmpeg=$(echo "${packages}" | tr ' ' '\n' | grep -v "ffmpeg${i}" | tr '\n' ' ')
+            packages="ffmpeg${i} ${packages_without_ffmpeg}"
+            break
+        fi
+    done
+done
 
-# if ffmpeg or one of its dependents is to build, ensure
-# ffmpeg is first package in the list of packages to build.
+# for synocli-videodriver that ffmpeg v5-7 depends on
+videodrv_dependent_packages=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile" -exec grep -Ho "spksrc.videodriver.mk" {} \; | grep -Po ".*spk/\K[^/]*" | sort | tr '\n' ' ')
+
+# If packages contain a package that depends on spksrc.videodriver.mk,
+# then ensure synocli-videodriver spk is first in list
 for package in ${packages}
 do
-    if [ "$(echo ffmpeg ${all_ffmpeg_packages} | grep -ow ${package})" != "" ]; then
-        packages_without_ffmpeg=$(echo "${packages}" | tr ' ' '\n' | grep -v "ffmpeg" | tr '\n' ' ')
-        packages="ffmpeg ${packages_without_ffmpeg}"
-        break;
+    if [ "$(echo synocli-videodriver ${videodrv_dependent_packages} | grep -ow ${package})" != "" ]; then
+        packages_without_videodrv=$(echo "${packages}" | tr ' ' '\n' | grep -v "synocli-videodriver" | tr '\n' ' ')
+        packages="synocli-videodriver ${packages_without_videodrv}"
+        break
     fi
 done
 
+# for python (310, 311) find all packages that depend on them
+for py in python310 python311; do
+    python_dependent_packages=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile" -exec grep -Ho "PYTHON_PACKAGE = ${py}" {} \; | grep -Po ".*spk/\K[^/]*" | sort | tr '\n' ' ')
+
+    # If packages contain a package that depends on python (or is python), then ensure
+    # relevant python spk is first in list
+    for package in ${packages}
+    do
+        if [ "$(echo ${py} ${python_dependent_packages} | grep -ow ${package})" != "" ]; then
+            packages_without_python=$(echo "${packages}" | tr ' ' '\n' | grep -v "${py}" | tr '\n' ' ')
+            packages="${py} ${packages_without_python}"
+            break
+        fi
+    done
+done
 
 # find all noarch packages
 all_noarch=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile" -exec grep -Ho "override ARCH" {} \; | grep -Po ".*spk/\K[^/]*" | sort | tr '\n' ' ')
@@ -76,19 +109,43 @@ all_noarch=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile" -exec grep -Ho "
 # and filter out packages that are removed or do not exist (e.g. nzbdrone)
 arch_packages=
 noarch_packages=
+has_arch_packages='false'
+has_noarch_packages='false'
 for package in ${packages}
 do
     if [ -f "./spk/${package}/Makefile" ]; then
         if [ "$(echo ${all_noarch} | grep -ow ${package})" = "" ]; then
             arch_packages+="${package} "
+            has_arch_packages='true'
         else
             noarch_packages+="${package} "
+            has_noarch_packages='true'
         fi
     fi
 done
 
+# evaluate packages that require DSM 7.2
+min_dsm72_packages=
+has_min_dsm72_packages='false'
+for package in ${packages}
+do
+    if [ -f "./spk/${package}/Makefile" ]; then
+        if [ "$(grep REQUIRED_MIN_DSM ./spk/${package}/Makefile | cut -d= -f2 | xargs)" = "7.2" ]; then
+            min_dsm72_packages+="${package} "
+            has_min_dsm72_packages='true'
+        fi
+    fi
+done
+
+if [ "${has_min_dsm72_packages}" = "true" ]; then
+    echo "===> Min DSM 7.2 packages found: ${min_dsm72_packages}"
+fi
+
 echo "arch_packages=${arch_packages}" >> $GITHUB_OUTPUT
 echo "noarch_packages=${noarch_packages}" >> $GITHUB_OUTPUT
+echo "has_arch_packages=${has_arch_packages}" >> $GITHUB_OUTPUT
+echo "has_noarch_packages=${has_noarch_packages}" >> $GITHUB_OUTPUT
+echo "has_min_dsm72_packages=${has_min_dsm72_packages}" >> $GITHUB_OUTPUT
 
 echo "::endgroup::"
 
@@ -101,10 +158,6 @@ else
     for package in ${packages}
     do
         DOWNLOAD_LIST+=$(echo "${DEPENDENCY_LIST}" | grep "^${package}:" | grep -o ":.*" | tr ':' ' ' | sort -u | tr '\n' ' ')
-        for version in ${DEFAULT_TC}
-        do
-           DOWNLOAD_LIST+=$(make -C spk/${package} TCVERSION=${version} dependency-kernel-list | grep "^${package}:" | grep -o ":.*" | tr ':' ' ' | sort -u | tr '\n' ' ')
-        done
     done
     # remove duplicate downloads
     downloads=$(printf %s "${DOWNLOAD_LIST}" | tr ' ' '\n' | sort -u | tr '\n' ' ')