Merge pull request #165 from fhem/fix-cpanfile-notAMD64

Fix dependency install from cpanfile if not a amd64 build
fhem · Mar 14, 2024 · e636e92 · e636e92
2 parents 2cf2df2 + 7dd459f
commit e636e92
Show file tree

Hide file tree

Showing 6 changed files with 142 additions and 88 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -38,8 +38,7 @@ jobs:
           perl-version: "5.38"
           install-modules-with: cpanm
           install-modules-args: --notest
-          install-modules: PPI Perl::PrereqScanner::NotQuiteLite File::Find::Rule List::MoreUtils CPAN::Meta Module::CPANfile CPAN::Meta::Merge Scalar::Util
-
+          install-modules: PPI Perl::PrereqScanner::NotQuiteLite File::Path File::Find::Rule List::MoreUtils CPAN::Meta Module::CPANfile CPAN::Meta::Merge Scalar::Util
 
       - name: clone 3rdparty repositories at github
         run: | 
@@ -48,6 +47,15 @@ jobs:
           cd ./3rdparty
           printf "%s\n" "${REPO_URLS[@]}" | xargs -I {} -P3 sh -c 'echo "{}: $(basename $(dirname {}))/$(basename {})"; git clone "{}" "$(basename $(dirname {}))/$(basename {})"; '
 
+      - name: Init PPI Cache 
+        uses: actions/cache@v4
+        with:
+          path: .cache/PPI
+          save-always: true
+          restore-keys: |
+            PPI-SVN-
+          key: PPI-SVN-${{ steps.prepareSVN.outputs.FHEM_REVISION_LATEST }}
+
       - name: "create private modules filter regex"    
         run: |
           echo "FHEM_MODULES=$(perl scripts/get-Packages.pl ./3rdparty ./src/fhem/trunk)"  >> $GITHUB_ENV
@@ -109,6 +117,67 @@ jobs:
           cache-to: type=gha,mode=max,scope=base_linux/${{ matrix.platform }}-${{ matrix.dockerfile }}
           tags: baseonly
 
+  base_cpan_build:
+    needs: [get_dependencies, base_build]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        dockerfile: [-bullseye, -threaded-bullseye]
+        platform: [arm/v7, arm64, 386]
+    steps:
+      - name: Checkout this repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/[email protected]
+
+      - name: Get git vars
+        shell: bash
+        run: |
+          echo "IMAGE_VERSION=$( git describe --tags --dirty --match "v[0-9]*")" >> $GITHUB_OUTPUT
+        id: gitVars
+
+      - name: Prepare SVN repository checkout and variables
+        id: prepareSVN
+        uses: ./.github/workflows/prepare-svn
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: cpanfile-FHEM
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: cpanfile-3rdParty
+          path: 3rdParty
+
+      - name: Prepare docker for build and publish
+        id: prepareDOCKER
+        uses: ./.github/workflows/prepare-docker
+        with:
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+          GHCR_OWNER: ${{ github.repository_owner }} 
+          GHCR_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}   
+          DOCKERFILE: ${{ matrix.dockerfile }}
+
+      - name: Build base cpan layer for ${{ matrix.platform }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true  
+          file: ./Dockerfile${{ matrix.dockerfile }}
+          platforms: linux/${{ matrix.platform }}
+          push: false
+          target: base-cpan
+          cache-from: | 
+            type=gha,scope=base-cpan_linux/${{ matrix.platform }}-${{ matrix.dockerfile }}
+            type=gha,scope=base_linux/${{ matrix.platform }}-${{ matrix.dockerfile }}
+          cache-to: type=gha,mode=max,scope=base-cpan_linux/${{ matrix.platform }}-${{ matrix.dockerfile }}
+          tags: basecpanonly
+
+
   test_build:
     # The type of runner that the job will run on
     needs: [get_dependencies, base_build]
@@ -243,7 +312,7 @@ jobs:
 
   published_build:
     runs-on: ubuntu-latest
-    needs: test_build
+    needs: [test_build, base_cpan_build]
     strategy:
       matrix:
         dockerfile: [-bullseye, -threaded-bullseye]
@@ -311,9 +380,8 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           target: with-fhem-python-nodejs
           cache-from: | 
-            type=gha,scope=base_linux/arm64-${{ matrix.dockerfile }}
-            type=gha,scope=base_linux/amd64-${{ matrix.dockerfile }}
-            type=gha,scope=base_linux/armv7-${{ matrix.dockerfile }}
+            type=gha,scope=base-cpan_linux/arm64-${{ matrix.dockerfile }}
+            type=gha,scope=base-cpan_linux/arm/v7-${{ matrix.dockerfile }}
             type=gha,scope=full_linux/amd64-${{ matrix.dockerfile }}
             type=gha,scope=full_linux/cross-${{ matrix.dockerfile }}
           cache-to: type=gha,mode=max,scope=full_linux/cross-${{ matrix.dockerfile }}
@@ -357,7 +425,7 @@ jobs:
           cache-from: | 
             type=gha,scope=base_linux/arm64-${{ matrix.dockerfile }}
             type=gha,scope=base_linux/amd64-${{ matrix.dockerfile }}
-            type=gha,scope=base_linux/armv7-${{ matrix.dockerfile }}
+            type=gha,scope=base_linux/arm/v7-${{ matrix.dockerfile }}
             type=gha,scope=full_linux/amd64-${{ matrix.dockerfile }}
             type=gha,scope=full_linux/cross-${{ matrix.dockerfile }}
             type=gha,scope=base_linux/cross-${{ matrix.dockerfile }}

diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,5 @@ fhem/cache/*
 fhem/log/*
 fhem/restoreDir/*
 .vscode/*
-src/FHEM/trunk/*
+src/FHEM/trunk/*
+.cache/*
diff --git a/Dockerfile-bullseye b/Dockerfile-bullseye
@@ -110,33 +110,32 @@ EOF
 
 FROM base as base-cpan
 # Install all CPAN Modules, needed from FHEM and standard modules
-COPY cpanfile /root/cpanfile
+COPY cpanfile /usr/src/app/core-cpanfile
 # Fixup modules which do not work on all platforms and install afterwards
 RUN <<EOF
   if [ "${TARGETPLATFORM}" != "linux/amd64" ] && [ "${TARGETPLATFORM}" != "linux/i386" ]; then 
-    sed -i '/Device::SerialPort/d' root/cpanfile  > /root/cpanfile 
-    sed -i '/Device::Firmata::Constants/d' root/cpanfile  >/root/cpanfile
+    sed -i '/Device::SerialPort/d' /usr/src/app/core-cpanfile
+    sed -i '/Device::Firmata::Constants/d' /usr/src/app/core-cpanfile  
   fi
-  cpanm --notest --cpanfile /root/cpanfile --installdeps  .
-  rm -rf /root/.cpanm 
-EOF
+  cpm install --without-test --cpanfile /usr/src/app/core-cpanfile --show-build-log-on-failure --global
+  cpm install --without-test --with-suggests --with-recommends --cpanfile /usr/src/app/core-cpanfile --show-build-log-on-failure --global
 
-RUN <<EOF
-  cpanm --notest --cpanfile /root/cpanfile --installdeps --with-suggests .
-  rm -rf /root/.cpanm 
+  rm -rf /root/.cpanm
+  rm -rf /root/.perl-cpm
 EOF
 
 # Install all CPAN Modules, needed from 3rd party module repositorys 
-COPY 3rdParty/cpanfile /root/cpanfile_3rdparty
+COPY 3rdParty/cpanfile /usr/src/app/3rdparty-cpanfile
 # Fixup modules which do not work on all platforms and install afterwards
 RUN <<EOF
   if [ "${TARGETPLATFORM}" != "linux/amd64" ] && [ "${TARGETPLATFORM}" != "linux/i386" ]; then 
-    sed -i '/Device::SerialPort/d' /root/cpanfile_3rdparty  > /root/cpanfile_3rdparty
-    sed -i '/Device::Firmata::Constants/d' /root/cpanfile_3rdparty  > /root/cpanfile_3rdparty
+    sed -i '/Device::SerialPort/d' /usr/src/app/3rdparty-cpanfile
+    sed -i '/Device::Firmata::Constants/d' /usr/src/app/3rdparty-cpanfile
   fi
 
-  cpanm --notest --cpanfile /root/cpanfile_3rdparty --installdeps --with-suggests .
+  cpm install --cpanfile /usr/src/app/3rdparty-cpanfile --without-test --with-recommends --with-suggests --show-build-log-on-failure --global
   rm -rf /root/.cpanm 
+  rm -rf /root/.perl-cpm
 EOF
 
 
@@ -243,7 +242,7 @@ FROM with-fhem as with-fhem-python
 
 RUN <<EOF 
     LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get update 
-    LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends 
+    LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends \
       python3 \
       python3-dev \
       python3-pip \

diff --git a/Dockerfile-threaded-bullseye b/Dockerfile-threaded-bullseye
@@ -110,33 +110,33 @@ EOF
 
 FROM base as base-cpan
 # Install all CPAN Modules, needed from FHEM and standard modules
-COPY cpanfile /root/cpanfile
+COPY cpanfile /usr/src/app/core-cpanfile
 # Fixup modules which do not work on all platforms and install afterwards
 RUN <<EOF
   if [ "${TARGETPLATFORM}" != "linux/amd64" ] && [ "${TARGETPLATFORM}" != "linux/i386" ]; then 
-    sed -i '/Device::SerialPort/d' root/cpanfile  > /root/cpanfile 
-    sed -i '/Device::Firmata::Constants/d' root/cpanfile  >/root/cpanfile
+    sed -i '/Device::SerialPort/d' /usr/src/app/core-cpanfile
+    sed -i '/Device::Firmata::Constants/d' /usr/src/app/core-cpanfile  
   fi
-  cpanm --notest --cpanfile /root/cpanfile --installdeps  .
-  rm -rf /root/.cpanm 
-EOF
+  cpm install --without-test --cpanfile /usr/src/app/core-cpanfile --show-build-log-on-failure --global
+  cpm install --without-test --with-suggests --with-recommends --cpanfile /usr/src/app/core-cpanfile --show-build-log-on-failure --global
 
-RUN <<EOF
-  cpanm --notest --cpanfile /root/cpanfile --installdeps --with-suggests .
   rm -rf /root/.cpanm 
+  rm -rf /root/.perl-cpm/
 EOF
 
+
 # Install all CPAN Modules, needed from 3rd party module repositorys 
-COPY 3rdParty/cpanfile /root/cpanfile_3rdparty
+COPY 3rdParty/cpanfile /usr/src/app/3rdparty-cpanfile
 # Fixup modules which do not work on all platforms and install afterwards
 RUN <<EOF
   if [ "${TARGETPLATFORM}" != "linux/amd64" ] && [ "${TARGETPLATFORM}" != "linux/i386" ]; then 
-    sed -i '/Device::SerialPort/d' /root/cpanfile_3rdparty  > /root/cpanfile_3rdparty
-    sed -i '/Device::Firmata::Constants/d' /root/cpanfile_3rdparty  > /root/cpanfile_3rdparty
+    sed -i '/Device::SerialPort/d' /usr/src/app/3rdparty-cpanfile
+    sed -i '/Device::Firmata::Constants/d' /usr/src/app/3rdparty-cpanfile
   fi
 
-  cpanm --notest --cpanfile /root/cpanfile_3rdparty --installdeps --with-suggests .
+  cpm install --cpanfile /usr/src/app/3rdparty-cpanfile --without-test --with-recommends --with-suggests --show-build-log-on-failure --global
   rm -rf /root/.cpanm 
+  rm -rf /root/.perl-cpm/
 EOF
 
 
@@ -243,7 +243,7 @@ FROM with-fhem as with-fhem-python
 
 RUN <<EOF 
     LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get update 
-    LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends 
+    LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends \
       python3 \
       python3-dev \
       python3-pip \

diff --git a/scripts/get-Packages.pl b/scripts/get-Packages.pl
@@ -5,7 +5,17 @@
 use File::Find::Rule;
 use List::MoreUtils qw(uniq);
 # use Data::Dumper;
+use File::Path qw(make_path);
 
+my $PPICacheDir;
+BEGIN {
+    $PPICacheDir = '.cache/PPI';
+    unless(-d $PPICacheDir) {
+        make_path($PPICacheDir) or die qq[Directory $PPICacheDir coldn't be created: $!];
+    }
+};
+
+use PPI::Cache path => $PPICacheDir;
 
 my @directories = @ARGV;
 
@@ -20,7 +30,7 @@
 
     foreach my $file (@files) {
 
-        my $document = PPI::Document->new($file);
+        my $document = PPI::Document->new($file, readonly => 1);
         next unless $document;
         # Alle package-Anweisungen in der Datei finden
         my $package_statements = $document->find('PPI::Statement::Package');

diff --git a/scripts/parse-METAJson.pl b/scripts/parse-METAJson.pl
@@ -3,7 +3,7 @@
 use warnings;
 use CPAN::Meta;
 use Module::CPANfile;
-# use Data::Dumper;
+use Data::Dumper;
 use CPAN::Meta::Merge;
 use Scalar::Util qw/blessed/;
 use File::Find::Rule;
@@ -18,7 +18,20 @@
 ## Load existing requirements from cpanfile
 my $cpanfile = Module::CPANfile->load('cpanfile');
 
-my $newCPANFile;
+sub merge_hashes {
+    my ($hash1, $hash2) = @_;
+    for my $key (keys %$hash2) {
+        if (ref $hash2->{$key} eq 'HASH' && ref $hash1->{$key} eq 'HASH') {
+            merge_hashes($hash1->{$key}, $hash2->{$key});
+        } else {
+            $hash1->{$key} = $hash2->{$key};
+        }
+    }
+    return $hash1;
+}
+
+
+#my $newCPANFile;
 # Alle Perl-Moduldateien im Verzeichnisbaum finden
 foreach my $directory (@directories) {
 
@@ -54,6 +67,7 @@
         }
 
         $jsonString = join '', @JSONlines;
+
 
         ## Script breaks here, because we may have no version field which is requred to pass here
 
@@ -66,67 +80,29 @@
                 next;
         };
 
-        #print Dumper $MetaHash;
+
 
         # fix missing version information
-        my @fixups = ();
-
-        if (!exists($MetaHash->{version}) )
-        {
-            push(@fixups, q[version] );
-
-            $MetaHash->{version} = "1";
-
-        }
-        if (!exists($MetaHash->{name}) )
-        {
-            push(@fixups, q[name] );
-            $MetaHash->{name} = $filename;
-        }
 
-        if (!exists($MetaHash->{'meta-spec'}) || !exists($MetaHash->{'meta-spec'}{'version'})  )
-        {
-            push(@fixups, q[meta-spec] );
-            $MetaHash->{'meta-spec'}{'version'} = 2;
-        }
-
-        if (scalar @fixups > 0)
-        {
-            print q[ fixups: ];
-            print join ", ", @fixups;
-            print q[ ];
-        }
-
-
-        my $moduleMeta;
-        eval {
-            $moduleMeta = CPAN::Meta->load_json_string(to_json($MetaHash));
-            1;
-        } or do {
-                print q[[ failed ]]. $@;
-                next;
-        };
-
-
-        # merge requirements
 
-        my $prereqs_hash = $cpanfile->prereqs->with_merged_prereqs($moduleMeta->effective_prereqs)->as_string_hash;
-        my $struct = { %{$moduleMeta->as_struct}, prereqs => $prereqs_hash };
+        my $cpanfile_requirements = $cpanfile->prereq_specs;            # requirements from our cpanfile
+        my $module_requirements = $MetaHash->{'prereqs'};               # requirements from the processed file
+        # print Dumper $cpanfile_requirements;                
+        # print Dumper $module_requirements;                
 
-        #print $moduleMeta->meta_spec->{version};
-
-        my $mergedMeta =  CPAN::Meta->new($struct);
-        $newCPANFile = Module::CPANfile->from_prereqs(  $mergedMeta->prereqs );
-
-        $cpanfile = $newCPANFile;
+        # merge requirements together
+        my $struct = merge_hashes($cpanfile_requirements, $module_requirements);
+        # print Dumper $struct;              
+
+        $cpanfile = Module::CPANfile->from_prereqs(  $struct );         # update cpanfile object
         print qq[$filename was processed successfull\n];
     }
 }
 
 # save our new cpanfile
-if (defined $newCPANFile)
+if (defined $cpanfile)
 {
-    $newCPANFile->save('cpanfile');
+    $cpanfile->save('cpanfile');
     print qq[\n\nResult: cpanfile was saved\n];
 }