HPCC-32168 Add Dynamic ECL-Bundles Listing Feature for Bundle Testing…

.github/workflows/bundleTest-thor.yml

@@ -17,7 +17,7 @@ on:
         type: string
         description: 'Dependencies'
         required: false
-        default: 'bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline pkg-config libtool autotools-dev automake git cmake xmlstarlet'
+        default: 'bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline libtool autotools-dev automake git cmake xmlstarlet'
       get-stat: 
         type: boolean
         description: 'Run Query stat'
@@ -37,11 +37,10 @@ on:
 env:
   ML_SUPPRESS_WARNING_FILES: "RegressionTestModified.ecl ClassificationTestModified.ecl"
   ML_EXCLUDE_FILES: "--ef ClassicTestModified.ecl,SVCTest.ecl,ClassificationTestModified.ecl"
-  BUNDLES_TO_TEST: "ML_Core PBblas GLM  GNN DBSCAN LearningTrees TextVectors KMeans SupportVectorMachines LinearRegression LogisticRegression"       
+  BUNDLES_TO_TEST: "ML_Core PBblas GLM GNN DBSCAN LearningTrees TextVectors KMeans SupportVectorMachines LinearRegression LogisticRegression"       
   uploadArtifact: false 
 
 jobs:
-
   main:
     runs-on: ${{ inputs.os }}
     steps:
@@ -78,6 +77,52 @@ jobs:
           sudo chown -R $USER:$USER /opt/HPCCSystems
           sudo xmlstarlet ed -L -u 'Environment/Software/ThorCluster/@slavesPerNode' -v 2 -u 'Environment/Software/ThorCluster/@channelsPerSlave' -v 1 /etc/HPCCSystems/environment.xml
 
+      - name: Checkout ecl-bundles repository
+        uses: actions/checkout@v4
+        with:
+          repository: hpcc-systems/ecl-bundles
+
+      - name: Scrap Bundles List
+        shell: "bash"
+        run: | 
+          if [[ -f README.rst ]] 
+          then 
+              # append all table information into TABLES_DATA from README.rst 
+              # tables are created using '|' (pipe character) in reStructuredText(.rst) files. So we are extracting all the lines that start with '|'(pipe character).
+              # along with tables, we also extract '============='. This is used to represent headings in .rst files. This helps us to differentiate between different sections/tables of the README file.
+              TABLES_DATA=$( cat README.rst | grep -oE -e "\|[^\|]*\|"  -e "==.*==" | sed 's/|//g' )
+              IFS=$'\n'
+              HEADER_NUM_OF_ECL_BUNDLES=2
+              HEADER_COUNT=0
+              for LINE in $TABLES_DATA
+              do
+                LINE=${LINE# }   #removing space from begining of the line
+                LINE=${LINE%% [^A-Za-z0-9]*} #remove trailing spaces.
+                if [[ ${LINE:0:1} == "=" ]]; then 
+                  HEADER_COUNT=$(( HEADER_COUNT + 1 ))
+                  continue
+                fi
+                if [[ $HEADER_COUNT -eq $HEADER_NUM_OF_ECL_BUNDLES ]]; then 
+                  if [[ ${LINE:0:4} == "http" ]]; then
+                    echo -e "Bundle Repo : ${LINE}\n" 
+                    BUNDLE=$( basename $LINE )
+                    BUNDLES_TO_TEST+=" ${BUNDLE/.git}" 
+                  else
+                    echo "Bundle Name : ${LINE}" 
+                  fi
+                elif [[ $HEADER_COUNT -eq $(( HEADER_NUM_OF_ECL_BUNDLES + 1 )) ]]; then 
+                  break
+                fi
+              done
+              BUNDLES_TO_TEST=$(echo $BUNDLES_TO_TEST | sed 's/ /\n/g' | sort -bf -u )
+              #                ( print bundles list   | replace all spaces with new-line | sort the list and select unique items )
+              BUNDLES_TO_TEST=${BUNDLES_TO_TEST//$'\n'/ } # replace all newline characters with spaces
+              echo "BUNDLES TO TEST : $BUNDLES_TO_TEST"
+              echo "BUNDLES_TO_TEST=$BUNDLES_TO_TEST" >> $GITHUB_ENV
+          else 
+              echo "README.rst file not found! in HPCC-Systems/ecl-bundles/ repository" 
+          fi
+
       - name: Install ML Dependencies
         shell: "bash"
         run: |  
@@ -97,16 +142,17 @@ jobs:
         if: ${{ inputs.test-core-file-generation }}
         shell: "bash"
         run: |
-            echo """
-            boolean seg() := beginc++ #option action
-            #include <csignal>
-            #body
-            raise(SIGABRT);
-            return false;
-            endc++;
-            output(seg()); """ > crash.ecl
-
-            ecl run -t hthor crash.ecl
+          cat > crash.ecl << EOF
+          boolean seg() := beginc++ #option action
+          #include <csignal>
+          #body
+          raise(SIGABRT);
+          return false;
+          endc++;
+          output(seg()); 
+          EOF
+
+          ecl run -t hthor crash.ecl
         continue-on-error: true
 
       - name: Get test from Github