Ruby support (#170)

* Ruby frontend Signed-off-by: Prabhu Subramanian <[email protected]> * Tag rails routes Signed-off-by: Prabhu Subramanian <[email protected]> * Tag sinatra routes Signed-off-by: Prabhu Subramanian <[email protected]> * Update typescript Signed-off-by: Prabhu Subramanian <[email protected]> * Use Ruby 3.4.1 in CI Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · Jan 10, 2025 · 87d6665 · 87d6665
1 parent a1845e1
commit 87d6665
Show file tree

Hide file tree

Showing 25 changed files with 207 additions and 40 deletions.
diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml
@@ -38,6 +38,10 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: '22.x'
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4.1'
+          bundler-cache: false
       - name: Delete `.rustup` directory
         run: rm -rf /home/runner/.rustup # to save disk space
         if: runner.os == 'Linux'

diff --git a/.github/workflows/nodejstests.yml b/.github/workflows/nodejstests.yml
@@ -20,6 +20,10 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4.1'
+          bundler-cache: false
       - uses: actions/checkout@v4
         with:
           repository: 'ShiftLeftSecurity/shiftleft-java-example'

diff --git a/.github/workflows/npm-release.yml b/.github/workflows/npm-release.yml
@@ -22,6 +22,10 @@ jobs:
       with:
         node-version: '23.x'
         registry-url: https://registry.npmjs.org/
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.4.1'
+        bundler-cache: false
     - uses: coursier/cache-action@v6
     - name: Set up JDK
       uses: actions/setup-java@v4

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -13,6 +13,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4.1'
+          bundler-cache: false
       - uses: actions/checkout@v4
         with:
           repository: 'ShiftLeftSecurity/shiftleft-java-example'

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -20,6 +20,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4.1'
+          bundler-cache: false
       - name: Set up JDK
         uses: graalvm/setup-graalvm@v1
         with:

diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -81,6 +81,10 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4.1'
+          bundler-cache: false
       - name: Install sbt
         run: brew install sbt
         if: runner.os == 'macOS'

diff --git a/README.md b/README.md
@@ -191,6 +191,7 @@ atom -o app.atom -l java --export-atom --export-dir <export dir> --with-data-dep
 - TypeScript
 - Python
 - PHP (Requires PHP >= 7.0. Supports PHP 5.2 to 8.3)
+- Ruby (Requires Ruby 3.4.0. Supports Ruby 1.8 - 3.4.0 syntax)
 
 ## Atom Specification
 

diff --git a/build.sbt b/build.sbt
@@ -1,9 +1,9 @@
 name                     := "atom"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "2.0.25"
-ThisBuild / scalaVersion := "3.5.2"
+ThisBuild / version      := "2.1.0"
+ThisBuild / scalaVersion := "3.6.2"
 
-val chenVersion = "2.2.3"
+val chenVersion = "2.3.0"
 
 lazy val atom = Projects.atom
 
@@ -23,6 +23,7 @@ libraryDependencies ++= Seq(
   "io.appthreat"  %% "jssrc2cpg"         % Versions.chen,
   "io.appthreat"  %% "jimple2cpg"        % Versions.chen,
   "io.appthreat"  %% "php2atom"          % Versions.chen,
+  "io.appthreat"  %% "ruby2atom"         % Versions.chen,
   ("io.appthreat" %% "semanticcpg"       % Versions.chen % Test).classifier("tests"),
   ("io.appthreat" %% "x2cpg"             % Versions.chen % Test).classifier("tests"),
   ("io.appthreat" %% "pysrc2cpg"         % Versions.chen % Test).classifier("tests"),
@@ -106,7 +107,6 @@ Compile / packageDoc / publishArtifact := false
 wartremoverWarnings ++= Seq(
   Wart.NoNeedImport,
   Wart.ArrayEquals,
-  Wart.Any,
   Wart.FinalCaseClass,
   Wart.FinalVal,
   Wart.ToString,

diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -4,14 +4,15 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/atom" \
       org.opencontainers.image.url="https://github.com/appthreat/atom" \
-      org.opencontainers.image.version="2.0.x" \
+      org.opencontainers.image.version="2.1.x" \
       org.opencontainers.image.vendor="appthreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="atom" \
       org.opencontainers.image.description="Container image for AppThreat atom" \
       org.opencontainers.docker.cmd="docker run --rm -v /tmp:/tmp -v $HOME:$HOME -v $(pwd):/app:rw -it ghcr.io/appthreat/atom atom -o /app/app.atom -l java /app"
 
 ARG MAVEN_VERSION=3.9.9
+ARG RUBY_VERSION=3.4.0
 
 ENV MAVEN_VERSION=$MAVEN_VERSION \
     MAVEN_HOME="/opt/maven/${MAVEN_VERSION}" \
@@ -21,9 +22,15 @@ ENV MAVEN_VERSION=$MAVEN_VERSION \
     PYTHONUNBUFFERED=1 \
     PYTHONIOENCODING="utf-8" \
     CDXGEN_NO_BANNER=true \
-    COMPOSER_ALLOW_SUPERUSER=1
+    COMPOSER_ALLOW_SUPERUSER=1 \
+    RUBY_VERSION=$RUBY_VERSION \
+    MALLOC_CONF="dirty_decay_ms:2000,narenas:2,background_thread:true" \
+    RUBY_CONFIGURE_OPTS="--with-jemalloc --enable-yjit" \
+    RUBYOPT="--yjit" \
+    RUBY_BUILD_BUILD_PATH="/tmp/rbenv" \
+    RUBY_BUILD_HTTP_CLIENT=curl
 
-ENV PATH=/opt/bin:/opt/vendor/bin:${PATH}:${MAVEN_HOME}/bin:/usr/local/bin/:/root/.local/bin:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:
+ENV PATH=/opt/bin:/opt/vendor/bin:${PATH}:${MAVEN_HOME}/bin:/usr/local/bin/:/root/.local/bin:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:/root/.rbenv/bin:
 
 WORKDIR /opt
 
@@ -43,15 +50,29 @@ RUN set -e; \
     && microdnf install -y gcc git-core php php-cli php-curl php-zip php-bcmath php-json php-pear php-mbstring php-devel make \
         python3.12 python3.12-devel python3.12-pip \
         wget bash glibc-common glibc-all-langpacks java-21-openjdk-headless \
-        pcre2 findutils which tar gzip zip unzip sudo nodejs \
+        openssl-devel libffi-devel readline-devel libyaml zlib-devel ncurses ncurses-devel \
+        pcre2 findutils which tar gzip zip unzip sudo nodejs rust \
+    && microdnf install -y epel-release \
+    && microdnf install --enablerepo=crb -y libyaml-devel jemalloc-devel \
+    && git clone https://github.com/rbenv/rbenv.git --depth=1 ~/.rbenv \
+    && echo 'export PATH="/root/.rbenv/bin:$PATH"' >> ~/.bashrc \
+    && echo 'eval "$(~/.rbenv/bin/rbenv init - bash)"' >> ~/.bashrc \
+    && source ~/.bashrc \
+    && mkdir -p "$(rbenv root)/plugins" \
+    && git clone https://github.com/rbenv/ruby-build.git --depth=1 "$(rbenv root)/plugins/ruby-build" \
+    && rbenv install ${RUBY_VERSION} \
+    && rbenv global ${RUBY_VERSION} \
+    && ruby --version \
+    && which ruby \
+    && rm -rf /root/.rbenv/cache $RUBY_BUILD_BUILD_PATH \
     && alternatives --install /usr/bin/python3 python /usr/bin/python3.12 10 \
     && alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 10 \
     && python3 --version \
     && node --version \
     && python3 -m pip install --upgrade pip poetry atom-tools \
     && curl -s "https://get.sdkman.io" | bash \
-    && source "$HOME/.sdkman/bin/sdkman-init.sh" \
-    && echo -e "sdkman_auto_answer=true\nsdkman_selfupdate_feature=false\nsdkman_auto_env=true\nsdkman_curl_connect_timeout=20\nsdkman_curl_max_time=0" >> $HOME/.sdkman/etc/config \
+    && source "/root/.sdkman/bin/sdkman-init.sh" \
+    && echo -e "sdkman_auto_answer=true\nsdkman_selfupdate_feature=false\nsdkman_auto_env=true\nsdkman_curl_connect_timeout=20\nsdkman_curl_max_time=0" >> /root/.sdkman/etc/config \
     && sdk install maven $MAVEN_VERSION \
     && sdk offline enable \
     && mv /root/.sdkman/candidates/* /opt/ \
@@ -65,7 +86,7 @@ RUN set -e; \
     && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'platform-tools' --sdk_root=/opt/android-sdk-linux \
     && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'platforms;android-34' --sdk_root=/opt/android-sdk-linux \
     && /opt/android-sdk-linux/cmdline-tools/latest/bin/sdkmanager 'build-tools;34.0.0' --sdk_root=/opt/android-sdk-linux \
-    && sudo npm install -g @cyclonedx/cdxgen --omit=optional \
+    && npm install -g @cyclonedx/cdxgen --omit=optional \
     && php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && php composer-setup.php \
     && mv composer.phar /usr/local/bin/composer \
     && curl -LO https://raw.githubusercontent.com/AppThreat/chen/main/platform/frontends/php2atom/composer.lock \
@@ -79,11 +100,12 @@ COPY ./wrapper .
 RUN unzip -q atom.zip \
     && composer update --no-progress --prefer-dist --ignore-platform-reqs \
     && cd /opt/nodejs && npm install --only=production && cd /opt \
-    && sudo npm install -g /opt/nodejs \
+    && npm install -g /opt/nodejs \
     && rm -rf atom.zip composer.json composer.lock composer-setup.php /usr/local/bin/atom \
     && /opt/bin/atom --help \
     && which astgen \
     && which phpastgen \
+    && which rbastgen \
     && microdnf clean all
 
 CMD ["/opt/bin/atom"]
diff --git a/ci/Dockerfile.sle b/ci/Dockerfile.sle
@@ -4,7 +4,7 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/atom" \
       org.opencontainers.image.url="https://github.com/appthreat/atom" \
-      org.opencontainers.image.version="2.0.x" \
+      org.opencontainers.image.version="2.1.x" \
       org.opencontainers.image.vendor="appthreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="atom" \
@@ -37,7 +37,7 @@ RUN set -e; \
             ;; \
         *) echo >&2 "error: unsupported architecture: '$ARCH_NAME'"; exit 1 ;; \
     esac; \
-    zypper --non-interactive install -l --no-recommends php8 php8-cli php8-curl php8-zip php8-bcmath php8-pear php8-mbstring php8-devel \
+    zypper --non-interactive install -l --no-recommends php8 php8-cli php8-curl php8-zip php8-bcmath php8-pear php8-mbstring php8-devel ruby ruby-devel \
     && source /root/.nvm/nvm.sh \
     && python3 --version \
     && source /root/.nvm/nvm.sh \
@@ -72,6 +72,7 @@ RUN unzip -q atom.zip \
     && /opt/bin/atom --help \
     && which astgen \
     && which phpastgen \
+    && which rbastgen \
     && zypper clean -a
 
 CMD ["/opt/bin/atom"]
diff --git a/codemeta.json b/codemeta.json
@@ -7,7 +7,7 @@
   "downloadUrl": "https://github.com/AppThreat/atom",
   "issueTracker": "https://github.com/AppThreat/atom/issues",
   "name": "atom",
-  "version": "2.0.25",
+  "version": "2.1.0",
   "description": "Atom is a novel intermediate representation for next-generation code analysis.",
   "applicationCategory": "code-analysis",
   "keywords": [

diff --git a/docs/docs/languages.md b/docs/docs/languages.md
@@ -14,3 +14,4 @@ title: Languages supported
 - TypeScript
 - Python
 - PHP (Requires PHP >= 7.0. Supports PHP 5.2 to 8.3)
+- Ruby (Requires Ruby 3.4.0. Supports Ruby 1.8 - 3.4.0 syntax)
diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=1.10.5
+sbt.version=1.10.7
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1,4 +1,4 @@
 addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")
 addSbtPlugin("com.github.sbt"  % "sbt-native-packager"   % "1.10.4")
 addSbtPlugin("com.codecommit" % "sbt-github-packages" % "0.5.3")
-addSbtPlugin("org.wartremover" % "sbt-wartremover" % "3.2.4")
+addSbtPlugin("org.wartremover" % "sbt-wartremover" % "3.2.5")
diff --git a/src/main/scala/io/appthreat/atom/Atom.scala b/src/main/scala/io/appthreat/atom/Atom.scala
@@ -18,6 +18,7 @@ import io.appthreat.jssrc2cpg.passes.{
 import io.appthreat.jssrc2cpg.{JsSrc2Cpg, Config as JSConfig}
 import io.appthreat.php2atom.passes.PhpSetKnownTypesPass
 import io.appthreat.php2atom.{Php2Atom, Config as PhpConfig}
+import io.appthreat.ruby2atom.{Ruby2Atom, Config as RubyConfig}
 import io.appthreat.pysrc2cpg.{
     DynamicTypeHintFullNamePass,
     Py2CpgOnFileSystem,
@@ -208,7 +209,7 @@ object Atom:
         )
     cmd("usages")
         .text("Extract local variable and parameter usages")
-        .action((_, *) => AtomUsagesConfig().withRemoveAtom(true))
+        .action((_, *) => AtomUsagesConfig())
         .children(
           opt[Int]("min-num-calls")
               .text(s"the minimum number of calls required for a usage slice - defaults to 1.")
@@ -566,6 +567,15 @@ object Atom:
                 new PhpSetKnownTypesPass(ag).createAndApply()
                 ag
               }
+          case Languages.RUBYSRC | "RUBY" | "RB" | "JRUBY" =>
+              new Ruby2Atom().createCpgWithOverlays(
+                RubyConfig()
+                    .withInputPath(config.inputPath.pathAsString)
+                    .withOutputPath(outputAtomFile)
+                    .withIgnoredFilesRegex(".*(samples|examples|docs|tests).*")
+              ).map { ag =>
+                  ag
+              }
           case _ => Failure(
                 new RuntimeException(
                   s"No language frontend supported for language '$language'"

diff --git a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala
@@ -88,7 +88,7 @@ object ReachableSlicing:
           atom.tag.name(CLI_SOURCE_TAG).call
         ).map(toSlice).toList
     end if
-    if language == Languages.PHP
+    if language == Languages.PHP || language == Languages.RUBYSRC
     then
       flowsList ++= atom.ret.where(_.tag.name(config.sinkTag)).reachableByFlows(
         atom.tag.name(config.sourceTag).parameter

diff --git a/wrapper/nodejs/build.sh b/wrapper/nodejs/build.sh
@@ -13,6 +13,12 @@ php -r "unlink('composer-setup.php');"
 export COMPOSER_VENDOR_DIR=plugins
 php composer.phar require nikic/php-parser:4.18.0 --ignore-platform-reqs --optimize-autoloader
 
-npm install
+cd plugins/rubyastgen
+bash setup.sh
+cd ../..
+
+rm plugins/bin/racc plugins/bin/ruby-parse plugins/bin/ruby-rewrite
+
+npm ci
 
 rm composer.phar composer.json composer.lock
diff --git a/wrapper/nodejs/package-lock.json b/wrapper/nodejs/package-lock.json
diff --git a/wrapper/nodejs/package.json b/wrapper/nodejs/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@appthreat/atom",
-  "version": "2.0.25",
+  "version": "2.1.0",
   "description": "Create atom (⚛) representation for your application, packages and libraries",
   "exports": "./index.js",
   "type": "module",
@@ -9,8 +9,8 @@
     "lint": "eslint *.mjs *.js"
   },
   "dependencies": {
-    "@babel/parser": "^7.26.2",
-    "typescript": "^5.6.3",
+    "@babel/parser": "^7.26.3",
+    "typescript": "^5.7.2",
     "yargs": "^17.7.2"
   },
   "devDependencies": {
@@ -19,7 +19,8 @@
   "bin": {
     "atom": "index.js",
     "astgen": "astgen.js",
-    "phpastgen": "phpastgen.js"
+    "phpastgen": "phpastgen.js",
+    "rbastgen": "rbastgen.js"
   },
   "engines": {
     "node": ">=16.0.0"