python + django

Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · Oct 24, 2023 · 7077bd7 · 7077bd7
1 parent 2828340
commit 7077bd7
Show file tree

Hide file tree

Showing 7 changed files with 90 additions and 11 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -29,6 +29,10 @@ jobs:
         with:
           repository: 'HooliCorp/DjanGoat'
           path: 'repotests/DjanGoat'
+      - uses: actions/checkout@v3
+        with:
+          repository: 'DefectDojo/django-DefectDojo'
+          path: 'repotests/django-DefectDojo'
       - uses: coursier/cache-action@v6
       - name: Set up JDK
         uses: actions/setup-java@v3
@@ -41,6 +45,7 @@ jobs:
           ./atom.sh -o /tmp/juice.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml
           ./atom.sh -o /tmp/ts.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml
           ./atom.sh -o /tmp/py.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml
+          ./atom.sh -o /tmp/py2.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml
           ./atom.sh -o /tmp/c.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml
 
           ./atom.sh data-flow -o /tmp/java2.atom -l java $GITHUB_WORKSPACE/repotests/shiftleft-java-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/java.slices.json
@@ -54,7 +59,14 @@ jobs:
           ./atom.sh usages -o /tmp/juice3.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/juice.usages.json
           ./atom.sh usages -o /tmp/ts3.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/ts.usages.json
           ./atom.sh usages -o /tmp/py3.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py.usages.json
+          ./atom.sh usages -o /tmp/py4.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py4.usages.json
           ./atom.sh usages -o /tmp/c3.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/c.usages.json
           ls -lh /tmp/*.atom /tmp/*.json
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: |
+          npm install -g @cyclonedx/cdxgen --omit=optional
+          cdxgen -t python --deep -o $GITHUB_WORKSPACE/repotests/django-DefectDojo/bom.json $GITHUB_WORKSPACE/repotests/django-DefectDojo
+          ./atom.sh reachables -o /tmp/django-DefectDojo.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/django-DefectDojo.reachables.json
+        env:
+          JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -40,6 +40,10 @@ jobs:
         with:
           repository: 'HooliCorp/DjanGoat'
           path: 'repotests/DjanGoat'
+      - uses: actions/checkout@v3
+        with:
+          repository: 'DefectDojo/django-DefectDojo'
+          path: 'repotests/django-DefectDojo'
       - uses: coursier/cache-action@v6
       - name: Set up JDK
         uses: actions/setup-java@v3
@@ -57,6 +61,7 @@ jobs:
           ./atom.sh -o /tmp/ts.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml
           ./atom.sh -o /tmp/py.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml
           ./atom.sh parsedeps -o /tmp/py.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml
+          ./atom.sh -o /tmp/py2.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml
           ./atom.sh -o /tmp/c.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml
 
           ./atom.sh data-flow -o /tmp/java2.atom -l java $GITHUB_WORKSPACE/repotests/shiftleft-java-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/java.slices.json
@@ -69,6 +74,7 @@ jobs:
           # ./atom.sh usages -o /tmp/juice3.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/juice.usages.json
           ./atom.sh usages -o /tmp/ts3.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/ts.usages.json
           ./atom.sh usages -o /tmp/py3.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py.usages.json
+          ./atom.sh usages -o /tmp/py4.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py4.usages.json
           ./atom.sh usages -o /tmp/c3.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/c.usages.json
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -77,6 +83,8 @@ jobs:
           npm install -g @cyclonedx/cdxgen --omit=optional
           cdxgen -t java --deep -o $GITHUB_WORKSPACE/repotests/java-sec-code/bom.json $GITHUB_WORKSPACE/repotests/java-sec-code
           ./atom.sh reachables -o /tmp/java-sec-code.atom -l java $GITHUB_WORKSPACE/repotests/java-sec-code -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/java-sec-code.reachables.json
+          cdxgen -t python --deep -o $GITHUB_WORKSPACE/repotests/django-DefectDojo/bom.json $GITHUB_WORKSPACE/repotests/django-DefectDojo
+          ./atom.sh reachables -o /tmp/django-DefectDojo.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/django-DefectDojo.reachables.json
         env:
           JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
       - run: |

diff --git a/build.sbt b/build.sbt
@@ -3,7 +3,7 @@ ThisBuild / organization := "io.appthreat"
 ThisBuild / version      := "1.5.3"
 ThisBuild / scalaVersion := "3.3.1"
 
-val chenVersion      = "0.0.21"
+val chenVersion      = "0.5.1"
 
 lazy val atom = Projects.atom
 

diff --git a/lib/README.md b/lib/README.md
@@ -1,3 +1,3 @@
 org.eclipse.cdt jars were downloaded from
 
-https://download.eclipse.org/tools/cdt/releases/11.2/cdt-11.2.0/plugins/
+https://download.eclipse.org/tools/cdt/releases/11.3/cdt-11.3.1/plugins/
diff --git a/...g.eclipse.cdt.core_8.2.0.202305101618.jar → ...g.eclipse.cdt.core_8.3.1.202309150117.jar b/...g.eclipse.cdt.core_8.2.0.202305101618.jar → ...g.eclipse.cdt.core_8.3.1.202309150117.jar
diff --git a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala
@@ -36,21 +36,23 @@ object ReachableSlicing {
         .toList
     flowsList ++=
       atom.tag.name(API_TAG).parameter.reachableByFlows(atom.tag.name(API_TAG).parameter).map(toSlice).toList
-    // For JavaScript, we need flows between arguments of call nodes to track callbacks and middlewares
-    if (language == Languages.JSSRC || language == Languages.JAVASCRIPT) {
-      def jsCallSource          = atom.tag.name(config.sourceTag).call.argument.isIdentifier
-      def jsFrameworkIdentifier = atom.tag.name(FRAMEWORK_TAG).identifier
-      def jsFrameworkParameter  = atom.tag.name(FRAMEWORK_TAG).parameter
-      def jsSink                = atom.tag.name(config.sinkTag).call.argument.isIdentifier
-      flowsList ++= jsSink
-        .reachableByFlows(jsCallSource, jsFrameworkIdentifier, jsFrameworkParameter)
+    // For JavaScript and Python, we need flows between arguments of call nodes to track callbacks and middlewares
+    if (
+      language == Languages.JSSRC || language == Languages.JAVASCRIPT || language == Languages.PYTHON || language == Languages.PYTHONSRC
+    ) {
+      def dynCallSource          = atom.tag.name(config.sourceTag).call.argument.isIdentifier
+      def dynFrameworkIdentifier = atom.tag.name(FRAMEWORK_TAG).identifier
+      def dynFrameworkParameter  = atom.tag.name(FRAMEWORK_TAG).parameter
+      def dynSink                = atom.tag.name(config.sinkTag).call.argument.isIdentifier
+      flowsList ++= dynSink
+        .reachableByFlows(dynCallSource, dynFrameworkIdentifier, dynFrameworkParameter)
         .map(toSlice)
         .toList
       flowsList ++= atom.tag
         .name(FRAMEWORK_TAG)
         .call
         .argument
-        .reachableByFlows(jsFrameworkParameter)
+        .reachableByFlows(dynFrameworkParameter)
         .map(toSlice)
         .toList
     }

diff --git a/src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala
@@ -4,6 +4,7 @@ import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{Languages, Operators, PropertyNames}
 import io.shiftleft.semanticcpg.language.*
+import overflowdb.PropertyKey
 
 import java.util.concurrent.*
 import java.util.concurrent.atomic.AtomicBoolean
@@ -44,6 +45,8 @@ object UsageSlicing {
     val userDefTypes = userDefinedTypes(atom)
     if (language.get == Languages.NEWC || language.get == Languages.C)
       ProgramUsageSlice(slices ++ importsAsSlices(atom), userDefTypes)
+    else if (language.get == Languages.PYTHON || language.get == Languages.PYTHONSRC)
+      ProgramUsageSlice(slices, userDefTypes ++ routesAsUDT(atom))
     else
       ProgramUsageSlice(slices, userDefTypes)
   }
@@ -101,6 +104,60 @@ object UsageSlicing {
     })
   }
 
+  /** Discovers internally defined routes.
+    *
+    * @param atom
+    *   the CPG to query for types.
+    * @return
+    *   a list of user defined types.
+    */
+  def routesAsUDT(atom: Cpg): List[UserDefinedType] = {
+
+    def generateUDT(call: Call): UserDefinedType = {
+      UserDefinedType(
+        call.name,
+        call.argument
+          .order(1)
+          .isLiteral
+          .map(m =>
+            LocalDef(
+              name = m.code,
+              typeFullName = m.typeFullName,
+              lineNumber = Option(m.property(new PropertyKey[Integer](PropertyNames.LINE_NUMBER))).map(_.toInt),
+              columnNumber = Option(m.property(new PropertyKey[Integer](PropertyNames.COLUMN_NUMBER))).map(_.toInt)
+            )
+          )
+          .collectAll[LocalDef]
+          .l,
+        call
+          .callee(NoResolve)
+          .method
+          .filterNot(m => m.name.startsWith("<clinit>"))
+          .map(m =>
+            ObservedCall(
+              m.name,
+              Option(m.fullName),
+              m.parameter.map(_.typeFullName).toList,
+              m.methodReturn.typeFullName,
+              Option(m.isExternal),
+              m.lineNumber.map(_.intValue()),
+              m.columnNumber.map(_.intValue())
+            )
+          )
+          .l,
+        call.location.filename,
+        call.lineNumber.map(_.intValue()),
+        call.columnNumber.map(_.intValue())
+      )
+    }
+
+    atom.call
+      .where(_.methodFullName("django/urls.py:<module>.(path|re_path)"))
+      .map(generateUDT)
+      .filter(udt => udt.fields.nonEmpty || udt.procedures.nonEmpty)
+      .l
+  }
+
   private def TimedGet(dsf: Future[Option[(Method, ObjectUsageSlice)]]) = {
     try {
       dsf.get(5, TimeUnit.SECONDS)