diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8de2b36..4997a32 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -29,6 +29,10 @@ jobs: with: repository: 'HooliCorp/DjanGoat' path: 'repotests/DjanGoat' + - uses: actions/checkout@v3 + with: + repository: 'DefectDojo/django-DefectDojo' + path: 'repotests/django-DefectDojo' - uses: coursier/cache-action@v6 - name: Set up JDK uses: actions/setup-java@v3 @@ -41,6 +45,7 @@ jobs: ./atom.sh -o /tmp/juice.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml ./atom.sh -o /tmp/ts.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml ./atom.sh -o /tmp/py.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml + ./atom.sh -o /tmp/py2.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml ./atom.sh -o /tmp/c.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml ./atom.sh data-flow -o /tmp/java2.atom -l java $GITHUB_WORKSPACE/repotests/shiftleft-java-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/java.slices.json @@ -54,7 +59,14 @@ jobs: ./atom.sh usages -o /tmp/juice3.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/juice.usages.json ./atom.sh usages -o /tmp/ts3.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/ts.usages.json ./atom.sh usages -o /tmp/py3.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py.usages.json + ./atom.sh usages -o /tmp/py4.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py4.usages.json ./atom.sh usages -o /tmp/c3.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/c.usages.json ls -lh /tmp/*.atom /tmp/*.json env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: | + npm install -g @cyclonedx/cdxgen --omit=optional + cdxgen -t python --deep -o $GITHUB_WORKSPACE/repotests/django-DefectDojo/bom.json $GITHUB_WORKSPACE/repotests/django-DefectDojo + ./atom.sh reachables -o /tmp/django-DefectDojo.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/django-DefectDojo.reachables.json + env: + JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml index 7b15afd..733cbe5 100644 --- a/.github/workflows/repotests.yml +++ b/.github/workflows/repotests.yml @@ -40,6 +40,10 @@ jobs: with: repository: 'HooliCorp/DjanGoat' path: 'repotests/DjanGoat' + - uses: actions/checkout@v3 + with: + repository: 'DefectDojo/django-DefectDojo' + path: 'repotests/django-DefectDojo' - uses: coursier/cache-action@v6 - name: Set up JDK uses: actions/setup-java@v3 @@ -57,6 +61,7 @@ jobs: ./atom.sh -o /tmp/ts.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml ./atom.sh -o /tmp/py.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml ./atom.sh parsedeps -o /tmp/py.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml + ./atom.sh -o /tmp/py2.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml ./atom.sh -o /tmp/c.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml ./atom.sh data-flow -o /tmp/java2.atom -l java $GITHUB_WORKSPACE/repotests/shiftleft-java-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/java.slices.json @@ -69,6 +74,7 @@ jobs: # ./atom.sh usages -o /tmp/juice3.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/juice.usages.json ./atom.sh usages -o /tmp/ts3.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/ts.usages.json ./atom.sh usages -o /tmp/py3.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py.usages.json + ./atom.sh usages -o /tmp/py4.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py4.usages.json ./atom.sh usages -o /tmp/c3.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/c.usages.json env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -77,6 +83,8 @@ jobs: npm install -g @cyclonedx/cdxgen --omit=optional cdxgen -t java --deep -o $GITHUB_WORKSPACE/repotests/java-sec-code/bom.json $GITHUB_WORKSPACE/repotests/java-sec-code ./atom.sh reachables -o /tmp/java-sec-code.atom -l java $GITHUB_WORKSPACE/repotests/java-sec-code -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/java-sec-code.reachables.json + cdxgen -t python --deep -o $GITHUB_WORKSPACE/repotests/django-DefectDojo/bom.json $GITHUB_WORKSPACE/repotests/django-DefectDojo + ./atom.sh reachables -o /tmp/django-DefectDojo.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/django-DefectDojo.reachables.json env: JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" - run: | diff --git a/build.sbt b/build.sbt index 1755496..84f2c9e 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ ThisBuild / organization := "io.appthreat" ThisBuild / version := "1.5.3" ThisBuild / scalaVersion := "3.3.1" -val chenVersion = "0.0.21" +val chenVersion = "0.5.1" lazy val atom = Projects.atom diff --git a/lib/README.md b/lib/README.md index 64118d5..937016e 100644 --- a/lib/README.md +++ b/lib/README.md @@ -1,3 +1,3 @@ org.eclipse.cdt jars were downloaded from -https://download.eclipse.org/tools/cdt/releases/11.2/cdt-11.2.0/plugins/ +https://download.eclipse.org/tools/cdt/releases/11.3/cdt-11.3.1/plugins/ diff --git a/lib/org.eclipse.cdt.core_8.2.0.202305101618.jar b/lib/org.eclipse.cdt.core_8.3.1.202309150117.jar similarity index 75% rename from lib/org.eclipse.cdt.core_8.2.0.202305101618.jar rename to lib/org.eclipse.cdt.core_8.3.1.202309150117.jar index a706497..56103ac 100644 Binary files a/lib/org.eclipse.cdt.core_8.2.0.202305101618.jar and b/lib/org.eclipse.cdt.core_8.3.1.202309150117.jar differ diff --git a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala index 7b070f1..ed0c4ff 100644 --- a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala +++ b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala @@ -36,21 +36,23 @@ object ReachableSlicing { .toList flowsList ++= atom.tag.name(API_TAG).parameter.reachableByFlows(atom.tag.name(API_TAG).parameter).map(toSlice).toList - // For JavaScript, we need flows between arguments of call nodes to track callbacks and middlewares - if (language == Languages.JSSRC || language == Languages.JAVASCRIPT) { - def jsCallSource = atom.tag.name(config.sourceTag).call.argument.isIdentifier - def jsFrameworkIdentifier = atom.tag.name(FRAMEWORK_TAG).identifier - def jsFrameworkParameter = atom.tag.name(FRAMEWORK_TAG).parameter - def jsSink = atom.tag.name(config.sinkTag).call.argument.isIdentifier - flowsList ++= jsSink - .reachableByFlows(jsCallSource, jsFrameworkIdentifier, jsFrameworkParameter) + // For JavaScript and Python, we need flows between arguments of call nodes to track callbacks and middlewares + if ( + language == Languages.JSSRC || language == Languages.JAVASCRIPT || language == Languages.PYTHON || language == Languages.PYTHONSRC + ) { + def dynCallSource = atom.tag.name(config.sourceTag).call.argument.isIdentifier + def dynFrameworkIdentifier = atom.tag.name(FRAMEWORK_TAG).identifier + def dynFrameworkParameter = atom.tag.name(FRAMEWORK_TAG).parameter + def dynSink = atom.tag.name(config.sinkTag).call.argument.isIdentifier + flowsList ++= dynSink + .reachableByFlows(dynCallSource, dynFrameworkIdentifier, dynFrameworkParameter) .map(toSlice) .toList flowsList ++= atom.tag .name(FRAMEWORK_TAG) .call .argument - .reachableByFlows(jsFrameworkParameter) + .reachableByFlows(dynFrameworkParameter) .map(toSlice) .toList } diff --git a/src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala index 812435f..ac75ae6 100644 --- a/src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala +++ b/src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala @@ -4,6 +4,7 @@ import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{Languages, Operators, PropertyNames} import io.shiftleft.semanticcpg.language.* +import overflowdb.PropertyKey import java.util.concurrent.* import java.util.concurrent.atomic.AtomicBoolean @@ -44,6 +45,8 @@ object UsageSlicing { val userDefTypes = userDefinedTypes(atom) if (language.get == Languages.NEWC || language.get == Languages.C) ProgramUsageSlice(slices ++ importsAsSlices(atom), userDefTypes) + else if (language.get == Languages.PYTHON || language.get == Languages.PYTHONSRC) + ProgramUsageSlice(slices, userDefTypes ++ routesAsUDT(atom)) else ProgramUsageSlice(slices, userDefTypes) } @@ -101,6 +104,60 @@ object UsageSlicing { }) } + /** Discovers internally defined routes. + * + * @param atom + * the CPG to query for types. + * @return + * a list of user defined types. + */ + def routesAsUDT(atom: Cpg): List[UserDefinedType] = { + + def generateUDT(call: Call): UserDefinedType = { + UserDefinedType( + call.name, + call.argument + .order(1) + .isLiteral + .map(m => + LocalDef( + name = m.code, + typeFullName = m.typeFullName, + lineNumber = Option(m.property(new PropertyKey[Integer](PropertyNames.LINE_NUMBER))).map(_.toInt), + columnNumber = Option(m.property(new PropertyKey[Integer](PropertyNames.COLUMN_NUMBER))).map(_.toInt) + ) + ) + .collectAll[LocalDef] + .l, + call + .callee(NoResolve) + .method + .filterNot(m => m.name.startsWith("")) + .map(m => + ObservedCall( + m.name, + Option(m.fullName), + m.parameter.map(_.typeFullName).toList, + m.methodReturn.typeFullName, + Option(m.isExternal), + m.lineNumber.map(_.intValue()), + m.columnNumber.map(_.intValue()) + ) + ) + .l, + call.location.filename, + call.lineNumber.map(_.intValue()), + call.columnNumber.map(_.intValue()) + ) + } + + atom.call + .where(_.methodFullName("django/urls.py:.(path|re_path)")) + .map(generateUDT) + .filter(udt => udt.fields.nonEmpty || udt.procedures.nonEmpty) + .l + } + private def TimedGet(dsf: Future[Option[(Method, ObjectUsageSlice)]]) = { try { dsf.get(5, TimeUnit.SECONDS)