Update packages (#31)

Match ddg reachable usage based on both code and name. Use contains edge instead of ast for js (from upstream) Added codemeta file Tweaks to table output. Tag identifiers with description tags Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · Nov 9, 2023 · e2b5696 · e2b5696
1 parent 02a63d8
commit e2b5696
Show file tree

Hide file tree

Showing 18 changed files with 225 additions and 128 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "0.6.1"
+ThisBuild / version      := "0.6.2"
 ThisBuild / scalaVersion := "3.3.1"
 
 val cpgVersion = "1.4.22"

diff --git a/codemeta.json b/codemeta.json
@@ -0,0 +1,45 @@
+{
+  "@context": "https://doi.org/10.5063/schema/codemeta-2.0",
+  "@type": "SoftwareSourceCode",
+  "license": "https://spdx.org/licenses/Apache-2.0",
+  "codeRepository": "git+https://github.com/AppThreat/chen.git",
+  "contIntegration": "https://github.com/AppThreat/chen/actions",
+  "downloadUrl": "https://github.com/AppThreat/chen",
+  "issueTracker": "https://github.com/AppThreat/chen/issues",
+  "name": "chen",
+  "version": "0.6.2",
+  "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.",
+  "applicationCategory": "code-analysis",
+  "keywords": [
+    "static-analysis",
+    "code-analysis",
+    "dependency-analysis",
+    "code-hierarchy-analysis"
+  ],
+  "programmingLanguage": [
+    "Scala 3",
+    "Node.js",
+    "Python 3"
+  ],
+  "runtimePlatform": [
+    "JVM",
+    "Python 3"
+  ],
+  "operatingSystem": [
+    "Linux",
+    "Windows",
+    "MacOS"
+  ],
+  "softwareRequirements": [
+    "Python >= 3.8",
+    "Java >= 17"
+  ],
+  "author": [
+    {
+      "@type": "Person",
+      "givenName": "Team",
+      "familyName": "AppThreat",
+      "email": "[email protected]"
+    }
+  ]
+}
diff --git a/console/build.sbt b/console/build.sbt
@@ -25,9 +25,9 @@ libraryDependencies ++= Seq(
   "io.circe"             %% "circe-parser"      % CirceVersion,
   "org.zeroturnaround"    % "zt-zip"            % ZeroturnaroundVersion,
   "com.lihaoyi"          %% "os-lib"            % "0.9.1",
-  "com.lihaoyi"          %% "pprint"            % "0.7.3",
+  "com.lihaoyi"          %% "pprint"            % "0.8.1",
   "com.lihaoyi"          %% "cask"              % CaskVersion,
-  "me.shadaj"            %% "scalapy-core"      % "0.5.2",
+  "dev.scalapy"          %% "scalapy-core"      % "0.5.3",
   "org.scala-lang.modules" % "scala-asm"        % "9.5.0-scala-1",
   "org.scalatest"        %% "scalatest"         % Versions.scalatest % Test
 )

diff --git a/console/src/main/scala/io/appthreat/console/cpgcreation/AtomGenerator.scala b/console/src/main/scala/io/appthreat/console/cpgcreation/AtomGenerator.scala
@@ -34,8 +34,6 @@ case class AtomGenerator(
   override def isAvailable: Boolean = true
 
   override def applyPostProcessingPasses(atom: Cpg): Cpg = {
-    new CdxPass(atom).createAndApply()
-    new ChennaiTagsPass(atom).createAndApply()
     atom
   }
 

diff --git a/dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/language/Path.scala b/dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/language/Path.scala
@@ -2,16 +2,11 @@ package io.appthreat.dataflowengineoss.language
 
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.semanticcpg.language.*
-import io.shiftleft.semanticcpg.language.Show
+import me.shadaj.scalapy.py
 import org.apache.commons.lang.StringUtils
 
 import scala.collection.mutable.{ArrayBuffer, Set}
 
-import me.shadaj.scalapy.py
-import me.shadaj.scalapy.py.SeqConverters
-import py.PyQuote
-import me.shadaj.scalapy.interpreter.CPythonInterpreter
-
 case class Path(elements: List[AstNode]) {
   def resultPairs(): List[(String, Option[Integer])] = {
     val pairs = elements.map {
@@ -30,9 +25,10 @@ case class Path(elements: List[AstNode]) {
 
 object Path {
 
-  val DefaultMaxTrackedWidth = 128
+  private val DefaultMaxTrackedWidth = 128
   // TODO replace with dynamic rendering based on the terminal's width, e.g. in scala-repl-pp
-  lazy val maxTrackedWidth = sys.env.get("CHEN_DATAFLOW_TRACKED_WIDTH").map(_.toInt).getOrElse(DefaultMaxTrackedWidth)
+  private lazy val maxTrackedWidth =
+    sys.env.get("CHEN_DATAFLOW_TRACKED_WIDTH").map(_.toInt).getOrElse(DefaultMaxTrackedWidth)
 
   private def tagAsString(tag: Iterator[Tag]) =
     if (tag.nonEmpty) tag.name.mkString(", ") else ""
@@ -50,13 +46,14 @@ object Path {
           val method = cfgNode.method
           sinkCode = method.fullName
       }
-      caption = s"Source: ${srcNode.code}"
+      caption = if (srcNode.code != "this") s"Source: ${srcNode.code}" else ""
       if (srcTags.nonEmpty) caption += s"\nSource Tags: ${srcTags}"
       caption += s"\nSink: ${sinkCode}\n"
       if (sinkTags.nonEmpty) caption += s"Sink Tags: ${sinkTags}\n"
     }
-    val tableRows  = ArrayBuffer[Array[String]]()
-    val addedPaths = Set[String]()
+    var hasCheckLike: Boolean = false;
+    val tableRows             = ArrayBuffer[Array[String]]()
+    val addedPaths            = Set[String]()
     path.elements.foreach { astNode =>
       val nodeType     = astNode.getClass.getSimpleName
       val lineNumber   = astNode.lineNumber.getOrElse("").toString
@@ -155,17 +152,28 @@ object Path {
           val tracked = StringUtils.normalizeSpace(StringUtils.abbreviate(statement, maxTrackedWidth))
           tableRows += Array[String]("cfgNode", fileLocation, methodName, "", tracked, tags)
       }
+      if (isCheckLike(tags)) hasCheckLike = true
       addedPaths += s"${fileName}#${lineNumber}"
     }
     try {
+      if (hasCheckLike) caption = s"This flow is safe with mitigation in place.\n$caption"
       printFlows(tableRows, caption)
     } catch {
       case exc: Exception =>
     }
     caption
   }
 
-  def printFlows(tableRows: ArrayBuffer[Array[String]], caption: String): Unit = {
+  private def addEmphasis(str: String, isCheckLike: Boolean): String = if (isCheckLike) s"[green]$str[/green]" else str
+
+  private def simplifyFilePath(str: String): String = str.replace("src/main/java/", "").replace("src/main/scala/", "")
+
+  private def isCheckLike(tagsStr: String): Boolean =
+    tagsStr.contains("valid") || tagsStr.contains("encrypt") || tagsStr.contains("encode") || tagsStr.contains(
+      "transform"
+    ) || tagsStr.contains("check")
+
+  private def printFlows(tableRows: ArrayBuffer[Array[String]], caption: String): Unit = {
     val richTableLib = py.module("rich.table")
     val richConsole  = py.module("chenpy.logger").console
     val table        = richTableLib.Table(highlight = true, expand = true, caption = caption)
@@ -174,14 +182,14 @@ object Path {
       {
         val end_section         = row.head == "call"
         val trow: Array[String] = row.tail
-        if (!trow(4).startsWith("<operator>.fieldAccess")) {
+        if (!trow(3).startsWith("<operator") && trow(3) != "<empty>" && trow(3) != "RET") {
           val tagsStr: String = if (trow(4).nonEmpty) s"Tags: ${trow(4)}" else ""
           val methodStr       = s"${trow(1)}\n${tagsStr}"
           table.add_row(
-            trow(0),
+            simplifyFilePath(trow(0)),
             methodStr.stripMargin,
-            trow(2),
-            trow(3).takeWhile(_ != '\n'),
+            addEmphasis(trow(2), isCheckLike(tagsStr)),
+            addEmphasis(trow(3).takeWhile(_ != '\n'), isCheckLike(tagsStr)),
             end_section = end_section
           )
         }

diff --git a/...neoss/src/main/scala/io/appthreat/dataflowengineoss/passes/reachingdef/DdgGenerator.scala b/...neoss/src/main/scala/io/appthreat/dataflowengineoss/passes/reachingdef/DdgGenerator.scala
@@ -352,13 +352,14 @@ private class UsageAnalyzer(
   private def sameVariable(use: StoredNode, inElement: StoredNode): Boolean = {
     inElement match {
       case param: MethodParameterIn =>
-        nodeToString(use).contains(param.name)
+        nodeToString(use).contains(param.name) || nodeToString(use).contains(param.code)
       case call: Call if indirectionAccessSet.contains(call.name) =>
         call.argumentOption(1).exists(x => nodeToString(use).contains(x.code))
       case call: Call =>
         nodeToString(use).contains(call.code)
-      case identifier: Identifier => nodeToString(use).contains(identifier.code)
-      case _                      => false
+      case identifier: Identifier =>
+        nodeToString(use).contains(identifier.name) || nodeToString(use).contains(identifier.code)
+      case _ => false
     }
   }
 

diff --git a/platform/frontends/c2cpg/build.sbt b/platform/frontends/c2cpg/build.sbt
@@ -4,13 +4,13 @@ dependsOn(Projects.semanticcpg, Projects.dataflowengineoss % Test, Projects.x2cp
 
 libraryDependencies ++= Seq(
   "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4",
-  "org.eclipse.platform"    % "org.eclipse.equinox.common"       % "3.18.0",
-  "org.eclipse.platform"    % "org.eclipse.core.resources"       % "3.19.0" excludeAll(
+  "org.eclipse.platform"    % "org.eclipse.equinox.common"       % "3.18.100",
+  "org.eclipse.platform"    % "org.eclipse.core.resources"       % "3.19.100" excludeAll(
     ExclusionRule(organization = "com.ibm.icu", name = "icu4j"),
     ExclusionRule(organization = "org.eclipse.platform", name = "org.eclipse.jface"),
     ExclusionRule(organization = "org.eclipse.platform", name = "org.eclipse.jface.text")
   ),
-  "org.jline"               % "jline"                      % "3.23.0",
+  "org.jline"               % "jline"                      % "3.24.1",
   "org.scalatest"          %% "scalatest"                  % Versions.scalatest % Test
 )
 

diff --git a/...form/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/AstCreatorHelper.scala b/...form/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/AstCreatorHelper.scala
@@ -195,8 +195,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
     node match {
       case f: CPPASTFieldReference =>
         f.getFieldOwner.getEvaluation match {
-          case evaluation: EvalMemberAccess => cleanType(evaluation.getOwnerType.toString, stripKeywords)
-          case evaluation: EvalBinding      => cleanType(evaluation.getType.toString, stripKeywords)
+          case evaluation: EvalBinding => cleanType(evaluation.getType.toString, stripKeywords)
           case _ => cleanType(ASTTypeUtil.getType(f.getFieldOwner.getExpressionType), stripKeywords)
         }
       case f: IASTFieldReference =>

diff --git a/platform/frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/types/ClassTypeTests.scala b/platform/frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/types/ClassTypeTests.scala
@@ -130,6 +130,34 @@ class ClassTypeTests extends CCodeToCpgSuite(FileDefaults.CPP_EXT) {
     "should allow traversing from type to enclosing file" in {
       cpg.typeDecl.file.filter(_.name.endsWith(FileDefaults.CPP_EXT)).l should not be empty
     }
+    "handling C++ classes (code example 3)" should {
+      "generate correct call fullnames" in {
+        val cpg = code(
+          """
+            |class B {
+            |public:
+            |  void foo2() {}
+            |};
+            |
+            |class A {
+            |private:
+            |  B b;
+            |
+            |public:
+            |  void foo1() {
+            |    b.foo2();
+            |   }
+            |};
+            |
+            |int main() {
+            |  A a;
+            |  a.foo1();
+            |  return 0;
+            |}""".stripMargin)
+        cpg.call("foo1").methodFullName.toSetMutable shouldBe Set("A.foo1")
+        cpg.call("foo2").methodFullName.toSetMutable shouldBe Set("B.foo2")
+      }
+    }
   }
 
 }
diff --git a/platform/frontends/javasrc2cpg/build.sbt b/platform/frontends/javasrc2cpg/build.sbt
@@ -4,7 +4,7 @@ dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->t
 
 libraryDependencies ++= Seq(
   "io.shiftleft"           %% "codepropertygraph"             % Versions.cpg,
-  "com.github.javaparser"   % "javaparser-symbol-solver-core" % "3.25.5",
+  "com.github.javaparser"   % "javaparser-symbol-solver-core" % "3.25.6",
   "org.gradle"              % "gradle-tooling-api"            % Versions.gradleTooling,
   "org.scalatest"          %% "scalatest"                     % Versions.scalatest % Test,
   "org.projectlombok"       % "lombok"                        % "1.18.30",

diff --git a/platform/frontends/jimple2cpg/build.sbt b/platform/frontends/jimple2cpg/build.sbt
@@ -5,11 +5,11 @@ dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->t
 libraryDependencies ++= Seq(
   "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
   "org.soot-oss"   % "soot"              % "4.4.1",
-  "org.scala-lang.modules" % "scala-asm" % "9.5.0-scala-1",
-  "org.ow2.asm"            % "asm"       % "9.5",
-  "org.ow2.asm"            % "asm-analysis"       % "9.5",
-  "org.ow2.asm"            % "asm-util"           % "9.5",
-  "org.ow2.asm"            % "asm-tree"           % "9.5",
+  "org.scala-lang.modules" % "scala-asm" % "9.6.0-scala-1",
+  "org.ow2.asm"            % "asm"       % "9.6",
+  "org.ow2.asm"            % "asm-analysis"       % "9.6",
+  "org.ow2.asm"            % "asm-util"           % "9.6",
+  "org.ow2.asm"            % "asm-tree"           % "9.6",
   "org.scalatest" %% "scalatest"         % Versions.scalatest % Test
 )
 

diff --git a/...frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/ImportResolverPass.scala b/...frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/ImportResolverPass.scala
@@ -59,7 +59,7 @@ class ImportResolverPass(cpg: Cpg) extends XImportResolverPass(cpg) {
 
     def targetAssignments = targetModule
       .nameExact(":program")
-      .ast
+      .flatMap(_._callViaContainsOut)
       .assignment
 
     val matchingExports = if (isImportingModule) {
@@ -82,11 +82,11 @@ class ImportResolverPass(cpg: Cpg) extends XImportResolverPass(cpg) {
          exp.argument.l match {
            case ::(expCall: Call, ::(b: Identifier, _))
                if expCall.code.matches("^(module.)?exports[.]?.*") && b.name == alias =>
-             val moduleMethods      = targetModule.ast.isMethod.l
+             val moduleMethods      = targetModule.repeat(_.astChildren.isMethod)(_.emit).l
              lazy val methodMatches = moduleMethods.name(b.name).l
              lazy val constructorMatches =
                moduleMethods.fullName(s".*${b.name}$pathSep${XDefines.ConstructorMethodName}$$").l
-             lazy val moduleExportsThisVariable = moduleMethods.ast.isLocal
+             lazy val moduleExportsThisVariable = moduleMethods.body.local
                .where(_.nameExact(b.name))
                .nonEmpty
              // Exported function with only the name of the function

diff --git a/...tends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/JavaScriptTypeRecovery.scala b/...tends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/JavaScriptTypeRecovery.scala
@@ -1,12 +1,12 @@
 package io.appthreat.jssrc2cpg.passes
 
+import io.appthreat.x2cpg.Defines as XDefines
 import io.appthreat.x2cpg.Defines.ConstructorMethodName
-import io.appthreat.x2cpg.passes.frontend._
-import io.appthreat.x2cpg.{Defines => XDefines}
+import io.appthreat.x2cpg.passes.frontend.*
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated.nodes._
+import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{Operators, PropertyNames}
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess
 import overflowdb.BatchedUpdate.DiffGraphBuilder
 
@@ -92,8 +92,7 @@ private class RecoverForJavaScriptFile(cpg: Cpg, cu: File, builder: DiffGraphBui
 
   private lazy val exportedIdentifiers = cu.method
     .nameExact(":program")
-    .ast
-    .isCall
+    .flatMap(_._callViaContainsOut)
     .nameExact(Operators.assignment)
     .filter(_.code.startsWith("exports.*"))
     .argument
@@ -183,7 +182,8 @@ private class RecoverForJavaScriptFile(cpg: Cpg, cu: File, builder: DiffGraphBui
 
   override protected def postSetTypeInformation(): Unit = {
     // often there are "this" identifiers with type hints but this can be set to a type hint if they meet the criteria
-    cu.ast.isIdentifier
+    cu.method
+      .flatMap(_._identifierViaContainsOut)
       .nameExact("this")
       .where(_.typeFullNameExact(Defines.Any))
       .filterNot(_.dynamicTypeHintFullName.isEmpty)

diff --git a/...ds/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/frontend/XInheritanceFullNamePass.scala b/...ds/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/frontend/XInheritanceFullNamePass.scala
@@ -57,7 +57,9 @@ abstract class XInheritanceFullNamePass(cpg: Cpg) extends ForkJoinParallelCpgPas
   }
 
   protected def resolveInheritedTypeFullName(td: TypeDecl, builder: DiffGraphBuilder): Seq[TypeDeclBase] = {
-    val qualifiedNamesInScope = td.file.ast
+    val callsOfInterest     = td.file.method.flatMap(_._callViaContainsOut)
+    val typeDeclsOfInterest = td.file.typeDecl
+    val qualifiedNamesInScope = (callsOfInterest ++ typeDeclsOfInterest)
       .flatMap(extractTypeDeclFromNode)
       .filterNot(_.endsWith(moduleName))
       .l

diff --git a/...orm/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/frontend/XTypeRecovery.scala b/...orm/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/frontend/XTypeRecovery.scala
@@ -7,7 +7,7 @@ import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, Operators
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.language.operatorextension.OpNodes
-import OpNodes.{Assignment, FieldAccess}
+import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.{Assignment, FieldAccess}
 import org.slf4j.{Logger, LoggerFactory}
 import overflowdb.BatchedUpdate
 import overflowdb.BatchedUpdate.DiffGraphBuilder
@@ -266,14 +266,26 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode](
     case x => x.getKnownTypes.nonEmpty
   }
 
-  protected def assignments: Iterator[Assignment] =
-    cu.ast.isCall.nameExact(Operators.assignment).map(new OpNodes.Assignment(_))
+  protected def assignments: Iterator[Assignment] = cu match {
+    case x: File =>
+      x.method.flatMap(_._callViaContainsOut).nameExact(Operators.assignment).map(new OpNodes.Assignment(_))
+    case x: Method => x.flatMap(_._callViaContainsOut).nameExact(Operators.assignment).map(new OpNodes.Assignment(_))
+    case x         => x.ast.isCall.nameExact(Operators.assignment).map(new OpNodes.Assignment(_))
+  }
 
   protected def members: Iterator[Member] = cu.ast.isMember
 
-  protected def returns: Iterator[Return] = cu.ast.isReturn
+  protected def returns: Iterator[Return] = cu match {
+    case x: File   => x.method.flatMap(_._returnViaContainsOut)
+    case x: Method => x._returnViaContainsOut
+    case x         => x.ast.isReturn
+  }
 
-  protected def importNodes: Iterator[Import] = cu.ast.isCall.referencedImports
+  protected def importNodes: Iterator[Import] = cu match {
+    case x: File   => x.method.flatMap(_._callViaContainsOut).referencedImports
+    case x: Method => x.file.method.flatMap(_._callViaContainsOut).referencedImports
+    case x         => x.ast.isCall.referencedImports
+  }
 
   override def compute(): Boolean = try {
     // Set known aliases that point to imports for local and external methods/modules