Improvements to the configuration (#2091)

cpg-concepts/src/integrationTest/kotlin/de/fraunhofer/aisec/cpg/concepts/ConfigurationPassTest.kt

@@ -32,20 +32,23 @@ import de.fraunhofer.aisec.cpg.graph.*
 import de.fraunhofer.aisec.cpg.graph.concepts.config.Configuration
 import de.fraunhofer.aisec.cpg.graph.concepts.config.ConfigurationGroup
 import de.fraunhofer.aisec.cpg.graph.concepts.config.ConfigurationOption
+import de.fraunhofer.aisec.cpg.graph.concepts.config.ConfigurationSource
 import de.fraunhofer.aisec.cpg.graph.concepts.config.LoadConfiguration
 import de.fraunhofer.aisec.cpg.graph.concepts.config.ReadConfigurationGroup
 import de.fraunhofer.aisec.cpg.graph.concepts.config.ReadConfigurationOption
 import de.fraunhofer.aisec.cpg.graph.concepts.config.RegisterConfigurationGroup
 import de.fraunhofer.aisec.cpg.graph.concepts.config.RegisterConfigurationOption
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.SubscriptExpression
-import de.fraunhofer.aisec.cpg.passes.concepts.config.ini.IniFileConfigurationPass
+import de.fraunhofer.aisec.cpg.passes.concepts.config.ProvideConfigPass
+import de.fraunhofer.aisec.cpg.passes.concepts.config.ini.IniFileConfigurationSourcePass
 import de.fraunhofer.aisec.cpg.passes.concepts.config.python.PythonStdLibConfigurationPass
 import de.fraunhofer.aisec.cpg.test.analyze
 import java.io.File
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertNotNull
 import kotlin.test.assertSame
+import kotlin.test.assertTrue
 
 class ConfigurationPassTest {
     @Test
@@ -55,8 +58,9 @@ class ConfigurationPassTest {
             analyze(listOf(), topLevel.toPath(), true) {
                 it.registerLanguage<PythonLanguage>()
                 it.registerLanguage<IniFileLanguage>()
-                it.registerPass<IniFileConfigurationPass>()
+                it.registerPass<IniFileConfigurationSourcePass>()
                 it.registerPass<PythonStdLibConfigurationPass>()
+                it.registerPass<ProvideConfigPass>()
                 it.softwareComponents(
                     mutableMapOf(
                         "conf" to listOf(topLevel.resolve("conf")),
@@ -76,6 +80,20 @@ class ConfigurationPassTest {
 
         val conf = result.conceptNodes.filterIsInstance<Configuration>().singleOrNull()
         assertNotNull(conf, "There should be a single configuration node")
+        assertEquals(11, conf.allOps.size, "There should be 11 overall ops in the configuration")
+
+        val confSources = result.conceptNodes.filterIsInstance<ConfigurationSource>()
+        assertEquals(3, confSources.size, "There should be three configuration source nodes")
+
+        confSources
+            .filter { it.name.toString() != "unused.ini" }
+            .forEach {
+                assertEquals(
+                    5,
+                    it.allOps.size,
+                    "There should be 5 overall ops in each configuration source (except for the unused one)",
+                )
+            }
 
         val loadConfig = result.operationNodes.filterIsInstance<LoadConfiguration>().singleOrNull()
         assertNotNull(loadConfig)
@@ -89,6 +107,11 @@ class ConfigurationPassTest {
 
         val sslGroup = groups["ssl"]
         assertNotNull(sslGroup)
+        assertEquals(
+            2,
+            sslGroup.ops.size,
+            "There should be two ops in the ssl group (register and read)",
+        )
 
         val readGroupOps = result.operationNodes.filterIsInstance<ReadConfigurationGroup>()
         assertEquals(
@@ -117,6 +140,17 @@ class ConfigurationPassTest {
             readOptionOps.associate { Pair(it.name.toString(), it.option) },
         )
 
+        readOptionOps
+            .map { it.underlyingNode }
+            .forEach {
+                // Prev DFG should include the option
+                assertNotNull(it)
+                assertTrue(
+                    it.prevDFG.any { dfg -> dfg is ConfigurationOption },
+                    "Prev DFG of $it should include the option",
+                )
+            }
+
         val registerOptionOps =
             result.operationNodes.filterIsInstance<RegisterConfigurationOption>()
         assertEquals(
@@ -134,5 +168,13 @@ class ConfigurationPassTest {
         val sslEnabled = result.refs["ssl_enabled"]
         assertNotNull(sslEnabled)
         assertEquals(setOf("true", "false"), sslEnabled.evaluate(MultiValueEvaluator()))
+
+        val unused = confSources["unused.ini"]
+        assertNotNull(unused)
+        assertEquals(0, unused.allOps.size, "There should be no ops in the unused configuration")
+
+        val verifySSL = unused.groups["DEFAULT"]?.options["verify_ssl"]
+        assertNotNull(verifySSL)
+        assertEquals(setOf("true"), verifySSL.evaluate(MultiValueEvaluator()))
     }
 }

cpg-concepts/src/integrationTest/resources/python/conf/unused.ini

@@ -0,0 +1,2 @@
+[DEFAULT]
+verify_ssl = true

cpg-concepts/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/concepts/config/Configuration.kt

@@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.graph.concepts.Concept
 import de.fraunhofer.aisec.cpg.graph.concepts.Operation
 import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration
 import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration
+import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression
 
 /**
@@ -44,6 +45,17 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression
  */
 class Configuration(underlyingNode: Node) : Concept(underlyingNode = underlyingNode) {
     var groups: MutableList<ConfigurationGroup> = mutableListOf()
+
+    /**
+     * The individual operations that target parts of the configuration are assigned to
+     * [Concept.ops] of their respective concept. For example a [ReadConfigurationGroup] is part of
+     * the [ConfigurationGroup]'s ops. This property returns all operations of all groups and
+     * options as well as the ones targeting the complete configuration.
+     */
+    val allOps: Set<Operation>
+        get() {
+            return ops + groups.flatMap { it.ops + it.options.flatMap { option -> option.ops } }
+        }
 }
 
 /**
@@ -110,10 +122,9 @@ class LoadConfiguration(
  */
 class ReadConfigurationGroup(
     underlyingNode: Node,
-    conf: Configuration,
     /** The config group that is being read with this operation. */
     var group: ConfigurationGroup,
-) : ConfigurationOperation(underlyingNode = underlyingNode, concept = conf) {
+) : ConfigurationOperation(underlyingNode = underlyingNode, concept = group) {
     init {
         name = group.name
     }
@@ -125,10 +136,9 @@ class ReadConfigurationGroup(
  */
 class ReadConfigurationOption(
     underlyingNode: Node,
-    conf: Configuration,
     /** The config option that is being read with this operation. */
     var option: ConfigurationOption,
-) : ConfigurationOperation(underlyingNode = underlyingNode, concept = conf) {
+) : ConfigurationOperation(underlyingNode = underlyingNode, concept = option) {
     init {
         name = option.name
     }
@@ -146,10 +156,9 @@ class ReadConfigurationOption(
  */
 class RegisterConfigurationGroup(
     underlyingNode: Node,
-    conf: Configuration,
     /** The config group that is being registered with this operation. */
     var group: ConfigurationGroup,
-) : ConfigurationOperation(underlyingNode = underlyingNode, concept = conf) {
+) : ConfigurationOperation(underlyingNode = underlyingNode, concept = group) {
     init {
         name = group.name
     }
@@ -167,56 +176,107 @@ class RegisterConfigurationGroup(
  */
 class RegisterConfigurationOption(
     underlyingNode: Node,
-    conf: Configuration,
     /** The config option that is being registered with this operation. */
     var option: ConfigurationOption,
     /** An optional default value of the option. */
     var defaultValue: Node? = null,
-) : ConfigurationOperation(underlyingNode = underlyingNode, concept = conf) {
+) : ConfigurationOperation(underlyingNode = underlyingNode, concept = option) {
     init {
         name = option.name
     }
 }
 
 /**
- * Represents an operation to provide a [Configuration], e.g., in the form of a configuration file.
+ * Represents an operation to provide a [Configuration], e.g., in the form of a configuration file
+ * (through a [ConfigurationSource]).
  *
  * When the configuration file is loaded, a [LoadConfiguration] operation would be found in the code
  * component (matching the configuration file's name in [LoadConfiguration.fileExpression]) and the
  * [ProvideConfiguration] operation would be found in the configuration component.
  *
  * But also other sources of configuration could be represented by a [ProvideConfiguration]
  * operation, such as environment variables or command-line arguments.
+ *
+ * Note: The [ProvideConfiguration] operation is part of the [ConfigurationSource.ops] and not of
+ * the [Configuration.ops] as its an operation of the source, not the target.
  */
-class ProvideConfiguration(underlyingNode: Node, var conf: Configuration) :
-    ConfigurationOperation(underlyingNode = underlyingNode, concept = conf)
+class ProvideConfiguration(
+    underlyingNode: Node,
+    var source: ConfigurationSource,
+    var conf: Configuration,
+) : ConfigurationOperation(underlyingNode = underlyingNode, concept = source)
 
 /**
- * Represents an operation to provide a [ConfigurationGroup]. For example, when loading an INI file
- * with our INI file frontend, each section is presented as a [RecordDeclaration]. This record
- * declaration would "provide" the configuration group.
+ * Represents an operation to provide a [ConfigurationGroup]. It connects a
+ * [ConfigurationGroupSource] with a [ConfigurationGroup].
  */
 class ProvideConfigurationGroup(
     underlyingNode: Node,
-    conf: Configuration,
+    var source: ConfigurationGroupSource,
     var group: ConfigurationGroup,
-) : ConfigurationOperation(underlyingNode = underlyingNode, concept = conf) {
+) : ConfigurationOperation(underlyingNode = underlyingNode, concept = source) {
     init {
         name = group.name
     }
 }
 
 /**
- * Represents an operation to provide a [ConfigurationOption]. For example, when loading an INI file
- * with our INI file frontend, each key-value pair is presented as a [FieldDeclaration]. This field
- * declaration would "provide" the configuration option.
+ * Represents an operation to provide a [ConfigurationOption]. It connects a
+ * [ConfigurationOptionSource] with a [ConfigurationOption].
  */
 class ProvideConfigurationOption(
     underlyingNode: Node,
+    var source: ConfigurationOptionSource,
     var option: ConfigurationOption,
     var value: Node?,
-) : ConfigurationOperation(underlyingNode = underlyingNode, concept = option.group.conf) {
+) : ConfigurationOperation(underlyingNode = underlyingNode, concept = source) {
     init {
         name = option.name
     }
 }
+
+/**
+ * Represents a possible source for a configuration. For example, when loading an INI file with our
+ * INI file frontend, the whole file would be represented as a [TranslationUnitDeclaration]. This
+ * translation unit declaration would be the source of the configuration.
+ */
+class ConfigurationSource(underlyingNode: Node) : Concept(underlyingNode = underlyingNode) {
+    val groups: MutableList<ConfigurationGroupSource> = mutableListOf()
+
+    /**
+     * The individual operations that target parts of the configuration are assigned to
+     * [Concept.ops] of their respective concept. For example a [ReadConfigurationGroup] is part of
+     * the [ConfigurationGroup]'s ops. This property returns all operations of all groups and
+     * options as well as the ones targeting the complete configuration.
+     */
+    val allOps: Set<Operation>
+        get() {
+            return ops + groups.flatMap { it.ops + it.options.flatMap { option -> option.ops } }
+        }
+}
+
+/**
+ * Represents a possible group source for a configuration group. For example, when loading an INI
+ * file with our INI file frontend, each section is presented as a [RecordDeclaration]. This record
+ * declaration would be the source of the configuration group.
+ */
+class ConfigurationGroupSource(underlyingNode: Node, conf: ConfigurationSource) :
+    Concept(underlyingNode = underlyingNode) {
+    val options: MutableList<ConfigurationOptionSource> = mutableListOf()
+
+    init {
+        conf.groups += this
+    }
+}
+
+/**
+ * Represents a possible option source for a configuration option. For example, when loading an INI
+ * file with our INI file frontend, each key-value pair is presented as a [FieldDeclaration]. This
+ * field declaration would be the source to the configuration option.
+ */
+class ConfigurationOptionSource(underlyingNode: Node, var group: ConfigurationGroupSource) :
+    Concept(underlyingNode = underlyingNode) {
+    init {
+        group.options += this
+    }
+}