Merge branch 'master' into ssrf-e2e

WebFuzzing · Jan 27, 2025 · 6b3d3ec · 6b3d3ec
2 parents ead4913 + 0ce7b5e
commit 6b3d3ec
Show file tree

Hide file tree

Showing 51 changed files with 1,969 additions and 144 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,6 +13,7 @@
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
 
+*.DS_Store
 /.idea/
 .idea/
 *.iml
@@ -123,6 +124,7 @@ Migrations/
 /e2e-tests/spring-rest-mysql-column-types/target/
 /e2e-tests/spring-rest-postgres-column-types/target/
 /e2e-tests/spring-rest-h2-column-types/target/
+/e2e-tests/spring-rest-h2-z3solver/target/
 /test-old-libraries/target/
 /e2e-tests/spring-web/target/
 /e2e-tests/spring-rest-mongo/target/
@@ -166,6 +168,7 @@ Migrations/
 /e2e-tests/spring-rest-bb/maven/target/
 /target/
 /wfc/target/
+
 /e2e-tests/emb-json/target/
 /process_data/
 /e2e-tests/spring-rest-multidb/target/
diff --git a/...ava/org/evomaster/client/java/controller/api/dto/database/execution/SqlExecutionsDto.java b/...ava/org/evomaster/client/java/controller/api/dto/database/execution/SqlExecutionsDto.java
@@ -54,9 +54,9 @@ public class SqlExecutionsDto {
      * Every time there is a WHERE clause which "failed" (ie, resulted in false),
      * we keep track of which tables/columns where involved in determining the
      * result of the WHERE.
-     *
      * If there is no WHERE, but still no data was returned, we consider it
      * as a failed WHERE
+     * The key is the table name and the value is the set of columns involved in the WHERE
      */
     public Map<String, Set<String>> failedWhere = new HashMap<>();
 

diff --git a/core/src/main/kotlin/org/evomaster/core/BaseModule.kt b/core/src/main/kotlin/org/evomaster/core/BaseModule.kt
@@ -14,8 +14,7 @@ import org.evomaster.core.search.service.mutator.MutationWeightControl
 import org.evomaster.core.search.service.mutator.genemutation.ArchiveGeneMutator
 import org.evomaster.core.search.tracer.ArchiveMutationTrackService
 import org.evomaster.core.search.tracer.TrackService
-
-
+import org.evomaster.core.solver.SMTLibZ3DbConstraintSolver
 
 
 /**
@@ -87,6 +86,9 @@ class BaseModule(val args: Array<String>, val noTests: Boolean = false) : Abstra
         bind(ExecutionInfoReporter::class.java)
                 .asEagerSingleton()
 
+        bind(SMTLibZ3DbConstraintSolver::class.java)
+            .asEagerSingleton()
+
         //no longer needed if TestSuiteWriter is moved out?
 //        if(noTests){
 //            bind(TestCaseWriter::class.java)

diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt
@@ -12,8 +12,6 @@ import org.evomaster.core.AnsiColor.Companion.inRed
 import org.evomaster.core.AnsiColor.Companion.inYellow
 import org.evomaster.core.config.ConfigProblemException
 import org.evomaster.core.logging.LoggingUtil
-import org.evomaster.core.output.OutputFormat
-import org.evomaster.core.output.Termination
 import org.evomaster.core.output.TestSuiteSplitter
 import org.evomaster.core.output.clustering.SplitResult
 import org.evomaster.core.output.service.TestSuiteWriter

diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt
@@ -591,7 +591,6 @@ class RestTestCaseWriter : HttpWsTestCaseWriter {
 
     private fun getAllUsedExamples(ind: RestIndividual) : List<String>{
         return ind.seeFullTreeGenes()
-            .filterIsInstance<EnumGene<*>>()
             .filter { it.name == RestActionBuilderV3.EXAMPLES_NAME }
             .filter { it.staticCheckIfImpactPhenotype() }
             .map { it.getValueAsRawString() }

diff --git a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt
@@ -22,6 +22,7 @@ import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene
 import org.evomaster.core.search.impact.impactinfocollection.ImpactsOfIndividual
 import org.evomaster.core.search.service.mutator.MutatedGeneSpecification
 import org.evomaster.core.search.service.mutator.StructureMutator
+import org.evomaster.core.solver.SMTLibZ3DbConstraintSolver
 import org.evomaster.core.sql.SqlAction
 import org.evomaster.core.sql.SqlActionUtils
 import org.evomaster.core.sql.SqlInsertBuilder
@@ -46,6 +47,9 @@ abstract class ApiWsStructureMutator : StructureMutator() {
     @Inject
     protected lateinit var harvestResponseHandler: HarvestActualHttpWsResponseHandler
 
+    @Inject
+    protected lateinit var z3Solver: SMTLibZ3DbConstraintSolver
+
     override fun addAndHarvestExternalServiceActions(
         individual: EvaluatedIndividual<*>,
         /**
@@ -268,7 +272,8 @@ abstract class ApiWsStructureMutator : StructureMutator() {
 
         val oldSqlActions = mutableListOf<EnvironmentAction>().plus(ind.seeInitializingActions())
 
-        val addedSqlInsertions = handleFailedWhereSQL(ind, fw, mutatedGenes, sampler)
+        val failedWhereQueries = evaluatedIndividual.fitness.getViewOfAggregatedFailedWhereQueries()
+        val addedSqlInsertions = handleFailedWhereSQL(ind, fw, failedWhereQueries, mutatedGenes, sampler)
 
         ind.repairInitializationActions(randomness)
         // update impact based on added genes
@@ -289,6 +294,10 @@ abstract class ApiWsStructureMutator : StructureMutator() {
          * Map of FAILED WHERE clauses. from table name key to column name values
          */
         fw: Map<String, Set<String>>,
+        /**
+         * List queries with FAILED WHERE clauses
+         */
+        failedWhereQueries: List<String>,
         mutatedGenes: MutatedGeneSpecification?, sampler: ApiWsSampler<T>
     ): MutableList<List<SqlAction>>? {
 
@@ -297,7 +306,7 @@ abstract class ApiWsStructureMutator : StructureMutator() {
         }
 
         if (config.generateSqlDataWithDSE) {
-            return handleDSE(sampler, fw)
+            return handleDSE(ind, sampler, failedWhereQueries)
         }
 
         return mutableListOf()
@@ -385,9 +394,18 @@ abstract class ApiWsStructureMutator : StructureMutator() {
         return addedSqlInsertions
     }
 
-    private fun <T : ApiWsIndividual> handleDSE(sampler: ApiWsSampler<T>, fw: Map<String, Set<String>>): MutableList<List<SqlAction>> {
-        /* TODO: DSE should be plugged in here */
-        return mutableListOf()
+    private fun <T : ApiWsIndividual> handleDSE(ind: T, sampler: ApiWsSampler<T>, failedWhereQueries: List<String>): MutableList<List<SqlAction>> {
+        val schemaDto = sampler.sqlInsertBuilder?.schemaDto
+            ?: throw IllegalStateException("No DB schema is available")
+
+        val newActions = mutableListOf<List<SqlAction>>()
+        for (query in failedWhereQueries) {
+            val newActionsForQuery = z3Solver.solve(schemaDto, query)
+            newActions.addAll(mutableListOf(newActionsForQuery))
+            ind.addInitializingDbActions(actions = newActionsForQuery)
+        }
+
+        return newActions
     }
 
     private fun <T : ApiWsIndividual> handleFailedFind(