From 8d271a6fc778ac8490779b269d3c0d7f5bcb70b1 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 3 Dec 2024 15:42:01 +0100 Subject: [PATCH 01/21] starting preparing E2E tests --- .../multidb/separatedschemas/EntityX.kt | 15 ++++++++ .../multidb/separatedschemas/EntityY.kt | 15 ++++++++ .../multidb/separatedschemas/RepositoryX.kt | 5 +++ .../multidb/separatedschemas/RepositoryY.kt | 5 +++ .../SeparatedSchemasApplication.kt | 23 +++++++++++ .../separatedschemas/SeparatedSchemasRest.kt | 38 +++++++++++++++++++ .../SeparatedSchemasController.kt | 20 ++++++++++ .../SeparatedSchemasEMTest.kt | 36 ++++++++++++++++++ 8 files changed, 157 insertions(+) create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityX.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityY.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryX.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryY.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasApplication.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt create mode 100644 e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityX.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityX.kt new file mode 100644 index 0000000000..b0e84d19fb --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityX.kt @@ -0,0 +1,15 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import javax.persistence.Entity +import javax.persistence.Id +import javax.persistence.Table +import javax.validation.constraints.NotNull + +@Entity +@Table(name = "EntityX", schema = "foo") +open class EntityX { + + @get:Id + @get:NotNull + open var id: String? = null +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityY.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityY.kt new file mode 100644 index 0000000000..30ed285600 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityY.kt @@ -0,0 +1,15 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import javax.persistence.Entity +import javax.persistence.Id +import javax.persistence.Table +import javax.validation.constraints.NotNull + +@Entity +@Table(name = "EntityY", schema = "bar") +open class EntityY { + + @get:Id + @get:NotNull + open var id: String? = null +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryX.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryX.kt new file mode 100644 index 0000000000..517e74d726 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryX.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import org.springframework.data.repository.CrudRepository + +interface RepositoryX : CrudRepository \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryY.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryY.kt new file mode 100644 index 0000000000..7783bbdac7 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryY.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import org.springframework.data.repository.CrudRepository + +interface RepositoryY : CrudRepository \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasApplication.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasApplication.kt new file mode 100644 index 0000000000..77a2129678 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasApplication.kt @@ -0,0 +1,23 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +open class SeparatedSchemasApplication { + + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(SeparatedSchemasApplication::class.java, *args) + } + } + + +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt new file mode 100644 index 0000000000..26ac3417c5 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt @@ -0,0 +1,38 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RequestMapping(path = ["/api/separatedschemas"]) +@RestController +open class SeparatedSchemasRest( + private val repositoryX: RepositoryX, + private val repositoryY: RepositoryY +) { + + @GetMapping(path = ["/x/{id}"]) + open fun getX(@PathVariable id: String) : ResponseEntity { + + val found = repositoryX.findById(id) + if(found.isPresent) { + return ResponseEntity.ok("OK") + } + + return ResponseEntity.notFound().build() + } + + @GetMapping(path = ["/y/{id}"]) + open fun getY(@PathVariable id: String) : ResponseEntity { + + val found = repositoryY.findById(id) + if(found.isPresent) { + return ResponseEntity.ok("OK") + } + + return ResponseEntity.notFound().build() + } + +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt new file mode 100644 index 0000000000..08790d2b20 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt @@ -0,0 +1,20 @@ +package org.evomaster.driver.multidb.separatedschemas + +import com.foo.rest.examples.multidb.separatedschemas.SeparatedSchemasApplication +import org.evomaster.client.java.sql.SqlScriptRunner +import org.evomaster.driver.multidb.SpringController + +class SeparatedSchemasController : SpringController(SeparatedSchemasApplication::class.java){ + + + override fun resetStateOfSUT() { + SqlScriptRunner.execCommand(connectionIfExist, """ + CREATE SCHEMA IF NOT EXISTS foo; + CREATE SCHEMA IF NOT EXISTS bar; + CREATE TABLE IF NOT EXISTS foo.EntityX; + CREATE TABLE IF NOT EXISTS bar.EntityY; + """.trimIndent()) + + super.resetStateOfSUT() + } +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt new file mode 100644 index 0000000000..baa1309fb5 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt @@ -0,0 +1,36 @@ +package org.evomaster.e2etests.spring.multidb.separatedschemas + +import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType +import org.evomaster.client.java.sql.SqlScriptRunner +import org.evomaster.core.problem.rest.HttpVerb +import org.evomaster.driver.multidb.SpringController +import org.evomaster.e2etests.spring.multidb.MultiDbParameterizedE2ETemplate +import org.junit.jupiter.api.Assertions.assertTrue +import org.evomaster.driver.multidb.separatedschemas.SeparatedSchemasController + +/** + * Created by arcuri82 on 03-Mar-20. + */ +class SeparatedSchemasEMTest : MultiDbParameterizedE2ETemplate() { + + override fun instantiateNewController(): SpringController { + return SeparatedSchemasController() + } + + override fun runEM(databaseType: DatabaseType) { + + runTestHandlingFlakyAndCompilation( + "SeparatedSchemasEM_$databaseType", + 100 + ) { args: MutableList -> + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + assertHasAtLeastOne(solution, HttpVerb.GET, 404, "/api/separatedschemas/x/{id}", null) + assertHasAtLeastOne(solution, HttpVerb.GET, 404, "/api/separatedschemas/y/{id}", null) + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/separatedschemas/x/{id}", "OK") + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/separatedschemas/y/{id}", "OK") + } + } +} \ No newline at end of file From fe202e69ca066605841e284f09ae92e928140103 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 4 Dec 2024 15:29:44 +0100 Subject: [PATCH 02/21] trying to get rid off JPA in new E2E --- .../rest/examples/multidb/base/BaseEntity.kt | 2 +- .../examples/multidb/base/BaseRepository.kt | 2 +- .../rest/examples/multidb/base/BaseRest.kt | 13 +++++++++--- .../separatedschemas/SeparatedSchemasRest.kt | 21 +++++++++++++------ .../driver/multidb/SpringController.kt | 6 ++++-- .../driver/multidb/base/BaseController.kt | 9 +++++++- .../SeparatedSchemasController.kt | 15 +++++-------- .../src/main/resources/application.yml | 10 +++++++++ .../src/main/resources/sql/base.sql | 5 +++++ .../main/resources/sql/separatedschemas.sql | 12 +++++++++++ 10 files changed, 71 insertions(+), 24 deletions(-) create mode 100644 e2e-tests/spring-rest-multidb/src/main/resources/application.yml create mode 100644 e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql create mode 100644 e2e-tests/spring-rest-multidb/src/main/resources/sql/separatedschemas.sql diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt index c0c8089187..23ca487e6f 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt @@ -4,7 +4,7 @@ import javax.persistence.Entity import javax.persistence.Id import javax.validation.constraints.NotNull -@Entity +//@Entity open class BaseEntity( @get:Id @get:NotNull diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt index 4e36e3a674..733bb2514b 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt @@ -2,4 +2,4 @@ package com.foo.rest.examples.multidb.base import org.springframework.data.repository.CrudRepository -interface BaseRepository : CrudRepository \ No newline at end of file +//interface BaseRepository : CrudRepository \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt index 3f9a38e097..c474984822 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt @@ -1,6 +1,8 @@ package com.foo.rest.examples.multidb.base +import com.foo.rest.examples.multidb.separatedschemas.EntityX import org.springframework.http.ResponseEntity +import org.springframework.jdbc.core.JdbcTemplate import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestMapping @@ -9,14 +11,19 @@ import org.springframework.web.bind.annotation.RestController @RequestMapping(path = ["/api/base"]) @RestController open class BaseRest( - private val repository: BaseRepository + //private val repository: BaseRepository + private val jdbc: JdbcTemplate ) { @GetMapping(path = ["/{id}"]) open fun get(@PathVariable id: String) : ResponseEntity { - val found = repository.findById(id) - if(found.isPresent) { + + val x = jdbc.queryForObject("SELECT * FROM foo.EntityX WHERE id = '$id'", BaseEntity::class.java) + +// val found = repository.findById(id) +// if(found.isPresent) { + if(x != null) { return ResponseEntity.ok("OK") } diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt index 26ac3417c5..f615488540 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt @@ -1,6 +1,8 @@ package com.foo.rest.examples.multidb.separatedschemas import org.springframework.http.ResponseEntity +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.jdbc.core.queryForObject import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestMapping @@ -9,15 +11,19 @@ import org.springframework.web.bind.annotation.RestController @RequestMapping(path = ["/api/separatedschemas"]) @RestController open class SeparatedSchemasRest( - private val repositoryX: RepositoryX, - private val repositoryY: RepositoryY + //private val repositoryX: RepositoryX, + //private val repositoryY: RepositoryY + private val jdbc: JdbcTemplate ) { @GetMapping(path = ["/x/{id}"]) open fun getX(@PathVariable id: String) : ResponseEntity { - val found = repositoryX.findById(id) - if(found.isPresent) { + val x = jdbc.queryForObject("SELECT * FROM foo.EntityX WHERE id = '$id'", EntityX::class.java) + + //val found = repositoryX.findById(id) +// if(found.isPresent) { + if(x!=null) { return ResponseEntity.ok("OK") } @@ -27,8 +33,11 @@ open class SeparatedSchemasRest( @GetMapping(path = ["/y/{id}"]) open fun getY(@PathVariable id: String) : ResponseEntity { - val found = repositoryY.findById(id) - if(found.isPresent) { +// val found = repositoryY.findById(id) +// if(found.isPresent) { + val y = jdbc.queryForObject("SELECT * FROM bar.EntityY WHERE id = ?", EntityY::class.java, id) + + if(y != null) { return ResponseEntity.ok("OK") } diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt index 7e2dc51fd6..dc61b5b818 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt @@ -50,6 +50,8 @@ abstract class SpringController(protected val applicationClass: Class<*>) : Embe //startSut() } + abstract fun extraSpringStartOptions() : List + override fun startSut(): String { //lot of problem if using same H2 instance. see: @@ -65,9 +67,9 @@ abstract class SpringController(protected val applicationClass: Class<*>) : Embe val commonSettings = arrayOf( "--server.port=0", "--spring.jpa.properties.hibernate.show_sql=true", - "--spring.jpa.hibernate.ddl-auto=create-drop", + "--spring.jpa.hibernate.ddl-auto=validate", "--spring.jmx.enabled=false" - ) + ).plus(extraSpringStartOptions()) ctx = when(databaseType) { DatabaseType.H2 -> { diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt index b5035f5851..bd6d808b7e 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt @@ -3,4 +3,11 @@ package org.evomaster.driver.multidb.base import com.foo.rest.examples.multidb.base.BaseApplication import org.evomaster.driver.multidb.SpringController -class BaseController : SpringController(BaseApplication::class.java) \ No newline at end of file +class BaseController : SpringController(BaseApplication::class.java){ + + override fun extraSpringStartOptions(): List { + return listOf( + "--spring.datasource.schema=classpath:/sql/base.sql" + ) + } +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt index 08790d2b20..b72314750c 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt @@ -6,15 +6,10 @@ import org.evomaster.driver.multidb.SpringController class SeparatedSchemasController : SpringController(SeparatedSchemasApplication::class.java){ - - override fun resetStateOfSUT() { - SqlScriptRunner.execCommand(connectionIfExist, """ - CREATE SCHEMA IF NOT EXISTS foo; - CREATE SCHEMA IF NOT EXISTS bar; - CREATE TABLE IF NOT EXISTS foo.EntityX; - CREATE TABLE IF NOT EXISTS bar.EntityY; - """.trimIndent()) - - super.resetStateOfSUT() + override fun extraSpringStartOptions(): List { + return listOf( + "--spring.datasource.schema=classpath:/sql/separatedschemas.sql" + ) } + } \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/resources/application.yml b/e2e-tests/spring-rest-multidb/src/main/resources/application.yml new file mode 100644 index 0000000000..2f3f7d6c13 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/resources/application.yml @@ -0,0 +1,10 @@ +spring: + jmx: + enabled: false + jpa: + show-sql: true +server: + error: + include-stacktrace: always + include-exception: true + diff --git a/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql b/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql new file mode 100644 index 0000000000..2146213bea --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS BaseEntity( + id VARCHAR(50) NOT NULL, + name VARCHAR(50) NOT NULL, + constraint pk_x primary key (id) +); \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/resources/sql/separatedschemas.sql b/e2e-tests/spring-rest-multidb/src/main/resources/sql/separatedschemas.sql new file mode 100644 index 0000000000..735f711f33 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/resources/sql/separatedschemas.sql @@ -0,0 +1,12 @@ +CREATE SCHEMA IF NOT EXISTS foo; +CREATE SCHEMA IF NOT EXISTS bar; + +CREATE TABLE IF NOT EXISTS foo.EntityX( + id VARCHAR(50) NOT NULL, + constraint pk_x primary key (id) +); + +CREATE TABLE IF NOT EXISTS bar.EntityY( + id VARCHAR(50) NOT NULL, + constraint pk_y primary key (id) +); \ No newline at end of file From 82e2f8753b2cf8f17245442b582d2c2d7b27090b Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 11:10:43 +0100 Subject: [PATCH 03/21] refactoring and cleaning --- .../controller/internal/SutController.java | 25 ++++++++++++------- .../client/java/sql/DbInfoExtractor.java | 13 +++------- ...DatabaseDtoUtils.java => SqlDtoUtils.java} | 11 ++++++-- .../sql/internal/HeuristicsCalculator.java | 2 +- .../sql/internal/QueryResultTransformer.java | 7 +++--- .../client/java/sql/internal/SqlHandler.java | 2 +- .../java/sql/internal/SqlNameContext.java | 10 ++++---- .../org/evomaster/core/sql/SqlAction.kt | 7 ++++++ .../core/sql/SqlActionTransformer.kt | 2 +- .../rest/examples/multidb/base/BaseEntity.kt | 15 ----------- .../examples/multidb/base/BaseRepository.kt | 5 ---- .../rest/examples/multidb/base/BaseRest.kt | 6 +---- .../rest/examples/multidb/base/BaseTable.kt | 9 +++++++ .../src/main/resources/sql/base.sql | 4 ++- .../MultiDbParameterizedE2ETemplate.kt | 2 +- 15 files changed, 62 insertions(+), 58 deletions(-) rename client-java/sql/src/main/java/org/evomaster/client/java/sql/{internal/SqlDatabaseDtoUtils.java => SqlDtoUtils.java} (87%) delete mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt delete mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseTable.kt diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index adf5b193b3..3df05b9e11 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -374,9 +374,11 @@ private void computeSQLHeuristics(ExtraHeuristicsDto dto, List a if (!additionalInfoList.isEmpty()) { AdditionalInfo last = additionalInfoList.get(additionalInfoList.size() - 1); last.getSqlInfoData().stream().forEach(it -> { -// String sql = it.getCommand(); try { - final SqlExecutionLogDto sqlExecutionLogDto = new SqlExecutionLogDto(it.getSqlCommand(), it.hasThrownSqlException(), it.getExecutionTime()); + SqlExecutionLogDto sqlExecutionLogDto = new SqlExecutionLogDto( + it.getSqlCommand(), + it.hasThrownSqlException(), + it.getExecutionTime()); sqlHandler.handle(sqlExecutionLogDto); } catch (Exception e) { SimpleLogger.error("FAILED TO HANDLE SQL COMMAND: " + it.getSqlCommand()); @@ -406,7 +408,6 @@ private void computeSQLHeuristics(ExtraHeuristicsDto dto, List a accessedTables.addAll(sqlExecutionsDto.deletedData); accessedTables.addAll(sqlExecutionsDto.insertedData.keySet()); //accessedTables.addAll(executionDto.queriedData.keySet()); -// accessedTables.addAll(sqlExecutionsDto.insertedData.keySet()); accessedTables.addAll(sqlExecutionsDto.updatedData.keySet()); } } @@ -495,13 +496,13 @@ public final void cleanAccessedTables(){ // clean accessed tables Set tableDataToInit = null; if (!accessedTables.isEmpty()){ - List tablesToClean = new ArrayList<>(); - getTableToClean(accessedTables, tablesToClean); + List tablesToClean = getTablesToClean(accessedTables); if (!tablesToClean.isEmpty()){ if (emDbClean.schemaNames != null && !emDbClean.schemaNames.isEmpty()){ emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType)); - }else - DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType); + } else { + DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType); + } tableDataToInit = tablesToClean.stream().filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> t.equalsIgnoreCase(a))).collect(Collectors.toSet()); } } @@ -582,7 +583,13 @@ public void addSuccessfulInitSqlInsertion(InsertionDto insertionDto){ successfulInitSqlInsertions.add(insertionDto); } - private void getTableToClean(List accessedTables, List tablesToClean){ + private List getTablesToClean(List accessedTables) { + List tablesToClean = new ArrayList<>(); + fillTablesToClean(accessedTables,tablesToClean); + return tablesToClean; + } + + private void fillTablesToClean(List accessedTables, List tablesToClean){ for (String t: accessedTables){ if (!findInCollectionIgnoreCase(t, tablesToClean).isPresent()){ if (findInMapIgnoreCase(t, fkMap).isPresent()){ @@ -591,7 +598,7 @@ private void getTableToClean(List accessedTables, List tablesToC findInCollectionIgnoreCase(t, e.getValue()).isPresent() && !findInCollectionIgnoreCase(e.getKey(), tablesToClean).isPresent()).map(Map.Entry::getKey).collect(Collectors.toList()); if (!fk.isEmpty()) - getTableToClean(fk, tablesToClean); + fillTablesToClean(fk, tablesToClean); }else { SimpleLogger.uniqueWarn("Cannot find the table "+t+" in ["+String.join(",", fkMap.keySet())+"]"); } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java index 1a78c486c4..9186fa3888 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java @@ -544,12 +544,7 @@ private static void addConstraints(DbInfoDto schemaDto, List } } - private static String getId(TableDto dto){ - if(dto.schema == null){ - return dto.name; - } - return dto.schema + "." + dto.name; - } + private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, DatabaseMetaData md, ResultSet tables, Set tableIds) throws SQLException { @@ -582,13 +577,13 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, tableDto.schema = tableSchema; tableDto.catalog = tableCatalog; - if (tableIds.contains(getId(tableDto))) { + if (tableIds.contains(SqlDtoUtils.getId(tableDto))) { /* * Perhaps we should throw a more specific exception than IllegalArgumentException */ - throw new IllegalArgumentException("Cannot handle repeated table " + getId(tableDto) + " in database"); + throw new IllegalArgumentException("Cannot handle repeated table " + SqlDtoUtils.getId(tableDto) + " in database"); } else { - tableIds.add(getId(tableDto)); + tableIds.add(SqlDtoUtils.getId(tableDto)); } Set pks = new HashSet<>(); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlDatabaseDtoUtils.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java similarity index 87% rename from client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlDatabaseDtoUtils.java rename to client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java index 9f5417aed4..0f22c237dd 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlDatabaseDtoUtils.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java @@ -1,4 +1,4 @@ -package org.evomaster.client.java.sql.internal; +package org.evomaster.client.java.sql; import org.evomaster.client.java.controller.api.dto.database.operations.InsertionDto; import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; @@ -9,9 +9,16 @@ import java.util.Set; import java.util.stream.Collectors; -public class SqlDatabaseDtoUtils { +public class SqlDtoUtils { + public static String getId(TableDto dto){ + if(dto.schema == null){ + return dto.name; + } + return dto.schema + "." + dto.name; + } + /** * * @param filter specifies which column should be returned, null means all columns should be returned diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java index 8027fbd3bf..8237cfa184 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java @@ -432,7 +432,7 @@ private Object getValue(Expression exp, DataRow data) { Column column = (Column) exp; String name = column.getColumnName(); - String table = context.getTableName(column); + String table = context.getFullyQualifiedTableName(column); return data.getValueByName(name, table); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java index 44a7c20213..47f132a18b 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java @@ -6,6 +6,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; import org.evomaster.client.java.sql.DataRow; import org.evomaster.client.java.sql.QueryResult; +import org.evomaster.client.java.sql.SqlDtoUtils; import org.evomaster.client.java.sql.VariableDescriptor; import java.time.Instant; @@ -129,7 +130,7 @@ public static List> cartesianProduct(List> values){ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto insertionDto, String tableName, Set relatedColumns, DbInfoDto dto, List existingQueryResults){ - List relatedColumnNames = SqlDatabaseDtoUtils.extractColumnNames(insertionDto, relatedColumns); + List relatedColumnNames = SqlDtoUtils.extractColumnNames(insertionDto, relatedColumns); if (!relatedColumnNames.isEmpty()){ QueryResult found = null; if (!existingQueryResults.isEmpty()) @@ -143,13 +144,13 @@ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto inserti if (foundTableSchema.isPresent()){ TableDto tableDto = foundTableSchema.get(); - List printableValue = SqlDatabaseDtoUtils.extractColumnPrintableValues(insertionDto, relatedColumns); + List printableValue = SqlDtoUtils.extractColumnPrintableValues(insertionDto, relatedColumns); assert printableValue.size() == relatedColumnNames.size(); List values = new ArrayList<>(); for (int i = 0; i < printableValue.size(); i++){ - ColumnDto columnDto = SqlDatabaseDtoUtils.extractColumnInfo(tableDto, relatedColumnNames.get(i)); + ColumnDto columnDto = SqlDtoUtils.extractColumnInfo(tableDto, relatedColumnNames.get(i)); if (columnDto == null) throw new IllegalArgumentException("Cannot find column schema of "+ relatedColumnNames.get(i) + " in Table "+ tableName); values.add(getColumnValueBasedOnPrintableValue(printableValue.get(i), columnDto)); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java index 9f79c597b1..126bfc2b06 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java @@ -361,7 +361,7 @@ public Map> extractColumnsInvolvedInWhere(Statement statemen @Override public void visit(Column column) { - String tn = context.getTableName(column); + String tn = context.getFullyQualifiedTableName(column); if (tn.equalsIgnoreCase(SqlNameContext.UNNAMED_TABLE)) { // TODO handle it properly when ll have support for sub-selects diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java index ec231749c6..5ee420f9ce 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java @@ -91,12 +91,12 @@ public boolean hasColumn(String tableName, String columnName){ * @param column a column object * @return the name of the table that this column belongs to */ - public String getTableName(Column column) { + public String getFullyQualifiedTableName(Column column) { Table table = column.getTable(); if (table != null) { - return tableAliases.getOrDefault(table.getName().toLowerCase(), table.getName().toLowerCase()); + return tableAliases.getOrDefault(table.getFullyQualifiedName().toLowerCase(), table.getFullyQualifiedName().toLowerCase()); } if(statement instanceof Select) { @@ -112,10 +112,10 @@ public String getTableName(Column column) { } } else if(statement instanceof Delete){ Delete delete = (Delete) statement; - return delete.getTable().getName().toLowerCase(); + return delete.getTable().getFullyQualifiedName().toLowerCase(); } else if(statement instanceof Update){ Update update = (Update) statement; - return update.getTable().getName().toLowerCase(); + return update.getTable().getFullyQualifiedName().toLowerCase(); }else { throw new IllegalArgumentException("Cannot handle table name for: " + statement); } @@ -131,7 +131,7 @@ private List getTableNamesInFrom() { FromItemVisitorAdapter visitor = new FromItemVisitorAdapter(){ @Override public void visit(Table table) { - names.add(table.getName().toLowerCase()); + names.add(table.getFullyQualifiedName().toLowerCase()); } }; diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt index edefe24ca3..0dffdc43dc 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt @@ -98,6 +98,13 @@ class SqlAction( } } + fun getFullQualifyingTableName() : String{ + if(openGroupName.isNullOrBlank()){ + return table.name + } + return "$openGroupName.${table.name}" + } + private val genes: List = (computedGenes ?: selectedColumns.map { SqlActionGeneBuilder().buildGene(id, table, it) } ).also { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt index f30eacfbc6..29277d4448 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt @@ -35,7 +35,7 @@ object SqlActionTransformer { } - val insertion = InsertionDto().apply { targetTable = action.table.name } + val insertion = InsertionDto().apply { targetTable = action.getFullQualifyingTableName() } for (g in action.seeTopGenes()) { if (g is SqlPrimaryKeyGene) { diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt deleted file mode 100644 index 23ca487e6f..0000000000 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.foo.rest.examples.multidb.base - -import javax.persistence.Entity -import javax.persistence.Id -import javax.validation.constraints.NotNull - -//@Entity -open class BaseEntity( - - @get:Id @get:NotNull - open var id: String? = null, - - @get:NotNull - open var name: String? = null, -) diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt deleted file mode 100644 index 733bb2514b..0000000000 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt +++ /dev/null @@ -1,5 +0,0 @@ -package com.foo.rest.examples.multidb.base - -import org.springframework.data.repository.CrudRepository - -//interface BaseRepository : CrudRepository \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt index c474984822..6f35333041 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt @@ -1,6 +1,5 @@ package com.foo.rest.examples.multidb.base -import com.foo.rest.examples.multidb.separatedschemas.EntityX import org.springframework.http.ResponseEntity import org.springframework.jdbc.core.JdbcTemplate import org.springframework.web.bind.annotation.GetMapping @@ -11,7 +10,6 @@ import org.springframework.web.bind.annotation.RestController @RequestMapping(path = ["/api/base"]) @RestController open class BaseRest( - //private val repository: BaseRepository private val jdbc: JdbcTemplate ) { @@ -19,10 +17,8 @@ open class BaseRest( open fun get(@PathVariable id: String) : ResponseEntity { - val x = jdbc.queryForObject("SELECT * FROM foo.EntityX WHERE id = '$id'", BaseEntity::class.java) + val x = jdbc.queryForObject("SELECT * FROM foo.BaseTable WHERE id = '$id'", BaseTable::class.java) -// val found = repository.findById(id) -// if(found.isPresent) { if(x != null) { return ResponseEntity.ok("OK") } diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseTable.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseTable.kt new file mode 100644 index 0000000000..0a4129fead --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseTable.kt @@ -0,0 +1,9 @@ +package com.foo.rest.examples.multidb.base + + +open class BaseTable( + + open var id: String? = null, + + open var name: String? = null, +) diff --git a/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql b/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql index 2146213bea..9542bf4655 100644 --- a/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql +++ b/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql @@ -1,4 +1,6 @@ -CREATE TABLE IF NOT EXISTS BaseEntity( +CREATE SCHEMA IF NOT EXISTS foo; + +CREATE TABLE IF NOT EXISTS foo.BaseTable( id VARCHAR(50) NOT NULL, name VARCHAR(50) NOT NULL, constraint pk_x primary key (id) diff --git a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt index b5406e5a48..da6aaf88c8 100644 --- a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt +++ b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt @@ -1,6 +1,5 @@ package org.evomaster.e2etests.spring.multidb -import org.evomaster.client.java.controller.EmbeddedSutController import org.evomaster.client.java.controller.InstrumentedSutStarter import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType import org.evomaster.client.java.instrumentation.InputProperties @@ -14,6 +13,7 @@ import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.EnumSource + abstract class MultiDbParameterizedE2ETemplate : RestTestBase(){ companion object { From 258a63942cb22a94a55086f06d81707e31be507c Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 13:06:51 +0100 Subject: [PATCH 04/21] more refactoring --- .../java/controller/api/dto/SqlDtoUtils.java | 116 ++++++++++++++++++ .../client/java/sql/DbInfoExtractor.java | 24 ++-- .../client/java/sql/SqlDtoUtils.java | 50 -------- .../sql/internal/QueryResultTransformer.java | 7 +- .../java/sql/internal/SqlNameContext.java | 7 +- .../evomaster/core/sql/SqlInsertBuilder.kt | 22 ++-- 6 files changed, 147 insertions(+), 79 deletions(-) create mode 100644 client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java delete mode 100644 client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java new file mode 100644 index 0000000000..fad84fbf8c --- /dev/null +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java @@ -0,0 +1,116 @@ +package org.evomaster.client.java.controller.api.dto; + +import org.evomaster.client.java.controller.api.dto.database.operations.InsertionDto; +import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; +import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class SqlDtoUtils { + + + /** + * Return a fully qualifying name for input table, which can be used as id. + */ + public static String getId(TableDto dto){ + if(dto.schema == null){ + return dto.name; + } + return dto.schema + "." + dto.name; + } + + /** + * @return a table DTO for a particular table name + */ + public static TableDto getTable(DbInfoDto schema, String tableName) { + return schema.tables.stream() + .filter(t -> matchByName(t, tableName)) + .findFirst().orElse(null); + } + + /** + * Check if given table dto is matched by name. + * This can be a partial name, or a full qualifying name. + */ + public static boolean matchByName(TableDto dto, String name){ + + Objects.requireNonNull(dto); + Objects.requireNonNull(name); + + if(name.isEmpty()){ + throw new IllegalArgumentException("Empty table name"); + } + + String[] tokens = name.split("\\."); + if(tokens.length > 3){ + throw new IllegalArgumentException("Invalid table name identifier. Too many '.': " + name); + } + if(tokens.length == 1){ + return dto.name.equalsIgnoreCase(tokens[0]); + } + if(tokens.length == 2){ + boolean mn = dto.name.equalsIgnoreCase(tokens[1]); + if(!mn){ + return false; + } + + if(dto.catalog == null && dto.schema == null){ + //there is no default schema, but DTO is unspecified, then false + return tokens[0].equalsIgnoreCase("public"); + } else if(dto.catalog != null && dto.schema != null){ + //both specified... so look at schema + return tokens[0].equalsIgnoreCase(dto.schema); + } else { + //only one specified, take it + if(dto.schema != null){ + return tokens[0].equalsIgnoreCase(dto.schema); + } else{ + //this can be the case for MySQL + return tokens[0].equalsIgnoreCase(dto.catalog); + } + } + } + if(tokens.length == 3){ + return tokens[0].equalsIgnoreCase(dto.catalog) + && tokens[1].equalsIgnoreCase(dto.schema) + && tokens[2].equalsIgnoreCase(dto.name); + } + + //shouldn't be reached + return false; + } + + /** + * + * @param filter specifies which column should be returned, null means all columns should be returned + * @return name of columns based on specified filter + */ + public static List extractColumnNames(InsertionDto dto, Set filter){ + return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f))).map(i-> i.variableName).collect(Collectors.toList()); + } + + /** + * + * @param filter specifies which column should be returned, null means all columns should be returned + * @return printable value of columns based on specified filter + */ + public static List extractColumnPrintableValues(InsertionDto dto, Set filter){ + return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f))).map(i-> i.printableValue).collect(Collectors.toList()); + } + + + /** + * + * @param columnName specified which ColumnDto should be returned based on its name + * @return ColumnDto based on specified columnName + */ + public static ColumnDto extractColumnInfo(TableDto dto, String columnName){ + Optional op = dto.columns.stream().filter(c-> columnName.equalsIgnoreCase(c.name)).findAny(); + return op.orElse(null); + } +} diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java index 9186fa3888..80c959ebfd 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java @@ -1,5 +1,6 @@ package org.evomaster.client.java.sql; +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils; import org.evomaster.client.java.controller.api.dto.database.schema.*; import org.evomaster.client.java.sql.internal.constraint.*; import org.evomaster.client.java.utils.SimpleLogger; @@ -42,7 +43,7 @@ private static void checkEnumeratedTypeIsDefined(DbInfoDto schema, TableDto tabl if (schema.enumeraredTypes.stream().noneMatch(k -> k.name.equals(column.type))) { throw new IllegalArgumentException("Missing enumerated type declaration for type " + column.type + " in column " + column.name - + " of table " + table.name); + + " of table " + SqlDtoUtils.getId(table)); } } } @@ -62,9 +63,8 @@ private static void checkForeignKeyToAutoIncrementMissing(DbInfoDto schema, Tabl } //TODO proper handling of multi-column PKs/FKs - Optional targetTable = schema.tables.stream() - .filter(t -> t.name.equals(fk.get().targetTable)) + .filter(t -> SqlDtoUtils.matchByName(t, fk.get().targetTable)) .findFirst(); if (!targetTable.isPresent()) { @@ -107,7 +107,7 @@ private static void checkForeignKeyToAutoIncrementPresent(DbInfoDto schema, Tabl //TODO proper handling of multi-column PKs/FKs Optional targetTable = schema.tables.stream() - .filter(t -> t.name.equals(fk.get().targetTable)) + .filter(t -> SqlDtoUtils.matchByName(t, fk.get().targetTable)) .findFirst(); if (!targetTable.isPresent()) { @@ -235,7 +235,7 @@ private static void addColumnAttributes(DbInfoDto schemaDto, List t.name.equals(tableName.toLowerCase())) + .filter(t -> SqlDtoUtils.matchByName(t,tableName)) .findFirst() .orElse(null); return tableDto.columns.stream() @@ -521,7 +521,9 @@ private static void addConstraints(Connection connection, DatabaseType dt, DbInf private static void addConstraints(DbInfoDto schemaDto, List constraintList) { for (DbTableConstraint constraint : constraintList) { String tableName = constraint.getTableName(); - TableDto tableDto = schemaDto.tables.stream().filter(t -> t.name.equalsIgnoreCase(tableName)).findFirst().orElse(null); + TableDto tableDto = schemaDto.tables.stream() + .filter(t -> SqlDtoUtils.matchByName(t,tableName)) + .findFirst().orElse(null); if (tableDto == null) { throw new NullPointerException("TableDto for table " + tableName + " was not found in the schemaDto"); @@ -808,14 +810,6 @@ private static void addForeignKeyToAutoIncrement(DbInfoDto schema) { } } - /** - * @return a table DTO for a particular table name - */ - private static TableDto getTable(DbInfoDto schema, String tableName) { - return schema.tables.stream() - .filter(t -> t.name.equalsIgnoreCase(tableName)) - .findFirst().orElse(null); - } private static ColumnDto getColumn(TableDto table, String columnName) { return table.columns.stream() @@ -861,7 +855,7 @@ private static boolean isFKToAutoIncrementColumn(DbInfoDto schema, TableDto tabl support for multi-column PKs/FKs */ int positionInFKSequence = fk.sourceColumns.indexOf(columnName); - TableDto targetTableDto = getTable(schema, fk.targetTable); + TableDto targetTableDto = SqlDtoUtils.getTable(schema, fk.targetTable); String targetColumnName = targetTableDto.primaryKeySequence.get(positionInFKSequence); ColumnDto targetColumnDto = getColumn(targetTableDto, targetColumnName); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java deleted file mode 100644 index 0f22c237dd..0000000000 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.evomaster.client.java.sql; - -import org.evomaster.client.java.controller.api.dto.database.operations.InsertionDto; -import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; -import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; - -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -public class SqlDtoUtils { - - - public static String getId(TableDto dto){ - if(dto.schema == null){ - return dto.name; - } - return dto.schema + "." + dto.name; - } - - /** - * - * @param filter specifies which column should be returned, null means all columns should be returned - * @return name of columns based on specified filter - */ - public static List extractColumnNames(InsertionDto dto, Set filter){ - return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f))).map(i-> i.variableName).collect(Collectors.toList()); - } - - /** - * - * @param filter specifies which column should be returned, null means all columns should be returned - * @return printable value of columns based on specified filter - */ - public static List extractColumnPrintableValues(InsertionDto dto, Set filter){ - return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f))).map(i-> i.printableValue).collect(Collectors.toList()); - } - - - /** - * - * @param columnName specified which ColumnDto should be returned based on its name - * @return ColumnDto based on specified columnName - */ - public static ColumnDto extractColumnInfo(TableDto dto, String columnName){ - Optional op = dto.columns.stream().filter(c-> columnName.equalsIgnoreCase(c.name)).findAny(); - return op.orElse(null); - } -} diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java index 47f132a18b..86ce31bb65 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java @@ -6,7 +6,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; import org.evomaster.client.java.sql.DataRow; import org.evomaster.client.java.sql.QueryResult; -import org.evomaster.client.java.sql.SqlDtoUtils; +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils; import org.evomaster.client.java.sql.VariableDescriptor; import java.time.Instant; @@ -140,7 +140,10 @@ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto inserti if (found == null) qr = new QueryResult(relatedColumnNames, tableName); - Optional foundTableSchema = dto.tables.stream().filter(t-> t.name.equalsIgnoreCase(tableName)).findFirst(); + Optional foundTableSchema = dto.tables.stream() + .filter(t-> SqlDtoUtils.matchByName(t,tableName)) + .findFirst(); + if (foundTableSchema.isPresent()){ TableDto tableDto = foundTableSchema.get(); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java index 5ee420f9ce..fb720ead8c 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java @@ -9,6 +9,7 @@ import net.sf.jsqlparser.statement.select.*; import net.sf.jsqlparser.statement.update.Update; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils; import java.util.*; @@ -67,8 +68,8 @@ public void setSchema(DbInfoDto schema) { * * If no schema is defined, this method returns false. */ - public boolean hasColumn(String tableName, String columnName){ - Objects.requireNonNull(tableName); + public boolean hasColumn(String fullyQualifyingTableName, String columnName){ + Objects.requireNonNull(fullyQualifyingTableName); Objects.requireNonNull(columnName); if(schema == null){ @@ -76,7 +77,7 @@ public boolean hasColumn(String tableName, String columnName){ } return this.schema.tables.stream() - .filter(t -> t.name.equalsIgnoreCase(tableName)) + .filter(t -> SqlDtoUtils.matchByName(t,fullyQualifyingTableName)) .flatMap(t -> t.columns.stream()) .filter(c -> c.name.equalsIgnoreCase(columnName)) .count() > 0; diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 8cf1a5e098..112df7c7cf 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -1,5 +1,6 @@ package org.evomaster.core.sql +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils import org.evomaster.client.java.controller.api.dto.database.operations.DataRowDto import org.evomaster.client.java.controller.api.dto.database.operations.DatabaseCommandDto import org.evomaster.client.java.controller.api.dto.database.operations.QueryResultDto @@ -96,24 +97,27 @@ class SqlInsertBuilder( val tableConstraints = parseTableConstraints(tableDto).toMutableList() val columns = generateColumnsFrom(tableDto, tableConstraints, schemaDto) - tableToConstraints[tableDto.name] = tableConstraints.toSet() - tableToColumns[tableDto.name] = columns + tableToConstraints[SqlDtoUtils.getId(tableDto)] = tableConstraints.toSet() + tableToColumns[SqlDtoUtils.getId(tableDto)] = columns } // After all columns are loaded, we can load foreign keys for (tableDto in schemaDto.tables) { - tableToForeignKeys[tableDto.name] = calculateForeignKeysFrom(tableDto, tableToColumns) + tableToForeignKeys[SqlDtoUtils.getId(tableDto)] = calculateForeignKeysFrom(tableDto, tableToColumns) } + + // Now we can create the tables for (tableDto in schemaDto.tables) { + val id = SqlDtoUtils.getId(tableDto) val table = Table( - tableDto.name, - tableToColumns[tableDto.name]!!, - tableToForeignKeys[tableDto.name]!!, - tableToConstraints[tableDto.name]!! + id, + tableToColumns[id]!!, + tableToForeignKeys[id]!!, + tableToConstraints[id]!! ) - tables[tableDto.name] = table + tables[id] = table } // Setup extended tables @@ -186,7 +190,7 @@ class SqlInsertBuilder( // val c = targetTable.find { it.name.equals(cname, ignoreCase = true) } // ?: throw IllegalArgumentException("Issue in foreign key: table ${f.targetTable} does not have a column called $cname") - val c = tableToColumns[tableDto.name]!!.find { it.name.equals(cname, ignoreCase = true) } + val c = tableToColumns[SqlDtoUtils.getId(tableDto)]!!.find { it.name.equals(cname, ignoreCase = true) } ?: throw IllegalArgumentException("Issue in foreign key: table ${tableDto.name} does not have a column called $cname") sourceColumns.add(c) From 949f2689817dac53639d42db4a2fb9e44dc96312 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 14:36:54 +0100 Subject: [PATCH 05/21] moe fixes --- .../controller/internal/SutController.java | 16 +++----- .../evomaster/client/java/sql/DbCleaner.java | 38 +++++++++++++++---- .../client/java/sql/QueryResult.java | 9 ++++- .../core/output/service/TestSuiteWriter.kt | 9 +++-- .../rest/examples/multidb/base/BaseRest.kt | 5 ++- 5 files changed, 54 insertions(+), 23 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 3df05b9e11..39d35d1ddd 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -651,12 +651,6 @@ private boolean registerInitSqlCommands(Connection connection, DbSpecification d return false; } - private void cleanDataInDbConnection(Connection connection, DbSpecification dbSpecification){ - if (dbSpecification.schemaNames != null && !dbSpecification.schemaNames.isEmpty()){ - dbSpecification.schemaNames.forEach(sch-> DbCleaner.clearDatabase(connection, sch, null, dbSpecification.dbType)); - }else - DbCleaner.clearDatabase(connection, null, dbSpecification.dbType); - } /** * Extra information about the SQL Database Schema, if any is present. @@ -1539,10 +1533,12 @@ public void resetDatabase(List tablesToClean) { if (tablesToClean.isEmpty()) return; - if (spec.schemaNames == null || spec.schemaNames.isEmpty()) - DbCleaner.clearDatabase(spec.connection, null, null, tablesToClean, spec.dbType); - else - spec.schemaNames.forEach(sp-> DbCleaner.clearDatabase(spec.connection, sp, null, tablesToClean, spec.dbType)); + DbCleaner.clearTables(spec.connection, tablesToClean, spec.dbType); + +// if (spec.schemaNames == null || spec.schemaNames.isEmpty()) +// DbCleaner.clearDatabase(spec.connection, null, null, tablesToClean, spec.dbType); +// else +// spec.schemaNames.forEach(sp-> DbCleaner.clearDatabase(spec.connection, sp, null, tablesToClean, spec.dbType)); try { handleInitSqlInDbClean(tablesToClean, spec); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java index 433eacbb53..a5f4e0b838 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java @@ -8,9 +8,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @@ -120,6 +118,36 @@ private static void clearDatabase(int retries, } } + public static void clearTables(Connection connection, List tablesToClean, DatabaseType type) { + + String defaultSchema = getDefaultSchema(type); + // schema -> table_base_name + Map> schemaToNames = new HashMap<>(); + + for(String t : tablesToClean) { + if(! t.contains(".")){ + Set x = schemaToNames.getOrDefault(defaultSchema, new HashSet<>()); + x.add(t); + } else { + String[] tokens = t.split("\\."); + String name = tokens[tokens.length - 1]; + String schema = tokens[tokens.length - 2]; + Set x = schemaToNames.getOrDefault(schema, new HashSet<>()); + x.add(name); + } + } + + for(Map.Entry> e : schemaToNames.entrySet()) { + List names = new ArrayList<>(e.getValue()); + clearDatabase(connection, e.getKey(), null, names, type, true); + } + } + + + public static void clearDatabase(Connection connection, List tableToSkip, List tableToClean, DatabaseType type) { + clearDatabase(connection, tableToSkip, tableToClean, type, true); + } + public static void clearDatabase(Connection connection, List tablesToSkip, DatabaseType type, boolean doResetSequence) { clearDatabase(connection, getDefaultSchema(type), tablesToSkip, type, doResetSequence); } @@ -128,10 +156,6 @@ public static void clearDatabase(Connection connection, List tablesToSki clearDatabase(connection, tablesToSkip, type, true); } - public static void clearDatabase(Connection connection, List tableToSkip, List tableToClean, DatabaseType type) { - clearDatabase(connection, tableToSkip, tableToClean, type, true); - } - public static void clearDatabase(Connection connection, List tableToSkip, List tableToClean, DatabaseType type, boolean doResetSequence) { clearDatabase(connection, getDefaultSchema(type), tableToSkip, tableToClean, type, doResetSequence); } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResult.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResult.java index ad11f36de8..c900b476d7 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResult.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResult.java @@ -68,10 +68,17 @@ public QueryResult(ResultSet resultSet) { for (int i = 0; i < md.getColumnCount(); i++) { int index = i + 1; + String schema = md.getSchemaName(index); + String name; + if (schema != null) { + name = schema + "." + md.getTableName(index); + } else { + name = md.getTableName(index); + } VariableDescriptor desc = new VariableDescriptor( getColumnName(md, index), md.getColumnLabel(index), - md.getTableName(index) + name ); variableDescriptors.add(desc); } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 5e30dc444b..4753b28ae1 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -1,6 +1,7 @@ package org.evomaster.core.output.service import com.google.inject.Inject +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils import org.evomaster.client.java.controller.api.dto.database.operations.InsertionDto import org.evomaster.client.java.controller.api.dto.database.operations.MongoInsertionDto import org.evomaster.client.java.instrumentation.shared.ExternalServiceSharedUtils @@ -207,9 +208,11 @@ class TestSuiteWriter { } val all = sampler.extractFkTables(accessedTable) - //if (all.isEmpty()) return "null" - - val tableNamesInSchema = remoteController.getCachedSutInfo()?.sqlSchemaDto?.tables?.map { it.name }?.toSet() + val tableNamesInSchema = remoteController.getCachedSutInfo() + ?.sqlSchemaDto + ?.tables + ?.map { SqlDtoUtils.getId(it) } + ?.toSet() ?: setOf() val missingTables = all.filter { x -> tableNamesInSchema.none { y -> y.equals(x,true) } }.sorted() diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt index 6f35333041..f2298d4971 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt @@ -1,6 +1,7 @@ package com.foo.rest.examples.multidb.base import org.springframework.http.ResponseEntity +import org.springframework.jdbc.core.BeanPropertyRowMapper import org.springframework.jdbc.core.JdbcTemplate import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable @@ -17,9 +18,9 @@ open class BaseRest( open fun get(@PathVariable id: String) : ResponseEntity { - val x = jdbc.queryForObject("SELECT * FROM foo.BaseTable WHERE id = '$id'", BaseTable::class.java) + val x = jdbc.query("SELECT * FROM foo.BaseTable WHERE id = '$id'", BeanPropertyRowMapper(BaseTable::class.java)) - if(x != null) { + if(x.isNotEmpty()) { return ResponseEntity.ok("OK") } From effb4c0b9eaa0b63e752f1dc996739c3a1800156 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 14:46:46 +0100 Subject: [PATCH 06/21] refactoring package location --- .../client/java/sql/{ => cleaner}/DbCleanerH2Test.java | 4 +++- .../java/sql/{ => cleaner}/DbCleanerMariaDBTest.java | 3 ++- .../client/java/sql/{ => cleaner}/DbCleanerMySQLTest.java | 3 ++- .../java/sql/{ => cleaner}/DbCleanerPostgresTest.java | 4 +++- .../java/sql/{ => cleaner}/DbCleanerSQLServerTest.java | 8 +++++++- .../{ => cleaner}/DbCleanerSQLServerWithSchemaTest.java | 6 +++++- .../client/java/sql/{ => cleaner}/DbCleanerTestBase.java | 2 +- 7 files changed, 23 insertions(+), 7 deletions(-) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerH2Test.java (89%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerMariaDBTest.java (95%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerMySQLTest.java (95%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerPostgresTest.java (93%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerSQLServerTest.java (93%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerSQLServerWithSchemaTest.java (95%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerTestBase.java (99%) diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerH2Test.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java similarity index 89% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerH2Test.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java index 0f45cf1f73..86d331d8e4 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerH2Test.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java @@ -1,6 +1,8 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; +import org.evomaster.client.java.sql.DbCleaner; +import org.evomaster.client.java.sql.SqlScriptRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMariaDBTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java similarity index 95% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMariaDBTest.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java index c1155071e3..9a046be9a3 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMariaDBTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java @@ -1,6 +1,7 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; +import org.evomaster.client.java.sql.DbCleaner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMySQLTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java similarity index 95% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMySQLTest.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java index b64ab9a78b..6e63975ff9 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMySQLTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java @@ -1,7 +1,8 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; +import org.evomaster.client.java.sql.DbCleaner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerPostgresTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java similarity index 93% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerPostgresTest.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java index 0737dd04e5..2cb69332a7 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerPostgresTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java @@ -1,6 +1,8 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; +import org.evomaster.client.java.sql.DbCleaner; +import org.evomaster.client.java.sql.SqlScriptRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java similarity index 93% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerTest.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java index c32b1cb8d4..a452a91600 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java @@ -1,12 +1,15 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; //TODO //import org.evomaster.e2etests.utils.CIUtils; import org.evomaster.ci.utils.CIUtils; +import org.evomaster.client.java.sql.DbCleaner; +import org.evomaster.client.java.sql.SqlScriptRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; @@ -17,6 +20,9 @@ import java.util.HashMap; import java.util.List; +//No longer support MSSQL +@Deprecated +@Disabled public class DbCleanerSQLServerTest extends DbCleanerTestBase{ private static final int PORT = 1433; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerWithSchemaTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java similarity index 95% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerWithSchemaTest.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java index e9e40b8d69..0dd838c7d0 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerWithSchemaTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java @@ -1,10 +1,11 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.ci.utils.CIUtils; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; import org.evomaster.client.java.sql.DbCleaner; import org.evomaster.client.java.sql.QueryResult; import org.evomaster.client.java.sql.SqlScriptRunner; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; @@ -16,6 +17,9 @@ import static org.junit.jupiter.api.Assertions.*; +//No longer support MSSQL +@Deprecated +@Disabled public class DbCleanerSQLServerWithSchemaTest { private static final int PORT = 1433; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerTestBase.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerTestBase.java similarity index 99% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerTestBase.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerTestBase.java index bfeafb0983..b819fb32e6 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerTestBase.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerTestBase.java @@ -1,4 +1,4 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; import org.evomaster.client.java.sql.QueryResult; From 506e4174478632313e37dab2c552c370b0bbc149 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 15:54:18 +0100 Subject: [PATCH 07/21] more cleaning and refactoring for DbCleaner --- .../controller/internal/SutController.java | 6 +- .../sql/h2/DatabaseFakeH2SutController.java | 2 +- .../mysql/DatabaseFakeMySQLSutController.java | 2 +- .../evomaster/client/java/sql/DbCleaner.java | 195 ++++++++---------- .../java/sql/cleaner/DbCleanerH2Test.java | 2 +- .../sql/cleaner/DbCleanerMariaDBTest.java | 2 +- .../java/sql/cleaner/DbCleanerMySQLTest.java | 2 +- .../sql/cleaner/DbCleanerPostgresTest.java | 2 +- .../sql/cleaner/DbCleanerSQLServerTest.java | 2 +- .../DbCleanerSQLServerWithSchemaTest.java | 4 +- .../spring/db/auth/DbAuthController.java | 2 +- .../MultiDbParameterizedE2ETemplate.kt | 3 + .../spring/db/auth/DbAuthController.java | 2 +- 13 files changed, 103 insertions(+), 123 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 39d35d1ddd..4924466a7e 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -499,9 +499,9 @@ public final void cleanAccessedTables(){ List tablesToClean = getTablesToClean(accessedTables); if (!tablesToClean.isEmpty()){ if (emDbClean.schemaNames != null && !emDbClean.schemaNames.isEmpty()){ - emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType)); + emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType, true)); } else { - DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType); + DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType, true); } tableDataToInit = tablesToClean.stream().filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> t.equalsIgnoreCase(a))).collect(Collectors.toSet()); } @@ -1522,7 +1522,7 @@ public void resetDatabase(List tablesToClean) { if(tablesToClean == null){ // all data will be reset - DbCleaner.clearDatabase(spec.connection, null, null, null, spec.dbType); + DbCleaner.clearDatabase(spec.connection, null, null, null, spec.dbType, true); try { reAddAllInitSql(); } catch (SQLException e) { diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/DatabaseFakeH2SutController.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/DatabaseFakeH2SutController.java index ea0b417ace..400b41de2d 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/DatabaseFakeH2SutController.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/DatabaseFakeH2SutController.java @@ -52,7 +52,7 @@ public SutInfoDto.OutputFormat getPreferredOutputFormat() { @Override public String startSut() { running = true; - DbCleaner.clearDatabase(sqlConnection, null, DatabaseType.H2); + DbCleaner.clearDatabase_H2(sqlConnection, null, null); return "foo"; } diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/DatabaseFakeMySQLSutController.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/DatabaseFakeMySQLSutController.java index 1f47e54271..33a149ca7c 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/DatabaseFakeMySQLSutController.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/DatabaseFakeMySQLSutController.java @@ -55,7 +55,7 @@ public SutInfoDto.OutputFormat getPreferredOutputFormat() { public String startSut() { running = true; - DbCleaner.clearDatabase(sqlConnection, DB_NAME,null, DatabaseType.MYSQL); + DbCleaner.clearDatabase_MySQL(sqlConnection, DB_NAME,null, null); return "foo"; } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java index a5f4e0b838..007233ee34 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java @@ -17,19 +17,38 @@ */ public class DbCleaner { - public static void clearDatabase_H2(Connection connection) { - clearDatabase_H2(connection, null); - } - public static void clearDatabase_H2(Connection connection, List tableToSkip) { - clearDatabase_H2(connection, getDefaultSchema(DatabaseType.H2), tableToSkip); + public static void clearTables(Connection connection, List fullyQualifyingTableNames, DatabaseType type) { + + String defaultSchema = getDefaultSchema(type); + // schema -> table_base_name + Map> schemaToNames = new HashMap<>(); + + for(String t : fullyQualifyingTableNames) { + if(! t.contains(".")){ + Set x = schemaToNames.getOrDefault(defaultSchema, new HashSet<>()); + x.add(t); + } else { + String[] tokens = t.split("\\."); + String name = tokens[tokens.length - 1]; + String schema = tokens[tokens.length - 2]; + Set x = schemaToNames.getOrDefault(schema, new HashSet<>()); + x.add(name); + } + } + + for(Map.Entry> e : schemaToNames.entrySet()) { + List names = new ArrayList<>(e.getValue()); + clearDatabase(connection, e.getKey(), null, names, type, true); + } } - public static void clearDatabase_H2(Connection connection, String schemaName, List tableToSkip) { - clearDatabase_H2(connection, schemaName, tableToSkip, null); + public static void clearDatabase_Postgres(Connection connection, String schemaName, List tableToSkip, List tableToClean) { + clearDatabase(getDefaultRetries(DatabaseType.POSTGRES), connection, getSchemaName(schemaName, DatabaseType.POSTGRES), tableToSkip, tableToClean, DatabaseType.POSTGRES, false, true, false); } - public static void clearDatabase_H2(Connection connection, String schemaName, List tableToSkip, List tableToClean) { + public static void clearDatabase_H2(Connection connection, List tableToSkip, List tableToClean) { + String schemaName = getDefaultSchema(DatabaseType.H2); final String h2Version; try { h2Version = H2VersionUtils.getH2Version(connection); @@ -40,14 +59,63 @@ public static void clearDatabase_H2(Connection connection, String schemaName, Li * The SQL command "TRUNCATE TABLE my_table RESTART IDENTITY" * is not supported by H2 version 1.4.199 or lower */ - final boolean restartIdentitiyWhenTruncating = H2VersionUtils.isVersionGreaterOrEqual(h2Version, H2VersionUtils.H2_VERSION_2_0_0); + boolean restartIdentityWhenTruncating = H2VersionUtils.isVersionGreaterOrEqual(h2Version, H2VersionUtils.H2_VERSION_2_0_0); clearDatabase(getDefaultRetries(DatabaseType.H2), connection, schemaName, tableToSkip, tableToClean, DatabaseType.H2, - false, true, restartIdentitiyWhenTruncating); + false, true, restartIdentityWhenTruncating); + } + + + public static void clearDatabase_MySQL(Connection connection, String schemaName, List tablesToSkip, List tableToClean) { + clearDatabase(connection, getSchemaName(schemaName, DatabaseType.MYSQL), tablesToSkip, tableToClean, DatabaseType.MYSQL, true); } + //TODO why this function? + public static void dropDatabaseTables(Connection connection, String schemaName, List tablesToSkip, DatabaseType type) { + if (type != DatabaseType.MYSQL && type != DatabaseType.MARIADB) + throw new IllegalArgumentException("Dropping tables are not supported by " + type); + clearDatabase(getDefaultRetries(type), connection, getSchemaName(schemaName, type), tablesToSkip, null, type, true, true, false); + } + + + public static void clearDatabase( + Connection connection, + String schemaName, + List tableToSkip, + List tableToClean, + DatabaseType type, + boolean doResetSequence + ) { + /* + * Enable the restarting of Identity fields only if sequences are to be restarted + * and the database type is H2 + */ + boolean restartIdentityWhenTruncating; + if (doResetSequence && type.equals(DatabaseType.H2)) { + try { + String h2Version = H2VersionUtils.getH2Version(connection); + restartIdentityWhenTruncating = H2VersionUtils.isVersionGreaterOrEqual(h2Version, H2VersionUtils.H2_VERSION_2_0_0); + } catch (SQLException ex) { + throw new RuntimeException("Unexpected SQL exception while getting H2 version", ex); + } + } else { + restartIdentityWhenTruncating = false; + } + clearDatabase(getDefaultRetries(type), connection, getSchemaName(schemaName, type), tableToSkip, tableToClean, type, false, doResetSequence, restartIdentityWhenTruncating); + } + + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + + private static String getSchemaName(String schemaName, DatabaseType type) { + if (schemaName != null) return schemaName; + return getDefaultSchema(type); + } + + /* - [non-determinism-source] Man: retries might lead to non-determinate logs - */ + [non-determinism-source] Man: retries might lead to non-determinate logs + */ private static void clearDatabase(int retries, Connection connection, String schemaName, @@ -118,98 +186,6 @@ private static void clearDatabase(int retries, } } - public static void clearTables(Connection connection, List tablesToClean, DatabaseType type) { - - String defaultSchema = getDefaultSchema(type); - // schema -> table_base_name - Map> schemaToNames = new HashMap<>(); - - for(String t : tablesToClean) { - if(! t.contains(".")){ - Set x = schemaToNames.getOrDefault(defaultSchema, new HashSet<>()); - x.add(t); - } else { - String[] tokens = t.split("\\."); - String name = tokens[tokens.length - 1]; - String schema = tokens[tokens.length - 2]; - Set x = schemaToNames.getOrDefault(schema, new HashSet<>()); - x.add(name); - } - } - - for(Map.Entry> e : schemaToNames.entrySet()) { - List names = new ArrayList<>(e.getValue()); - clearDatabase(connection, e.getKey(), null, names, type, true); - } - } - - - public static void clearDatabase(Connection connection, List tableToSkip, List tableToClean, DatabaseType type) { - clearDatabase(connection, tableToSkip, tableToClean, type, true); - } - - public static void clearDatabase(Connection connection, List tablesToSkip, DatabaseType type, boolean doResetSequence) { - clearDatabase(connection, getDefaultSchema(type), tablesToSkip, type, doResetSequence); - } - - public static void clearDatabase(Connection connection, List tablesToSkip, DatabaseType type) { - clearDatabase(connection, tablesToSkip, type, true); - } - - public static void clearDatabase(Connection connection, List tableToSkip, List tableToClean, DatabaseType type, boolean doResetSequence) { - clearDatabase(connection, getDefaultSchema(type), tableToSkip, tableToClean, type, doResetSequence); - } - - public static void clearDatabase(Connection connection, String schemaName, List tablesToSkip, DatabaseType type) { - clearDatabase(connection, getSchemaName(schemaName, type), tablesToSkip, type, true); - } - - public static void clearDatabase(Connection connection, String schemaName, List tablesToSkip, DatabaseType type, boolean doResetSequence) { - clearDatabase(connection, getSchemaName(schemaName, type), tablesToSkip, null, type, doResetSequence); - } - - public static void clearDatabase(Connection connection, String schemaName, List tableToSkip, List tableToClean, DatabaseType type) { - clearDatabase(connection, getSchemaName(schemaName, type), tableToSkip, tableToClean, type, true); - } - - public static void clearDatabase(Connection connection, String schemaName, List tableToSkip, List tableToClean, DatabaseType type, boolean doResetSequence) { - /* - * Enable the restarting of Identity fields only if sequences are to be restarted - * and the database type is H2 - */ - boolean restartIdentityWhenTruncating; - if (doResetSequence && type.equals(DatabaseType.H2)) { - try { - String h2Version = H2VersionUtils.getH2Version(connection); - restartIdentityWhenTruncating = H2VersionUtils.isVersionGreaterOrEqual(h2Version, H2VersionUtils.H2_VERSION_2_0_0); - } catch (SQLException ex) { - throw new RuntimeException("Unexpected SQL exception while getting H2 version", ex); - } - } else { - restartIdentityWhenTruncating = false; - } - clearDatabase(getDefaultRetries(type), connection, getSchemaName(schemaName, type), tableToSkip, tableToClean, type, false, doResetSequence, restartIdentityWhenTruncating); - } - - public static void dropDatabaseTables(Connection connection, String schemaName, List tablesToSkip, DatabaseType type) { - if (type != DatabaseType.MYSQL && type != DatabaseType.MARIADB) - throw new IllegalArgumentException("Dropping tables are not supported by " + type); - clearDatabase(getDefaultRetries(type), connection, getSchemaName(schemaName, type), tablesToSkip, null, type, true, true, false); - } - - - public static void clearDatabase_Postgres(Connection connection, String schemaName, List tablesToSkip) { - clearDatabase_Postgres(connection, getSchemaName(schemaName, DatabaseType.POSTGRES), tablesToSkip, null); - } - - public static void clearDatabase_Postgres(Connection connection, String schemaName, List tableToSkip, List tableToClean) { - clearDatabase(getDefaultRetries(DatabaseType.POSTGRES), connection, getSchemaName(schemaName, DatabaseType.POSTGRES), tableToSkip, tableToClean, DatabaseType.POSTGRES, false, true, false); - } - - private static String getSchemaName(String schemaName, DatabaseType type) { - if (schemaName != null) return schemaName; - return getDefaultSchema(type); - } /** * @param tableToSkip are tables to skip @@ -228,9 +204,12 @@ private static List cleanDataInTables(List tableToSkip, String schema, boolean singleCommand, boolean doDropTable, - boolean restartIdentityWhenTruncating) throws SQLException { - if (tableToSkip != null && (!tableToSkip.isEmpty()) && tableToClean != null && (!tableToClean.isEmpty())) + boolean restartIdentityWhenTruncating + ) throws SQLException { + + if (tableToSkip != null && (!tableToSkip.isEmpty()) && tableToClean != null && (!tableToClean.isEmpty())) { throw new IllegalArgumentException("tableToSkip and tableToClean cannot be configured at the same time."); + } // Find all tables and truncate them Set tables = new HashSet<>(); @@ -331,7 +310,7 @@ private static void deleteTables(Statement statement, String table, String schem if (!schema.isEmpty() && !schema.equals(getDefaultSchema(DatabaseType.MS_SQL_SERVER))) tableWithSchema = schema + "." + schema; statement.executeUpdate("DELETE FROM " + tableWithSchema); -// NOTE TAHT ideally we should reseed identify here, but there would case an issue, i.e., does not contain an identity column +// NOTE THAT ideally we should reseed identify here, but there would case an issue, i.e., does not contain an identity column if (tableHasIdentify.contains(table)) statement.executeUpdate("DBCC CHECKIDENT ('" + tableWithSchema + "', RESEED, 0)"); } @@ -468,8 +447,6 @@ private static String getAllTableCommand(DatabaseType type, String schema) { return command + " AND TABLE_SCHEMA='" + schema + "'"; } throw new DbUnsupportedException(type); -// case MS_SQL_SERVER_2000: -// return "SELECT sobjects.name FROM sysobjects sobjects WHERE sobjects.xtype = '"+schema+"'"; // shcema can be 'U' } private static String getAllSequenceCommand(DatabaseType type, String schemaName) { diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java index 86d331d8e4..05486fd36f 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java @@ -44,7 +44,7 @@ protected Connection getConnection() { @Override protected void clearDatabase(List tablesToSkip, List tableToClean) { - DbCleaner.clearDatabase_H2(connection, "PUBLIC", tablesToSkip, tableToClean); + DbCleaner.clearDatabase_H2(connection, tablesToSkip, tableToClean); } @Override diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java index 9a046be9a3..d4da1809bc 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java @@ -59,7 +59,7 @@ protected Connection getConnection() { @Override protected void clearDatabase(List tablesToSkip, List tableToClean) { - DbCleaner.clearDatabase(connection, DB_NAME, tablesToSkip, tableToClean, getDbType()); + DbCleaner.clearDatabase(connection, DB_NAME, tablesToSkip, tableToClean, getDbType(), true); } @Override diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java index 6e63975ff9..135d271116 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java @@ -65,7 +65,7 @@ protected Connection getConnection() { @Override protected void clearDatabase(List tablesToSkip, List tableToClean) { - DbCleaner.clearDatabase(connection, DB_NAME, tablesToSkip, tableToClean, DatabaseType.MYSQL); + DbCleaner.clearDatabase(connection, DB_NAME, tablesToSkip, tableToClean, DatabaseType.MYSQL, true); } diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java index 2cb69332a7..866741ec7d 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java @@ -38,7 +38,7 @@ public static void initClass() throws Exception{ } @AfterAll - private static void afterClass() throws Exception{ + public static void afterClass() throws Exception{ connection.close(); postgres.stop(); } diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java index a452a91600..7ae17be264 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java @@ -94,7 +94,7 @@ protected Connection getConnection() { @Override protected void clearDatabase(List tablesToSkip, List tableToClean) { - DbCleaner.clearDatabase(connection, tablesToSkip, tableToClean, getDbType()); + //DbCleaner.clearDatabase(connection, tablesToSkip, tableToClean, getDbType()); } @Override diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java index 0dd838c7d0..e84ea99ac7 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java @@ -63,10 +63,10 @@ public void testSkipTableMisconfigured() throws Exception{ assertThrows(RuntimeException.class, ()-> - DbCleaner.clearDatabase(connection, "", null, null, DatabaseType.MS_SQL_SERVER) + DbCleaner.clearDatabase(connection, "", null, null, DatabaseType.MS_SQL_SERVER, true) ); - DbCleaner.clearDatabase(connection, "Foo", null, null, DatabaseType.MS_SQL_SERVER); + DbCleaner.clearDatabase(connection, "Foo", null, null, DatabaseType.MS_SQL_SERVER, true); res = SqlScriptRunner.execCommand(connection, "SELECT * FROM Foo.Foo;"); assertEquals(0, res.seeRows().size()); } diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java b/e2e-tests/spring-rest-h2-v1/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java index d38c8619ae..706cd6b268 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java @@ -29,7 +29,7 @@ public List getInfoForAuthentication() { @Override public void resetStateOfSUT() { - DbCleaner.clearDatabase_H2(sqlConnection); + DbCleaner.clearDatabase_H2(sqlConnection,null,null); try { SqlScriptRunner.execInsert(sqlConnection, diff --git a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt index da6aaf88c8..7ab653833f 100644 --- a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt +++ b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt @@ -10,10 +10,13 @@ import org.evomaster.e2etests.utils.EnterpriseTestBase import org.evomaster.e2etests.utils.RestTestBase import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Disabled import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.EnumSource +//FIXME put back once fixed +@Disabled abstract class MultiDbParameterizedE2ETemplate : RestTestBase(){ companion object { diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java index c34fbb21dc..cf05930fa7 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java @@ -29,7 +29,7 @@ public List getInfoForAuthentication() { @Override public void resetStateOfSUT() { - DbCleaner.clearDatabase_H2(sqlConnection); + DbCleaner.clearDatabase_H2(sqlConnection, null, null); try { SqlScriptRunner.execInsert(sqlConnection, From fa9fee10136e4139cd56962643607f5dfeac7743 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 22:22:42 +0100 Subject: [PATCH 08/21] fix for failing test --- .../evomaster/client/java/sql/DataRow.java | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java index 991d2c16f4..541940a4c5 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java @@ -98,23 +98,41 @@ public Object getValueByName(String name, String table) { } } + List candidates = new ArrayList<>(); + //if none, then check column names for (int i = 0; i < variableDescriptors.size(); i++) { VariableDescriptor desc = variableDescriptors.get(i); - if (n.equalsIgnoreCase(desc.getColumnName()) && - (t == null || t.isEmpty() - || t.equalsIgnoreCase(desc.getTableName()) - /* - TODO: this does not cover all possible cases, as in theory - there can be many unnamed tables (eg results of sub-selects) - with same column names. At this moment, we would not - be able to distinguish them - */ - || t.equalsIgnoreCase(SqlNameContext.UNNAMED_TABLE) - ) - ) { + + if (!n.equalsIgnoreCase(desc.getColumnName())){ + continue; + } + //no defined table, or exact match + if(t == null || t.isEmpty() || t.equalsIgnoreCase(desc.getTableName())){ return getValue(i); } + /* + TODO: this does not cover all possible cases, as in theory + there can be many unnamed tables (eg results of sub-selects) + with same column names. At this moment, we would not + be able to distinguish them + */ + if(t.equalsIgnoreCase(SqlNameContext.UNNAMED_TABLE)){ + candidates.add(i); + } + /* + We just specified the name without schema... if unique, we would be fine + */ + if(!t.contains(".") && desc.getTableName().toLowerCase().endsWith("."+t.toLowerCase())){ + candidates.add(i); + } + } + if(candidates.size() > 1){ + SimpleLogger.uniqueWarn("More than one table candidate for: " + t); + } + + if(candidates.size() >= 1){ + return getValue(candidates.get(0)); } throw new IllegalArgumentException("No variable called '" + name + "' for table '" + table + "'"); From 5c5a4e40d151ccfc3c75264c66634200a3e4843c Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 17 Dec 2024 13:28:07 +0100 Subject: [PATCH 09/21] more fixes related to schema in table ids --- .../evomaster/core/sql/SqlInsertBuilder.kt | 66 +++++++++++++++---- .../core/sql/SqlInsertBuilderTest.kt | 6 +- .../sql/extract/h2/CatwatchSqlExtractTest.kt | 4 +- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 112df7c7cf..701dfcdd2c 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -180,9 +180,11 @@ class SqlInsertBuilder( for (fk in tableDto.foreignKeys) { - tableToColumns[fk.targetTable] - ?: throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}") + val tableKey = getTableKey(tableToColumns.keys, fk.targetTable) + if(tableKey == null || tableToColumns[tableKey] == null) { + throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}") + } val sourceColumns = mutableSetOf() for (cname in fk.sourceColumns) { @@ -196,7 +198,7 @@ class SqlInsertBuilder( sourceColumns.add(c) } - fks.add(ForeignKey(sourceColumns, fk.targetTable)) + fks.add(ForeignKey(sourceColumns, tableKey)) } return fks } @@ -616,19 +618,52 @@ class SqlInsertBuilder( /** * SQL is not case sensitivity. * Therefore, table/column must ignore case sensitivity. + * + * No. This is not strictly true: + * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL. + * + * quotes "" can be used to force case-sensitivity */ - fun isTable(tableName: String) = tables.keys.any { it.equals(tableName, ignoreCase = true) } + fun isTable(tableName: String) = getTableKey(tables.keys, tableName) != null - fun getTable(tableName: String, useExtraConstraints: Boolean): Table { + private fun getValueByTableNameKey(map: Map, tableName: String) : T?{ - val data = if (useExtraConstraints) extendedTables else tables + val key = getTableKey(map.keys, tableName) + return map[key] + } + private fun getTableKey(keys: Set, tableName: String) : String?{ /* * SQL is not case sensitivity, table/column must ignore case sensitivity. + * No, this is not really true... + * Usually, names are lowered-cased by the DB, unless quoted in "": + * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL. + * */ - val tableNameKey = data.keys.find { tableName.equals(it, ignoreCase = true) } - return data[tableNameKey] ?: throw IllegalArgumentException("No table called $tableName") + val tableNameKey = keys.find { tableName.equals(it, ignoreCase = true) } + if (!tableName.contains(".") && tableNameKey == null){ + //input name might be without schema, so check for partial match + val candidates = keys.filter { it.endsWith(".${tableName}", true) } + if(candidates.size > 1){ + throw IllegalArgumentException("Ambiguity." + + " More than one candidate of table called $tableName." + + " Values: ${candidates.joinToString(", ")}") + } + if(candidates.size == 1){ + return candidates[0] + } + } + return tableNameKey + } + + fun getTable(tableName: String, useExtraConstraints: Boolean): Table { + + val data = if (useExtraConstraints) extendedTables else tables + + val tableNameKey = getTableKey(data.keys, tableName) + ?: throw IllegalArgumentException("No table called $tableName") + return data[tableNameKey] ?: throw IllegalArgumentException("No table called $tableName") } /** @@ -1009,11 +1044,18 @@ class SqlInsertBuilder( private fun formatNameInSql(name: String): String { + return name + + /* + The handling of "" quotes is extremely tricky. + TODO if we really want to support it, would need to have a flag on each name (eg have + to pass them as pojo, not string) + */ //TODO: why this??? needs explanation - return when { - databaseType == DatabaseType.MYSQL || name == SQLKey.ALL.key -> name - else -> "\"$name\"" - } +// return when { +// databaseType == DatabaseType.MYSQL || name == SQLKey.ALL.key -> name +// else -> "\"$name\"" +// } } /** diff --git a/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt index 2bbd4c78d0..ed40fa4094 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt @@ -1067,9 +1067,9 @@ class SqlInsertBuilderTest { // ABCD or ACBD assertEquals(4, enabled.size) - assertEquals("ROOTA", enabled[0].table.name) - assertEquals("LEAFD", enabled[3].table.name) - assertTrue(enabled.subList(1,3).map { it.table.name }.containsAll(listOf("NODEB", "NODEC"))) + assertEquals("PUBLIC.ROOTA", enabled[0].table.name) + assertEquals("PUBLIC.LEAFD", enabled[3].table.name) + assertTrue(enabled.subList(1,3).map { it.table.name }.containsAll(listOf("PUBLIC.NODEB", "PUBLIC.NODEC"))) } ////// Tests Numeric parsing for Min/Max Bounds diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt index d217f57dae..c0ebd13d7c 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt @@ -59,8 +59,8 @@ class CatwatchSqlExtractTest : ExtractTestBaseH2(){ SqlActionUtils.randomizeDbActionGenes(insertions.toMutableList(), Randomness()) assertEquals(2, insertions.size) - assert(insertions[0].table.name.equals("PROJECT", ignoreCase = true)) - assert(insertions[1].table.name.equals("lANGUAGE_LIST", ignoreCase = true)) + assert(insertions[0].table.name.equals("PUBLIC.PROJECT", ignoreCase = true)) + assert(insertions[1].table.name.equals("PUBLIC.lANGUAGE_LIST", ignoreCase = true)) val projectId = (insertions[0].seeTopGenes().filterIsInstance()).first().uniqueId From a572f059cb9ca1e291dcdca7515abb4b9f4f66b7 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 18 Dec 2024 10:54:25 +0100 Subject: [PATCH 10/21] more cleaning and fixing --- .../problem/rest/resource/ResourceCluster.kt | 10 +- .../dependency/ResourceRelatedToTable.kt | 101 ++++++++++++------ .../org/evomaster/core/sql/SqlActionUtils.kt | 52 ++++++++- .../evomaster/core/sql/SqlInsertBuilder.kt | 32 +----- .../org/evomaster/core/sql/schema/Table.kt | 3 + .../rest/resource/ResourceNodeWithDbTest.kt | 4 +- 6 files changed, 137 insertions(+), 65 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt index 5043156bc2..a2bc2bfe3a 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt @@ -127,16 +127,16 @@ class ResourceCluster { * @return existing rows of the table based on the specified [tableName] */ fun getDataInDb(tableName: String) : MutableList?{ - val found = dataInDB.filterKeys { k-> k.equals(tableName, ignoreCase = true) }.keys - if (found.isEmpty()) return null - Lazy.assert{found.size == 1} - return dataInDB.getValue(found.first()) + val key = SqlActionUtils.getTableKey(dataInDB.keys, tableName) + ?: return null + return dataInDB.getValue(key) } /** * @return table class based on specified [name] */ - fun getTableByName(name : String) = tables.keys.find { it.equals(name, ignoreCase = true) }?.run { tables[this] } + private fun getTableByName(name : String) = + SqlActionUtils.getTableKey(tables.keys, name)?.run { tables[this] } /** diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt index 579494cd37..bc9d513124 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt @@ -5,6 +5,7 @@ import org.evomaster.core.sql.SQLKey import org.evomaster.core.problem.rest.param.BodyParam import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.util.inference.model.MatchedInfo +import org.evomaster.core.sql.SqlActionUtils /** * related info between resource and tables @@ -58,8 +59,14 @@ class ResourceRelatedToTable(val key: String) { fun updateActionRelatedToTable(verb : String, dto: SqlExecutionsDto, existingTables : Set) : Boolean{ - val tables = mutableListOf().plus(dto.deletedData).plus(dto.updatedData.keys).plus(dto.insertedData.keys).plus(dto.queriedData.keys) - .filter { existingTables.contains(it) || existingTables.any { e->e.toLowerCase() == it.toLowerCase() }}.toHashSet() + val tables = mutableListOf() + .plus(dto.deletedData) + .plus(dto.updatedData.keys) + .plus(dto.insertedData.keys) + .plus(dto.queriedData.keys) + .filter { x -> + existingTables.any { x.equals(it, true) } + }.toHashSet() if (tables.isEmpty()) return false @@ -74,7 +81,7 @@ class ResourceRelatedToTable(val key: String) { actionToTables[verb]!!.add(access) } - access.updateTableWithFields(dto.deletedData.map { Pair(it, mutableSetOf()) }.toMap(), SQLKey.DELETE) + access.updateTableWithFields(dto.deletedData.associateWith { mutableSetOf() }, SQLKey.DELETE) access.updateTableWithFields(dto.insertedData, SQLKey.INSERT) access.updateTableWithFields(dto.queriedData, SQLKey.SELECT) access.updateTableWithFields(dto.updatedData, SQLKey.UPDATE) @@ -83,15 +90,23 @@ class ResourceRelatedToTable(val key: String) { } fun getConfirmedDirectTables() : Set{ - return derivedMap.keys.filter { t-> confirmedSet.any { it.key.equals(t, ignoreCase = true) && it.value } }.toHashSet() + return derivedMap.keys + .filter { t-> + confirmedSet.any { it.key.equals(t, ignoreCase = true) && it.value } + }.toHashSet() } - fun findBestTableForParam(tables: Set, simpleP2Table : SimpleParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair, Double>? { - val map = simpleP2Table.derivedMap.filter { tables.any { t-> t.equals(it.key, ignoreCase = true) } } - .map { + fun findBestTableForParam( + tables: Set, + simpleP2Table : SimpleParamRelatedToTable, + onlyConfirmedColumn : Boolean = false + ) : Pair, Double>? { + val map = simpleP2Table.derivedMap + .filter { d -> tables.any { t-> SqlActionUtils.isMatchingTableName(d.key, t) } } + .map { Pair(it.key, it.value.similarity) - }.toMap() + }.toMap() if(map.isEmpty()) return null val best = map.asSequence().sortedBy { it.value }.last().value return Pair(map.filter { it.value == best }.keys, best) @@ -100,11 +115,18 @@ class ResourceRelatedToTable(val key: String) { /** * return Pair.first is name of table, Pair.second is column of Table */ - fun getSimpleParamToSpecifiedTable(table: String, simpleP2Table : SimpleParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair? { - simpleP2Table.derivedMap.filter { table.equals(it.key, ignoreCase = true) }.let { map-> - if (map.isEmpty()) return null - else return Pair(table, map.values.first().targetMatched) - } + fun getSimpleParamToSpecifiedTable( + table: String, + simpleP2Table : SimpleParamRelatedToTable, + onlyConfirmedColumn : Boolean = false + ) : Pair? { + + return simpleP2Table.derivedMap + .filter { table.equals(it.key, ignoreCase = true) } + .let { map-> + if (map.isEmpty()) null + else Pair(table, map.values.first().targetMatched) + } } @@ -115,7 +137,9 @@ class ResourceRelatedToTable(val key: String) { */ fun findBestTableForParam(tables: Set, bodyP2Table : BodyParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : MutableMap, Double>>?{ val fmap = bodyP2Table.fieldsMap - .filter { it.value.derivedMap.any { m-> tables.any { t-> t.equals(m.key, ignoreCase = true) } } } + .filter { + it.value.derivedMap.any { m-> tables.any { t-> t.equals(m.key, ignoreCase = true) } } + } if(fmap.isEmpty()) return null val result : MutableMap, Double>> = mutableMapOf() @@ -134,7 +158,9 @@ class ResourceRelatedToTable(val key: String) { */ fun getBodyParamToSpecifiedTable(table:String, bodyP2Table : BodyParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair>? { val fmap = bodyP2Table.fieldsMap - .filter { it.value.derivedMap.any { m->m.key.toLowerCase() == table.toLowerCase() } } + .filter { + it.value.derivedMap.any { m->m.key.toLowerCase() == table.toLowerCase() } + } if(fmap.isEmpty()) return null else{ return Pair(table, fmap.map { f-> Pair(f.key, f.value.getRelatedColumn(table)!!.first())}.toMap()) @@ -148,7 +174,7 @@ class ResourceRelatedToTable(val key: String) { */ fun getBodyParamToSpecifiedTable(table:String, bodyP2Table : BodyParamRelatedToTable, fieldName : String, onlyConfirmedColumn : Boolean = false) : Pair>? { val fmap = bodyP2Table.fieldsMap[fieldName]?: return null - fmap.derivedMap.filter { it.key.toLowerCase() == table.toLowerCase() }.let { + fmap.derivedMap.filter { SqlActionUtils.isMatchingTableName(it.key, table)}.let { if (it.isEmpty()) return null else return Pair(table, Pair(fieldName, it.values.first().targetMatched)) } @@ -165,9 +191,10 @@ class ResourceRelatedToTable(val key: String) { } private fun getTablesInDerivedMap(input : String) : List{ - return derivedMap.values.flatMap { - it.filter { m-> m.input.toLowerCase() == input.toLowerCase()} - } + return derivedMap.values + .flatMap { + it.filter { m-> m.input.toLowerCase() == input.toLowerCase()} + } } } @@ -232,7 +259,13 @@ abstract class ParamRelatedToTable ( val confirmedColumn : MutableSet = mutableSetOf() - open fun getRelatedColumn(table: String) : Set? = derivedMap.filterKeys { it.equals(table, ignoreCase = true) }.values.firstOrNull().run { if (this == null) null else setOf(this.targetMatched) } + open fun getRelatedColumn(table: String) : Set? = + derivedMap.filterKeys { SqlActionUtils.isMatchingTableName(it, table)} + .values.firstOrNull() + .run { + if (this == null) null + else setOf(this.targetMatched) + } } /** @@ -254,12 +287,19 @@ class BodyParamRelatedToTable(key: String, val referParam: Param): ParamRelatedT */ val fieldsMap : MutableMap = mutableMapOf() - override fun getRelatedColumn(table: String) : Set? = fieldsMap.values.filter { it.derivedMap.any { m-> m.key.toLowerCase() == table.toLowerCase() }}.run { - if (this.isEmpty()) return null - else - this.map { f->f.derivedMap.filterKeys { k->k.toLowerCase() == table.toLowerCase() }.asSequence().first().value.targetMatched}.toHashSet() - } - + override fun getRelatedColumn(table: String) : Set? = + fieldsMap.values + .filter { + it.derivedMap.any { m-> SqlActionUtils.isMatchingTableName(m.key, table)} + }.run { + if (this.isEmpty()) + return null + else + this.map { f-> + f.derivedMap.filterKeys { k -> SqlActionUtils.isMatchingTableName(k, table) } + .asSequence().first().value.targetMatched + }.toHashSet() + } } /** * related info between a field of BodyParam and a table @@ -267,10 +307,11 @@ class BodyParamRelatedToTable(key: String, val referParam: Param): ParamRelatedT class ParamFieldRelatedToTable(key: String) : ParamRelatedToTable(key){ override fun getRelatedColumn(table: String) : Set? { - derivedMap.filter { it.key.toLowerCase() == table.toLowerCase() }.let { - if(it.isEmpty()) return null - else return it.values.map {m-> m.targetMatched}.toSet() - } + return derivedMap.filter { SqlActionUtils.isMatchingTableName(it.key, table)} + .let { + if(it.isEmpty()) null + else it.values.map {m-> m.targetMatched}.toSet() + } } } diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt index baca2044ea..10e2504fc7 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt @@ -495,7 +495,57 @@ object SqlActionUtils { * @return a list of dbactions from [sqlActions] whose related table is [tableName] */ fun findDbActionsByTableName(sqlActions: List, tableName : String) : List{ - return sqlActions.filter { it.table.name.equals(tableName, ignoreCase = true) } + return sqlActions.filter { it.table.name.equals(tableName, ignoreCase = true) + || it.table.name.endsWith(".$tableName", true)} } + + /** + * Are the 2 names matching? This ignore case. + * The first [fullName] is a full qualifying name, including schema. + * The second [name] "might" be simple, or full qualifying. + */ + fun isMatchingTableName(fullName: String, name: String) : Boolean{ + + if(fullName.equals(name, ignoreCase = true)){ + return true + } + + if(name.contains(".")){ + return false + } + + return fullName.endsWith(".$name", true) + } + + /** + * Given a set of table ids, returns the one matching the given table name. + * There are at least 2 problems handled here: + * 1) case sensitiveness + * 2) keys having full names (eg, including schemas and possibly catalog) whereas input only having the name. + * this latter case is not a problem if names are unique + */ + fun getTableKey(keys: Set, tableName: String) : String?{ + /* + * SQL is not case sensitivity, table/column must ignore case sensitivity. + * No, this is not really true... + * Usually, names are lowered-cased by the DB, unless quoted in "": + * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL. + * + */ + val tableNameKey = keys.find { tableName.equals(it, ignoreCase = true) } + if (!tableName.contains(".") && tableNameKey == null){ + //input name might be without schema, so check for partial match + val candidates = keys.filter { it.endsWith(".${tableName}", true) } + if(candidates.size > 1){ + throw IllegalArgumentException("Ambiguity." + + " More than one candidate of table called $tableName." + + " Values: ${candidates.joinToString(", ")}") + } + if(candidates.size == 1){ + return candidates[0] + } + } + return tableNameKey + } } \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 701dfcdd2c..bdd522b44a 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -180,7 +180,7 @@ class SqlInsertBuilder( for (fk in tableDto.foreignKeys) { - val tableKey = getTableKey(tableToColumns.keys, fk.targetTable) + val tableKey = SqlActionUtils.getTableKey(tableToColumns.keys, fk.targetTable) if(tableKey == null || tableToColumns[tableKey] == null) { throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}") @@ -624,43 +624,21 @@ class SqlInsertBuilder( * * quotes "" can be used to force case-sensitivity */ - fun isTable(tableName: String) = getTableKey(tables.keys, tableName) != null + fun isTable(tableName: String) = SqlActionUtils.getTableKey(tables.keys, tableName) != null private fun getValueByTableNameKey(map: Map, tableName: String) : T?{ - val key = getTableKey(map.keys, tableName) + val key = SqlActionUtils.getTableKey(map.keys, tableName) return map[key] } - private fun getTableKey(keys: Set, tableName: String) : String?{ - /* - * SQL is not case sensitivity, table/column must ignore case sensitivity. - * No, this is not really true... - * Usually, names are lowered-cased by the DB, unless quoted in "": - * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL. - * - */ - val tableNameKey = keys.find { tableName.equals(it, ignoreCase = true) } - if (!tableName.contains(".") && tableNameKey == null){ - //input name might be without schema, so check for partial match - val candidates = keys.filter { it.endsWith(".${tableName}", true) } - if(candidates.size > 1){ - throw IllegalArgumentException("Ambiguity." + - " More than one candidate of table called $tableName." + - " Values: ${candidates.joinToString(", ")}") - } - if(candidates.size == 1){ - return candidates[0] - } - } - return tableNameKey - } + fun getTable(tableName: String, useExtraConstraints: Boolean): Table { val data = if (useExtraConstraints) extendedTables else tables - val tableNameKey = getTableKey(data.keys, tableName) + val tableNameKey = SqlActionUtils.getTableKey(data.keys, tableName) ?: throw IllegalArgumentException("No table called $tableName") return data[tableNameKey] ?: throw IllegalArgumentException("No table called $tableName") diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt index a8dd1e5c35..93497032c7 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt @@ -7,6 +7,9 @@ import org.evomaster.dbconstraint.TableConstraint * Should be immutable */ data class Table( + /** + * This usually would be fully qualified, ie, including schema + */ val name: String, val columns: Set, diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt index 9e0290b981..f4bca4a9b2 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt @@ -69,7 +69,7 @@ class ResourceNodeWithDbTest { assertEquals(6, cluster.getCluster().size) // table in db - assertTrue(cluster.getTableInfo().keys.containsAll(setOf("RFOO", "RBAR", "RXYZ"))) + assertTrue(cluster.getTableInfo().keys.containsAll(setOf("PUBLIC.RFOO", "PUBLIC.RBAR", "PUBLIC.RXYZ"))) // data in db assertEquals(2, cluster.getDataInDb("RFOO")?.size) @@ -79,7 +79,7 @@ class ResourceNodeWithDbTest { val rfooNode = cluster.getResourceNode("/v3/api/rfoo") assertNotNull(rfooNode) rfooNode!!.resourceToTable.apply { - assertTrue(derivedMap.keys.contains("RFOO")) + assertTrue(derivedMap.keys.contains("PUBLIC.RFOO"), "Keys: ${derivedMap.keys.joinToString(", ")}") assertEquals(1, paramToTable.size) assertTrue(paramToTable.values.first() is BodyParamRelatedToTable) (paramToTable.values.first() as BodyParamRelatedToTable).apply { From 5707132912407e0ac591bf8cd38a55d6c8bedb81 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Thu, 2 Jan 2025 11:07:00 +0800 Subject: [PATCH 11/21] add calculateStringSimilarityScoreWithTableName --- .../problem/util/inference/SimpleDeriveR2T.kt | 29 ++++++++++++++++--- .../org/evomaster/core/sql/SqlActionUtils.kt | 2 ++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index 4946699781..9538d9917d 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -10,11 +10,14 @@ import org.evomaster.core.problem.rest.resource.RestResourceCalls import org.evomaster.core.problem.rest.resource.RestResourceNode import org.evomaster.core.problem.rest.resource.dependency.* import org.evomaster.core.problem.util.ParamUtil +import org.evomaster.core.problem.util.SimilarityAlgorithm import org.evomaster.core.problem.util.inference.model.MatchedInfo import org.evomaster.core.problem.util.inference.model.ParamGeneBindMap import org.evomaster.core.problem.util.StringSimilarityComparator import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.ObjectGene +import org.evomaster.core.sql.SqlActionUtils.SCHEMA_TABLE_SEPARATOR +import kotlin.math.max /** * process inference related to resource @@ -44,8 +47,13 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { if(allTables.isNotEmpty()){ resourceNode.getAllSegments(flatten = true).forEach { seg -> ParamUtil.parseParams(seg).forEachIndexed stop@{ sindex, token -> - //check whether any table name matches token - val matchedMap = allTables.keys.map { Pair(it, StringSimilarityComparator.stringSimilarityScore(it, token)) }.asSequence().sortedBy { e->e.second } + /* + check whether any table name matches token + table name might contain schema, + */ + val matchedMap = allTables.keys.map { + Pair(it, StringSimilarityComparator.stringSimilarityScore(it, token)) + }.asSequence().sortedBy { e->e.second } if(matchedMap.lastOrNull()!= null && matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ matchedMap.filter { it.second == matchedMap.last().second }.forEach { @@ -78,7 +86,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { if(reftypes.isNotEmpty()){ reftypes.forEach { type-> if(!resourceNode.isPartOfStaticTokens(type) && allTables.isNotEmpty()){ - val matchedMap = allTables.keys.map { Pair(it, StringSimilarityComparator.stringSimilarityScore(it, type)) }.asSequence().sortedBy { e->e.second } + val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it, type)) }.asSequence().sortedBy { e->e.second } if(matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ matchedMap.filter { it.second == matchedMap.last().second }.forEach { resourceNode.resourceToTable.derivedMap.getOrPut(it.first){ @@ -92,7 +100,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { //1.3 derive resource to tables based on tokens on POST action resourceNode.actions.filter {it.verb == HttpVerb.POST }.forEach { post-> post.tokens.values.filter { !resourceNode.getName().toLowerCase().contains(it.getKey().toLowerCase()) }.forEach { atoken-> - val matchedMap = allTables.keys.map { Pair(it, StringSimilarityComparator.stringSimilarityScore(it, atoken.getKey())) }.asSequence().sortedBy { e->e.second } + val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it, atoken.getKey())) }.asSequence().sortedBy { e->e.second } matchedMap.last().apply { if(second >= StringSimilarityComparator.SimilarityThreshold){ resourceNode.resourceToTable.derivedMap.getOrPut(first){ @@ -107,6 +115,19 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { deriveParamsToTable(resourceNode.paramsInfo, resourceNode, allTables) } + /** + * compare [content] with given [tableName] + * tableName can contain info of schema, + * in this case we also calculate similarity with the last segment + * then return the higher similarity + */ + private fun calculateStringSimilarityScoreWithTableName(tableName: String, content: String): Double{ + val score = StringSimilarityComparator.stringSimilarityScore(tableName, content) + if (!tableName.contains(SCHEMA_TABLE_SEPARATOR)) return score + + val lastScore = StringSimilarityComparator.stringSimilarityScore(tableName.split(SCHEMA_TABLE_SEPARATOR).last(), content) + return max(score, lastScore) + } fun deriveParamsToTable(mapParamInfo : Map, r: RestResourceNode, allTables : Map){ mapParamInfo.forEach { (paramId, paramInfo) -> diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt index 10e2504fc7..202a5e7da3 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt @@ -12,6 +12,8 @@ import org.evomaster.core.sql.schema.Table object SqlActionUtils { + const val SCHEMA_TABLE_SEPARATOR = "." + private val log: Logger = LoggerFactory.getLogger(SqlActionUtils::class.java) fun verifyForeignKeys(actions: List): Boolean { From 434bef478a02ffd1c37d47628c40f3073dd5b929 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Thu, 2 Jan 2025 11:29:44 +0800 Subject: [PATCH 12/21] fix --- .../problem/util/inference/SimpleDeriveR2T.kt | 39 ++++++++++++------- .../rest/resource/ResourceNodeWithDbTest.kt | 14 ++++--- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index 9538d9917d..9b91a085b8 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -10,10 +10,10 @@ import org.evomaster.core.problem.rest.resource.RestResourceCalls import org.evomaster.core.problem.rest.resource.RestResourceNode import org.evomaster.core.problem.rest.resource.dependency.* import org.evomaster.core.problem.util.ParamUtil -import org.evomaster.core.problem.util.SimilarityAlgorithm import org.evomaster.core.problem.util.inference.model.MatchedInfo import org.evomaster.core.problem.util.inference.model.ParamGeneBindMap import org.evomaster.core.problem.util.StringSimilarityComparator +import org.evomaster.core.problem.util.StringSimilarityComparator.stringSimilarityScore import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.ObjectGene import org.evomaster.core.sql.SqlActionUtils.SCHEMA_TABLE_SEPARATOR @@ -52,7 +52,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { table name might contain schema, */ val matchedMap = allTables.keys.map { - Pair(it, StringSimilarityComparator.stringSimilarityScore(it, token)) + Pair(it, calculateStringSimilarityScoreWithTableName(it, token)) }.asSequence().sortedBy { e->e.second } if(matchedMap.lastOrNull()!= null && matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ @@ -65,7 +65,9 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { } val matchedPropertyMap = allTables.flatMap { t->t.value.columns.filter { c-> !ParamUtil.isGeneralName(c.name) }.map { c->Pair(t.value.name, c.name) } } - .map { p-> Pair(p.first, Pair(p.second,StringSimilarityComparator.stringSimilarityScore(p.second, token))) }.asSequence().sortedBy { e->e.second.second } + .map { p-> Pair(p.first, + Pair(p.second, calculateStringSimilarityScoreWithTableName(p.second, token) + )) }.asSequence().sortedBy { e->e.second.second } if(matchedPropertyMap.lastOrNull() != null && matchedPropertyMap.last().second.second >= StringSimilarityComparator.SimilarityThreshold){ matchedPropertyMap.filter { it.second.second == matchedPropertyMap.last().second.second }.forEach { @@ -122,11 +124,18 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { * then return the higher similarity */ private fun calculateStringSimilarityScoreWithTableName(tableName: String, content: String): Double{ - val score = StringSimilarityComparator.stringSimilarityScore(tableName, content) - if (!tableName.contains(SCHEMA_TABLE_SEPARATOR)) return score - - val lastScore = StringSimilarityComparator.stringSimilarityScore(tableName.split(SCHEMA_TABLE_SEPARATOR).last(), content) - return max(score, lastScore) + var score = stringSimilarityScore(tableName, content) // full A vs full B + val shortContent = if (content.contains(SCHEMA_TABLE_SEPARATOR)) content.split(SCHEMA_TABLE_SEPARATOR).last() else null + val shortTable = if (tableName.contains(SCHEMA_TABLE_SEPARATOR)) tableName.split(SCHEMA_TABLE_SEPARATOR).last() else null + if (shortContent == null && shortTable == null) return score + + if (shortContent != null) + score = max(score, stringSimilarityScore(tableName, shortContent)) // full A vs short B + if (shortTable != null) + score = max(score, stringSimilarityScore(shortTable, content)) // short A vs full B + if (shortTable != null && shortContent != null) + score = max(score, stringSimilarityScore(shortTable, shortContent)) + return score } fun deriveParamsToTable(mapParamInfo : Map, r: RestResourceNode, allTables : Map){ @@ -211,16 +220,16 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { .map { Pair(it.name, if(ParamUtil.isGeneralName(it.name)) - t.foreignKeys.map { f-> StringSimilarityComparator.stringSimilarityScore(paramName, "${f.targetTable}${it.name}")} - .plus(StringSimilarityComparator.stringSimilarityScore(paramName, it.name)) - .plus(StringSimilarityComparator.stringSimilarityScore(paramName, "$tableName${it.name}")) + t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName(paramName, "${f.targetTable}${it.name}")} + .plus(calculateStringSimilarityScoreWithTableName(paramName, it.name)) + .plus(calculateStringSimilarityScoreWithTableName(paramName, "$tableName${it.name}")) .asSequence().sorted().last() else if(ParamUtil.isGeneralName(paramName)) - t.foreignKeys.map { f-> StringSimilarityComparator.stringSimilarityScore("${f.targetTable}$paramName", it.name)} - .plus(StringSimilarityComparator.stringSimilarityScore(paramName, it.name)) - .plus(StringSimilarityComparator.stringSimilarityScore("$tableName$paramName", it.name)) + t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName("${f.targetTable}$paramName", it.name)} + .plus(calculateStringSimilarityScoreWithTableName(paramName, it.name)) + .plus(calculateStringSimilarityScoreWithTableName("$tableName$paramName", it.name)) .asSequence().sorted().last() - else StringSimilarityComparator.stringSimilarityScore(paramName, it.name)) + else calculateStringSimilarityScoreWithTableName(paramName, it.name)) }.asSequence().sortedBy { e->e.second } if(matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ matchedMap.filter { it.second == matchedMap.last().second }.forEach { diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt index f4bca4a9b2..cb1b084abd 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt @@ -14,6 +14,7 @@ import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.Gene import org.evomaster.core.search.gene.numeric.LongGene import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.SqlActionUtils.isMatchingTableName import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test @@ -87,7 +88,8 @@ class ResourceNodeWithDbTest { fieldsMap.forEach { t, u -> assertEquals(1, u.derivedMap.size) u.derivedMap.forEach { ut, uu -> - assertEquals("RFOO", ut) +// assertEquals("RFOO", ut) + assertTrue(isMatchingTableName(ut, "RFOO")) assertEquals(uu.input.toLowerCase(), uu.targetMatched.toLowerCase()) } } @@ -97,16 +99,16 @@ class ResourceNodeWithDbTest { val rbarNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}") assertNotNull(rbarNode) rbarNode!!.resourceToTable.apply { - assertTrue(derivedMap.keys.contains("RFOO")) - assertTrue(derivedMap.keys.contains("RBAR")) + assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RFOO") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RBAR") }) } val rxyzNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}/rxyz/{rxyzId}") assertNotNull(rxyzNode) rxyzNode!!.resourceToTable.apply { - assertTrue(derivedMap.keys.contains("RFOO")) - assertTrue(derivedMap.keys.contains("RBAR")) - assertTrue(derivedMap.keys.contains("RXYZ")) + assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RFOO") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RBAR") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RXYZ") }) } } From 507defb08d8e2b5c9a29d09e07aa9c28ba1931db Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Thu, 2 Jan 2025 16:18:25 +0800 Subject: [PATCH 13/21] fix due to merge --- .../client/java/sql/internal/SqlExpressionEvaluator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlExpressionEvaluator.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlExpressionEvaluator.java index 50744c842a..434ffb781a 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlExpressionEvaluator.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlExpressionEvaluator.java @@ -291,7 +291,7 @@ public void visit(ParenthesedSelect parenthesedSelect) { @Override public void visit(Column column) { String name = column.getColumnName(); - String table = sqlNameContext.getTableName(column); + String table = sqlNameContext.getFullyQualifiedTableName(column); Object value = dataRow.getValueByName(name, table); concreteValues.push(value); } From 04d66bd812d173cbccbcafe41ff41e969d8b77c3 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 8 Jan 2025 10:08:11 +0100 Subject: [PATCH 14/21] refactoring table id in dto --- .../controller/internal/SutController.java | 4 +- .../db/sql/h2/H2SchemaExtractorTest.java | 56 ++++++------- .../internal/db/sql/h2/H2SqlHandlerTest.java | 4 +- .../sql/mysql/MySQLSchemaExtractorTest.java | 6 +- .../java/controller/api/dto/SqlDtoUtils.java | 28 +++---- .../api/dto/database/schema/TableDto.java | 14 +--- .../api/dto/database/schema/TableIdDto.java | 23 ++++++ .../client/java/sql/DbInfoExtractor.java | 37 ++++----- .../constraint/H2ConstraintExtractor.java | 12 +-- .../constraint/MySQLConstraintExtractor.java | 4 +- .../PostgresConstraintExtractor.java | 4 +- ...risticsCalculatorWithInsertionDtoTest.java | 2 +- .../internal/QueryResultTransformerTest.java | 2 +- .../core/solver/SMTConditionVisitor.kt | 2 +- .../core/solver/SMTLibZ3DbConstraintSolver.kt | 6 +- .../evomaster/core/solver/SmtLibGenerator.kt | 22 ++--- .../evomaster/core/sql/SqlInsertBuilder.kt | 8 +- .../core/sql/SqlInsertBuilderTest.kt | 2 +- .../sql/extract/h2/AlterTableCheckEnumTest.kt | 10 +-- .../sql/extract/h2/AlterTableCheckTest.kt | 10 +-- .../sql/extract/h2/AlterTableUniqueTest.kt | 12 +-- .../sql/extract/h2/CatwatchSqlExtractTest.kt | 18 ++--- .../extract/h2/CreateTableCheckEnumTest.kt | 10 +-- .../sql/extract/h2/CreateTableCheckTest.kt | 10 +-- .../h2/FeaturesServiceSqlExtractTest.kt | 14 ++-- .../core/sql/extract/h2/NewsSqlExtractTest.kt | 2 +- .../core/sql/extract/h2/OcvnExtractTest.kt | 4 +- .../extract/h2/ProxyPrintSqlExtractTest.kt | 80 +++++++++---------- .../sql/extract/h2/ScoutApiSqlExtractTest.kt | 56 ++++++------- .../mysql/AlterTableExtractCheckTest.kt | 10 +-- .../sql/extract/mysql/AlterTableUniqueTest.kt | 2 +- .../mysql/CreateTableBoundedNumberTest.kt | 4 +- .../extract/mysql/CreateTableCheckT1Test.kt | 8 +- .../sql/extract/mysql/CreateTableCheckTest.kt | 10 +-- .../sql/extract/mysql/CreateTableEnumTest.kt | 2 +- .../extract/mysql/CreateTableUniqueTest.kt | 2 +- .../postgres/AlterTableExtractCheckTest.kt | 10 +-- .../postgres/AlterTableExtractUniqueTest.kt | 12 +-- .../sql/extract/postgres/ArrayTypesTest.kt | 4 +- .../extract/postgres/CompositeTypesTest.kt | 4 +- .../postgres/CreateTableExtractCheckTest.kt | 10 +-- .../sql/extract/postgres/Ind0ExtractTest.kt | 26 +++--- .../postgres/ManySimilarToChecksTest.kt | 2 +- .../postgres/ObjectIdentifierTypesTest.kt | 4 +- .../core/sql/extract/postgres/PgLsnTest.kt | 4 +- .../sql/extract/postgres/SqlTextColumnTest.kt | 4 +- .../sql/multidb/ConflictingConstraintsTest.kt | 6 +- .../sql/multidb/ConflictingSchemasTest.kt | 6 +- .../core/sql/multidb/SecondSchemaTest.kt | 10 +-- 49 files changed, 307 insertions(+), 295 deletions(-) create mode 100644 client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableIdDto.java diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 4924466a7e..02d621c34a 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -680,10 +680,10 @@ public final DbInfoDto getSqlDatabaseSchema() { if (fkMap.isEmpty()){ schemaDto.tables.forEach(t->{ - fkMap.putIfAbsent(t.name, new ArrayList<>()); + fkMap.putIfAbsent(t.id.name, new ArrayList<>()); if (t.foreignKeys!=null && !t.foreignKeys.isEmpty()){ t.foreignKeys.forEach(f->{ - fkMap.get(t.name).add(f.targetTable.toUpperCase()); + fkMap.get(t.id.name).add(f.targetTable.toUpperCase()); }); } }); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SchemaExtractorTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SchemaExtractorTest.java index 38be462a73..831c325732 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SchemaExtractorTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SchemaExtractorTest.java @@ -32,7 +32,7 @@ public void testBasic() throws Exception { assertAll(() -> assertEquals("db_test", schema.name.toLowerCase()), () -> assertEquals(DatabaseType.H2, schema.databaseType), () -> assertEquals(1, schema.tables.size()), - () -> assertEquals("foo", schema.tables.get(0).name.toLowerCase()), + () -> assertEquals("foo", schema.tables.get(0).id.name.toLowerCase()), () -> assertEquals(1, schema.tables.get(0).columns.size()) ); } @@ -45,8 +45,8 @@ public void testTwoTables() throws Exception { assertNotNull(schema); assertEquals(2, schema.tables.size()); - assertTrue(schema.tables.stream().map(t -> t.name.toLowerCase()).anyMatch(n -> n.equals("foo"))); - assertTrue(schema.tables.stream().map(t -> t.name.toLowerCase()).anyMatch(n -> n.equals("bar"))); + assertTrue(schema.tables.stream().map(t -> t.id.name.toLowerCase()).anyMatch(n -> n.equals("foo"))); + assertTrue(schema.tables.stream().map(t -> t.id.name.toLowerCase()).anyMatch(n -> n.equals("bar"))); } @@ -135,8 +135,8 @@ public void testBasicForeignKey() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(2, schema.tables.size()); - TableDto bar = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Bar")).findAny().get(); - TableDto foo = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto bar = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Bar")).findAny().get(); + TableDto foo = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(0, bar.foreignKeys.size()); assertEquals(1, foo.foreignKeys.size()); @@ -184,7 +184,7 @@ public void testColumnUpperBoundConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(2, fooTable.columns.size()); @@ -212,7 +212,7 @@ public void testTableConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(2, fooTable.columns.size()); @@ -236,7 +236,7 @@ public void testPrimaryKey() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -257,7 +257,7 @@ public void testEnumStringConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(2, fooTable.columns.size()); @@ -282,7 +282,7 @@ public void testEnumBooleanConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -306,7 +306,7 @@ public void testEnumIntegerConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -331,7 +331,7 @@ public void testEnumTinyIntConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -355,7 +355,7 @@ public void testEnumSmallIntConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -380,7 +380,7 @@ public void testEnumBigIntConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -404,7 +404,7 @@ public void testEnumDoubleConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -429,7 +429,7 @@ public void testEnumRealConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -453,7 +453,7 @@ public void testEnumDecimalConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -475,7 +475,7 @@ public void testEnumCharConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -498,7 +498,7 @@ public void testEnumLikeConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -520,7 +520,7 @@ public void testCreateEnumIntColumn() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -546,7 +546,7 @@ public void testCreateEnumColumn() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -571,7 +571,7 @@ public void testEnumColumn() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -598,7 +598,7 @@ public void testArray() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); @@ -621,7 +621,7 @@ public void testMultidimensionalArrayOfTwoDimensions() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); @@ -644,7 +644,7 @@ public void testMultidimensionalArrayOfThreeDimensions() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); @@ -667,7 +667,7 @@ public void testBooleanNonlArray() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); @@ -690,7 +690,7 @@ public void testVarCharArray() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); @@ -714,7 +714,7 @@ public void testIntegerArrayWithMaxLengthColumn() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SqlHandlerTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SqlHandlerTest.java index ff25f92386..8e3eb30d10 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SqlHandlerTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SqlHandlerTest.java @@ -28,7 +28,7 @@ public void testHandleNoRows() throws Exception { DbInfoExtractor dbInfoExtractor = new DbInfoExtractor(); DbInfoDto schema = dbInfoExtractor.extract(connection); assertEquals(1, schema.tables.size()); - assertEquals("Person".toUpperCase(), schema.tables.get(0).name.toUpperCase()); + assertEquals("Person".toUpperCase(), schema.tables.get(0).id.name.toUpperCase()); SqlHandler sqlHandler = new SqlHandler(null); sqlHandler.setConnection(connection); @@ -60,7 +60,7 @@ public void testHandleOneOrMoreRows() throws Exception { DbInfoExtractor dbInfoExtractor = new DbInfoExtractor(); DbInfoDto schema = dbInfoExtractor.extract(connection); assertEquals(1, schema.tables.size()); - assertEquals("Person".toUpperCase(), schema.tables.get(0).name.toUpperCase()); + assertEquals("Person".toUpperCase(), schema.tables.get(0).id.name.toUpperCase()); SqlScriptRunner.execCommand(connection, "INSERT INTO Person (person_id, first_name, last_name, age, email)\n" + "VALUES (1, 'John', 'Doe', 30, 'john.doe@example.com');"); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/MySQLSchemaExtractorTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/MySQLSchemaExtractorTest.java index 010f1941e8..a0f05e9253 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/MySQLSchemaExtractorTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/MySQLSchemaExtractorTest.java @@ -86,7 +86,7 @@ public void testCaseSensitivityOfExtractor() throws Exception { TableDto table = schema.tables.get(0); assertEquals("intColumn",table.columns.get(0).name); - assertEquals("TableNot", table.name); + assertEquals("TableNot", table.id.name); } @@ -163,7 +163,7 @@ public void testTwoSchemas() throws Exception { DbInfoDto schemaTest0 = DbInfoExtractor.extract(testUser0Connection); assertEquals("test", schemaTest0.name); assertEquals(1, schemaTest0.tables.size()); - assertEquals("my_table", schemaTest0.tables.get(0).name); + assertEquals("my_table", schemaTest0.tables.get(0).id.name); DbInfoDto newSchema = DbInfoExtractor.extract(testUser1Connection); // assertEquals("new_schema", newSchema.name); @@ -171,7 +171,7 @@ public void testTwoSchemas() throws Exception { //assertEquals(1, newSchema.tables.size()); //we are now fetching data for all schemas (/catalogs in MySQL) assertEquals(2, newSchema.tables.size()); - assertEquals("my_table", newSchema.tables.get(0).name); + assertEquals("my_table", newSchema.tables.get(0).id.name); SqlScriptRunner.execCommand(testUser1Connection, "DROP SCHEMA IF EXISTS new_schema"); deleteUserInDatabase(url, anotherTestUserName); diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java index fad84fbf8c..922a6884da 100644 --- a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java @@ -18,10 +18,10 @@ public class SqlDtoUtils { * Return a fully qualifying name for input table, which can be used as id. */ public static String getId(TableDto dto){ - if(dto.schema == null){ - return dto.name; + if(dto.id.schema == null){ + return dto.id.name; } - return dto.schema + "." + dto.name; + return dto.id.schema + "." + dto.id.name; } /** @@ -51,34 +51,34 @@ public static boolean matchByName(TableDto dto, String name){ throw new IllegalArgumentException("Invalid table name identifier. Too many '.': " + name); } if(tokens.length == 1){ - return dto.name.equalsIgnoreCase(tokens[0]); + return dto.id.name.equalsIgnoreCase(tokens[0]); } if(tokens.length == 2){ - boolean mn = dto.name.equalsIgnoreCase(tokens[1]); + boolean mn = dto.id.name.equalsIgnoreCase(tokens[1]); if(!mn){ return false; } - if(dto.catalog == null && dto.schema == null){ + if(dto.id.catalog == null && dto.id.schema == null){ //there is no default schema, but DTO is unspecified, then false return tokens[0].equalsIgnoreCase("public"); - } else if(dto.catalog != null && dto.schema != null){ + } else if(dto.id.catalog != null && dto.id.schema != null){ //both specified... so look at schema - return tokens[0].equalsIgnoreCase(dto.schema); + return tokens[0].equalsIgnoreCase(dto.id.schema); } else { //only one specified, take it - if(dto.schema != null){ - return tokens[0].equalsIgnoreCase(dto.schema); + if(dto.id.schema != null){ + return tokens[0].equalsIgnoreCase(dto.id.schema); } else{ //this can be the case for MySQL - return tokens[0].equalsIgnoreCase(dto.catalog); + return tokens[0].equalsIgnoreCase(dto.id.catalog); } } } if(tokens.length == 3){ - return tokens[0].equalsIgnoreCase(dto.catalog) - && tokens[1].equalsIgnoreCase(dto.schema) - && tokens[2].equalsIgnoreCase(dto.name); + return tokens[0].equalsIgnoreCase(dto.id.catalog) + && tokens[1].equalsIgnoreCase(dto.id.schema) + && tokens[2].equalsIgnoreCase(dto.id.name); } //shouldn't be reached diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableDto.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableDto.java index 48ff605fcb..8451a21aad 100644 --- a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableDto.java +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableDto.java @@ -4,19 +4,7 @@ public class TableDto { - /** - * The schema this table belongs to. - * Note that databases like MySQL make no distinction between catalog and schema. - */ - public String schema; - - - public String catalog; - - /** - * The name of the table - */ - public String name; + public TableIdDto id; /** * A list of descriptions for each column in the table diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableIdDto.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableIdDto.java new file mode 100644 index 0000000000..98ff593f88 --- /dev/null +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableIdDto.java @@ -0,0 +1,23 @@ +package org.evomaster.client.java.controller.api.dto.database.schema; + +public class TableIdDto { + + /** + * The schema this table belongs to. + * Note that databases like MySQL make no distinction between catalog and schema. + */ + public String schema; + + + /** + * A physical database can have several independent catalogs. + * For Postgres, can only access the catalog specified in the connection, ie., one per connection. + * MySQL can access all catalogs even with a single connection (they work more like schemas...) + */ + public String catalog; + + /** + * The name of the table + */ + public String name; +} diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java index 80c959ebfd..b6d8208c66 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java @@ -68,7 +68,7 @@ private static void checkForeignKeyToAutoIncrementMissing(DbInfoDto schema, Tabl .findFirst(); if (!targetTable.isPresent()) { - throw new IllegalArgumentException("Foreign key in table " + table.name + + throw new IllegalArgumentException("Foreign key in table " + table.id.name + " pointing to non-existent table " + fk.get().targetTable); } @@ -77,14 +77,14 @@ private static void checkForeignKeyToAutoIncrementMissing(DbInfoDto schema, Tabl .collect(Collectors.toList()); if (pks.isEmpty()) { - throw new IllegalArgumentException("No PK in table " + targetTable.get().name + " that has FKs pointing to it"); + throw new IllegalArgumentException("No PK in table " + targetTable.get().id.name + " that has FKs pointing to it"); } for (ColumnDto pk : pks) { if (pk.autoIncrement || pk.foreignKeyToAutoIncrement) { throw new IllegalArgumentException("Column " + pk.name + " in table " + pk.table + " is auto-increment, although FK pointing to it does not mark it " + - "as autoincrement in " + column.name + " in " + table.name + "as autoincrement in " + column.name + " in " + table.id.name ); } } @@ -101,7 +101,7 @@ private static void checkForeignKeyToAutoIncrementPresent(DbInfoDto schema, Tabl if (!fk.isPresent()) { throw new IllegalArgumentException("No foreign key constraint for marked column " + - column.name + " in table " + table.name); + column.name + " in table " + table.id.name); } //TODO proper handling of multi-column PKs/FKs @@ -111,7 +111,7 @@ private static void checkForeignKeyToAutoIncrementPresent(DbInfoDto schema, Tabl .findFirst(); if (!targetTable.isPresent()) { - throw new IllegalArgumentException("Foreign key in table " + table.name + + throw new IllegalArgumentException("Foreign key in table " + table.id.name + " pointing to non-existent table " + fk.get().targetTable); } @@ -123,15 +123,15 @@ private static void checkForeignKeyToAutoIncrementPresent(DbInfoDto schema, Tabl if (pks.size() != 1) { throw new IllegalArgumentException("There must be only 1 PK in table " + - targetTable.get().name + " pointed by the FK-to-autoincrement " + - column.name + " in " + table.name + ". However, there were: " + pks.size()); + targetTable.get().id.name + " pointed by the FK-to-autoincrement " + + column.name + " in " + table.id.name + ". However, there were: " + pks.size()); } ColumnDto pk = pks.get(0); if (!pk.autoIncrement && !pk.foreignKeyToAutoIncrement) { throw new IllegalArgumentException("Column " + pk.name + " in table " + pk.table + " is not auto-increment, although FK pointing to it does mark it" + - "as autoincrement in " + column.name + " in " + table.name + "as autoincrement in " + column.name + " in " + table.id.name ); } } @@ -498,7 +498,7 @@ public static void addUniqueConstraintToColumn(TableDto tableDto, String columnN .filter(c -> c.name.equals(columnName)).findAny().orElse(null); if (columnDto == null) { - throw new IllegalArgumentException("Missing column DTO for column:" + tableDto.name + "." + columnName); + throw new IllegalArgumentException("Missing column DTO for column:" + tableDto.id.name + "." + columnName); } columnDto.unique = true; @@ -575,9 +575,10 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, TableDto tableDto = new TableDto(); schemaDto.tables.add(tableDto); - tableDto.name = tables.getString("TABLE_NAME"); - tableDto.schema = tableSchema; - tableDto.catalog = tableCatalog; + tableDto.id = new TableIdDto(); + tableDto.id.name = tables.getString("TABLE_NAME"); + tableDto.id.schema = tableSchema; + tableDto.id.catalog = tableCatalog; if (tableIds.contains(SqlDtoUtils.getId(tableDto))) { /* @@ -590,7 +591,7 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, Set pks = new HashSet<>(); SortedMap primaryKeySequence = new TreeMap<>(); - ResultSet rsPK = md.getPrimaryKeys(tableDto.catalog, tableDto.schema, tableDto.name); + ResultSet rsPK = md.getPrimaryKeys(tableDto.id.catalog, tableDto.id.schema, tableDto.id.name); while (rsPK.next()) { String pkColumnName = rsPK.getString("COLUMN_NAME"); @@ -603,21 +604,21 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, tableDto.primaryKeySequence.addAll(primaryKeySequence.values()); - ResultSet columns = md.getColumns(tableDto.catalog, tableDto.schema, tableDto.name, null); + ResultSet columns = md.getColumns(tableDto.id.catalog, tableDto.id.schema, tableDto.id.name, null); Set columnNames = new HashSet<>(); while (columns.next()) { ColumnDto columnDto = new ColumnDto(); tableDto.columns.add(columnDto); - columnDto.table = tableDto.name; + columnDto.table = tableDto.id.name; columnDto.name = columns.getString("COLUMN_NAME"); if (columnNames.contains(columnDto.name)) { /* * Perhaps we should throw a more specific exception than IllegalArgumentException */ - throw new IllegalArgumentException("Cannot handle repeated column " + columnDto.name + " in table " + tableDto.name); + throw new IllegalArgumentException("Cannot handle repeated column " + columnDto.name + " in table " + tableDto.id.name); } else { columnNames.add(columnDto.name); } @@ -652,7 +653,7 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, columns.close(); - ResultSet fks = md.getImportedKeys(tableDto.catalog, tableDto.schema, tableDto.name); + ResultSet fks = md.getImportedKeys(tableDto.id.catalog, tableDto.id.schema, tableDto.id.name); while (fks.next()) { //TODO need to see how to handle case of multi-columns @@ -727,7 +728,7 @@ private static void extractMySQLColumn(DbInfoDto schemaDto, * corresponding [DATA_TYPE] column value. */ String sqlQuery = String.format("SELECT DATA_TYPE, table_schema from INFORMATION_SCHEMA.COLUMNS where\n" + - " table_schema = '%s' and table_name = '%s' and column_name= '%s' ", tableDto.schema, tableDto.name, columnDto.name); + " table_schema = '%s' and table_name = '%s' and column_name= '%s' ", tableDto.id.schema, tableDto.id.name, columnDto.name); try (Statement statement = connection.createStatement()) { ResultSet rs = statement.executeQuery(sqlQuery); if (rs.next()) { diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/H2ConstraintExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/H2ConstraintExtractor.java index 90b8d5fdb8..bb5049c879 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/H2ConstraintExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/H2ConstraintExtractor.java @@ -94,8 +94,8 @@ private List extractTableConstraintsVersionTwoOrHigher(Connec List tableCheckExpressions = new ArrayList<>(); for (TableDto tableDto : schemaDto.tables) { - String tableSchema = tableDto.schema; - String tableName = tableDto.name; + String tableSchema = tableDto.id.schema; + String tableName = tableDto.id.name; try (Statement statement = connectionToH2.createStatement()) { final String query = String.format("Select CONSTRAINT_CATALOG,CONSTRAINT_SCHEMA,CONSTRAINT_NAME,CONSTRAINT_TYPE From INFORMATION_SCHEMA.TABLE_CONSTRAINTS\n" + " where TABLE_CONSTRAINTS.TABLE_SCHEMA='%s' \n" @@ -192,8 +192,8 @@ private List extractTableConstraintsVersionOneOrLower(Connect List tableCheckExpressions = new ArrayList<>(); for (TableDto tableDto : schemaDto.tables) { - String tableSchema = tableDto.schema; - String tableName = tableDto.name; + String tableSchema = tableDto.id.schema; + String tableName = tableDto.id.name; try (Statement statement = connectionToH2.createStatement()) { final String query = String.format("Select CONSTRAINT_TYPE, CHECK_EXPRESSION, COLUMN_LIST From INFORMATION_SCHEMA.CONSTRAINTS\n" + " where CONSTRAINTS.TABLE_SCHEMA='%s' \n" @@ -255,8 +255,8 @@ private List extractColumnConstraintsVersion1OrLower(Connecti List columnConstraints = new ArrayList<>(); for (TableDto tableDto : schemaDto.tables) { - String tableSchema = tableDto.schema; - String tableName = tableDto.name; + String tableSchema = tableDto.id.schema; + String tableName = tableDto.id.name; try (Statement statement = connectionToH2.createStatement()) { diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/MySQLConstraintExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/MySQLConstraintExtractor.java index 6836ca44ef..abd3ed59df 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/MySQLConstraintExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/MySQLConstraintExtractor.java @@ -42,8 +42,8 @@ public List extract(Connection connectionToMySQL, DbInfoDto s List constraints = new ArrayList<>(); for (TableDto tableDto : schemaDto.tables){ - String tableSchema = tableDto.schema; - String tableName = tableDto.name; + String tableSchema = tableDto.id.schema; + String tableName = tableDto.id.name; try (Statement statement = connectionToMySQL.createStatement()) { String query = String.format("SELECT *\n" + " FROM information_schema.table_constraints\n" + diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/PostgresConstraintExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/PostgresConstraintExtractor.java index 69e6cbe6af..896020990a 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/PostgresConstraintExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/PostgresConstraintExtractor.java @@ -72,8 +72,8 @@ public List extract(Connection connectionToPostgres, DbInfoDt List constraints = new ArrayList<>(); for (TableDto tableDto : schemaDto.tables) { - String tableSchema = tableDto.schema; - String tableName = tableDto.name; + String tableSchema = tableDto.id.schema; + String tableName = tableDto.id.name; try (Statement statement = connectionToPostgres.createStatement()) { diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java index 224bfda896..b069264b5c 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java @@ -20,7 +20,7 @@ public class HeuristicsCalculatorWithInsertionDtoTest { private DbInfoDto createSchemaDtoWithFooTableAndXColumn(String xDataType){ DbInfoDto schemaDto = new DbInfoDto(); TableDto tableDto = new TableDto(); - tableDto.name = "Foo"; + tableDto.id.name = "Foo"; ColumnDto dto = new ColumnDto(); dto.name = "x"; dto.type = xDataType; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java index 7a21398d19..b1994d7b03 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java @@ -23,7 +23,7 @@ public class QueryResultTransformerTest { private TableDto createTableDate(List columnTypes, List columnNames, String tableName){ assertEquals(columnTypes.size(), columnNames.size()); TableDto tableDto = new TableDto(); - tableDto.name = tableName; + tableDto.id.name = tableName; for (int i = 0; i < columnTypes.size(); i++){ ColumnDto dto = new ColumnDto(); dto.name = columnNames.get(i); diff --git a/core/src/main/kotlin/org/evomaster/core/solver/SMTConditionVisitor.kt b/core/src/main/kotlin/org/evomaster/core/solver/SMTConditionVisitor.kt index 62aecc50ae..fb67f66103 100644 --- a/core/src/main/kotlin/org/evomaster/core/solver/SMTConditionVisitor.kt +++ b/core/src/main/kotlin/org/evomaster/core/solver/SMTConditionVisitor.kt @@ -115,7 +115,7 @@ class SMTConditionVisitor( */ private fun isAColumn(operand: String): Boolean { return tables.any { - it.name.equals(defaultTableName, ignoreCase = true) && + it.id.name.equals(defaultTableName, ignoreCase = true) && it.columns.any { column -> column.name.equals(operand, ignoreCase = true) } } } diff --git a/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt b/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt index c0ad5b1bf1..8c44924e46 100644 --- a/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt +++ b/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt @@ -140,7 +140,7 @@ class SMTLibZ3DbConstraintSolver( } private fun isBoolean(table: Table, columnName: String?): Boolean { - val col = schemaDto.tables.first { it.name == table.name }.columns.first { it.name == columnName } + val col = schemaDto.tables.first { it.id.name == table.name }.columns.first { it.name == columnName } return col.type == "BOOLEAN" } @@ -167,10 +167,10 @@ class SMTLibZ3DbConstraintSolver( * @return The Table object. */ private fun findTableByName(schema: DbInfoDto, tableName: String): Table { - val tableDto = schema.tables.find { it.name.equals(tableName, ignoreCase = true) } + val tableDto = schema.tables.find { it.id.name.equals(tableName, ignoreCase = true) } ?: throw RuntimeException("Table not found: $tableName") return Table( - tableDto.name, + tableDto.id.name, findColumns(tableDto), // Convert columns from DTO findForeignKeys(tableDto) // TODO: Implement this method ) diff --git a/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt b/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt index 9ada18e561..040a08691f 100644 --- a/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt +++ b/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt @@ -54,7 +54,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I */ private fun appendTableDefinitions(smt: SMTLib) { for (table in schema.tables) { - val dataTypeName = "${StringUtils.capitalization(table.name)}Row" + val dataTypeName = "${StringUtils.capitalization(table.id.name)}Row" // Declare datatype for the table smt.addNode( @@ -64,7 +64,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I // Declare constants for each row for (i in 1..numberOfRows) { smt.addNode( - DeclareConstSMTNode("${table.name.lowercase(Locale.getDefault())}$i", dataTypeName) + DeclareConstSMTNode("${table.id.name.lowercase(Locale.getDefault())}$i", dataTypeName) ) } } @@ -89,7 +89,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @param table The table for which unique constraints are added. */ private fun appendUniqueConstraints(smt: SMTLib, table: TableDto) { - val tableName = table.name.lowercase(Locale.getDefault()) + val tableName = table.id.name.lowercase(Locale.getDefault()) for (column in table.columns) { if (column.unique) { val nodes = assertForDistinctField(column.name, tableName) @@ -127,7 +127,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @return The corresponding SMT node. */ private fun parseCheckExpression(table: TableDto, condition: SqlCondition, index: Int): SMTNode { - val visitor = SMTConditionVisitor(table.name.lowercase(Locale.getDefault()), emptyMap(), schema.tables, index) + val visitor = SMTConditionVisitor(table.id.name.lowercase(Locale.getDefault()), emptyMap(), schema.tables, index) return condition.accept(visitor, null) as SMTNode } @@ -163,7 +163,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I private fun appendBooleanConstraints(smt: SMTLib) { for (table in schema.tables) { - val tableName = table.name.lowercase(Locale.getDefault()) + val tableName = table.id.name.lowercase(Locale.getDefault()) for (column in table.columns) { if (column.type.equals("BOOLEAN", ignoreCase = true)) { val columnName = column.name.uppercase() @@ -190,7 +190,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I private fun appendTimestampConstraints(smt: SMTLib) { for (table in schema.tables) { - val tableName = table.name.lowercase(Locale.getDefault()) + val tableName = table.id.name.lowercase(Locale.getDefault()) for (column in table.columns) { if (column.type.equals("TIMESTAMP", ignoreCase = true)) { val columnName = column.name.uppercase() @@ -228,7 +228,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @param table The table for which primary key constraints are added. */ private fun appendPrimaryKeyConstraints(smt: SMTLib, table: TableDto) { - val tableName = table.name.lowercase(Locale.getDefault()) + val tableName = table.id.name.lowercase(Locale.getDefault()) val primaryKeys = table.columns.filter { it.primaryKey } for (primaryKey in primaryKeys) { @@ -270,11 +270,11 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @param table The table for which foreign key constraints are added. */ private fun appendForeignKeyConstraints(smt: SMTLib, table: TableDto) { - val sourceTableName = table.name.lowercase(Locale.getDefault()) + val sourceTableName = table.id.name.lowercase(Locale.getDefault()) for (foreignKey in table.foreignKeys) { val referencedTable = findReferencedTable(foreignKey) - val referencedTableName = referencedTable.name.lowercase(Locale.getDefault()) + val referencedTableName = referencedTable.id.name.lowercase(Locale.getDefault()) val referencedColumnSelector = findReferencedPKSelector(referencedTable, foreignKey) for (sourceColumn in foreignKey.sourceColumns) { @@ -343,7 +343,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @return The referenced table. */ private fun findReferencedTable(foreignKey: ForeignKeyDto): TableDto { - return schema.tables.firstOrNull { it.name.equals(foreignKey.targetTable, ignoreCase = true) } + return schema.tables.firstOrNull { it.id.name.equals(foreignKey.targetTable, ignoreCase = true) } ?: throw RuntimeException("Referenced table not found: ${foreignKey.targetTable}") } @@ -479,7 +479,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I // Only add GetValueSMTNode for the mentioned tables for (table in schema.tables) { - val tableNameLower = table.name.lowercase(Locale.getDefault()) + val tableNameLower = table.id.name.lowercase(Locale.getDefault()) if (tablesMentioned.contains(tableNameLower)) { for (i in 1..numberOfRows) { smt.addNode(GetValueSMTNode("$tableNameLower$i")) diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index bdd522b44a..49f870a523 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -193,7 +193,7 @@ class SqlInsertBuilder( // ?: throw IllegalArgumentException("Issue in foreign key: table ${f.targetTable} does not have a column called $cname") val c = tableToColumns[SqlDtoUtils.getId(tableDto)]!!.find { it.name.equals(cname, ignoreCase = true) } - ?: throw IllegalArgumentException("Issue in foreign key: table ${tableDto.name} does not have a column called $cname") + ?: throw IllegalArgumentException("Issue in foreign key: table ${tableDto.id.name} does not have a column called $cname") sourceColumns.add(c) } @@ -212,8 +212,8 @@ class SqlInsertBuilder( for (column in tableDto.columns) { - if (!column.table.equals(tableDto.name, ignoreCase = true)) { - throw IllegalArgumentException("Column in different table: ${column.table}!=${tableDto.name}") + if (!column.table.equals(tableDto.id.name, ignoreCase = true)) { + throw IllegalArgumentException("Column in different table: ${column.table}!=${tableDto.id.name}") } val newColumn = createColumnFrom(column, tableConstraints, schemaDto) @@ -599,7 +599,7 @@ class SqlInsertBuilder( private fun parseTableConstraints(t: TableDto): List { val tableConstraints = mutableListOf() - val tableName = t.name + val tableName = t.id.name for (sqlCheckExpression in t.tableCheckExpressions) { val builder = TableConstraintBuilder() val constraintDatabaseType = getConstraintDatabaseType(this.databaseType) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt index ed40fa4094..28ab5568a3 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt @@ -1283,7 +1283,7 @@ class SqlInsertBuilderTest { val sqlInsertBuilder = SqlInsertBuilder(schemaDto, null) return sqlInsertBuilder.createSqlInsertionAction( - schemaDto.tables[0].name, + schemaDto.tables[0].id.name, setOf("*"), mutableListOf(), false, diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckEnumTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckEnumTest.kt index 2808b70409..f983b0afd6 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckEnumTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckEnumTest.kt @@ -22,14 +22,14 @@ class AlterTableCheckEnumTest : ExtractTestBaseH2() { assertEquals("db_test", schema.name.lowercase()) assertEquals(DatabaseType.H2, schema.databaseType) - assertTrue(schema.tables.any { it.name == "X" }) + assertTrue(schema.tables.any { it.id.name == "X" }) - assertEquals(10, schema.tables.first { it.name == "X" }.columns.size) + assertEquals(10, schema.tables.first { it.id.name == "X" }.columns.size) - assertTrue(schema.tables.first { it.name == "X" }.columns.any { it.name == "STATUS" }); + assertTrue(schema.tables.first { it.id.name == "X" }.columns.any { it.name == "STATUS" }); - assertEquals(1, schema.tables.first { it.name == "X" }.tableCheckExpressions.size) - assertEquals("(\"STATUS\" IN('A', 'B'))", schema.tables.first { it.name == "X" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "X" }.tableCheckExpressions.size) + assertEquals("(\"STATUS\" IN('A', 'B'))", schema.tables.first { it.id.name == "X" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckTest.kt index a345410aee..4e4894ab98 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckTest.kt @@ -22,14 +22,14 @@ class AlterTableCheckTest : ExtractTestBaseH2() { assertEquals("db_test", schema.name.lowercase()) assertEquals(DatabaseType.H2, schema.databaseType) - assertTrue(schema.tables.any { it.name == "PEOPLE" }) + assertTrue(schema.tables.any { it.id.name == "PEOPLE" }) - assertEquals(2, schema.tables.first { it.name == "PEOPLE" }.columns.size) + assertEquals(2, schema.tables.first { it.id.name == "PEOPLE" }.columns.size) - assertTrue(schema.tables.first { it.name == "PEOPLE" }.columns.any { it.name == "AGE" }); + assertTrue(schema.tables.first { it.id.name == "PEOPLE" }.columns.any { it.name == "AGE" }); - assertEquals(1, schema.tables.first { it.name == "PEOPLE" }.tableCheckExpressions.size) - assertEquals("(\"AGE\" <= 100)", schema.tables.first { it.name == "PEOPLE" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "PEOPLE" }.tableCheckExpressions.size) + assertEquals("(\"AGE\" <= 100)", schema.tables.first { it.id.name == "PEOPLE" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableUniqueTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableUniqueTest.kt index 37cdefdbbb..4c714ac54b 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableUniqueTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableUniqueTest.kt @@ -20,16 +20,16 @@ class AlterTableUniqueTest : ExtractTestBaseH2() { assertAll(Executable { assertEquals("db_test", schema.name.lowercase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(2, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "COUNTRIES" }, "missing table COUNTRIES") }, - Executable { assertTrue(schema.tables.any { it.name == "PASSPORTS" }, "missing table PASSPORTS") } + Executable { assertTrue(schema.tables.any { it.id.name == "COUNTRIES" }, "missing table COUNTRIES") }, + Executable { assertTrue(schema.tables.any { it.id.name == "PASSPORTS" }, "missing table PASSPORTS") } ) - assertEquals(true, schema.tables.filter { it.name == "PASSPORTS" }.first().columns.filter { it.name == "COUNTRY_ID"}.first().unique) - assertEquals(true, schema.tables.filter { it.name == "PASSPORTS" }.first().columns.filter { it.name == "PASSPORT_NUMBER"}.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "PASSPORTS" }.first().columns.filter { it.name == "COUNTRY_ID"}.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "PASSPORTS" }.first().columns.filter { it.name == "PASSPORT_NUMBER"}.first().unique) - assertEquals(1, schema.tables.filter { it.name == "PASSPORTS" }.first().tableCheckExpressions.size) - assertEquals("(\"PASSPORT_NUMBER\" > CAST(0 AS BIGINT))", schema.tables.filter { it.name == "PASSPORTS" }.first().tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.filter { it.id.name == "PASSPORTS" }.first().tableCheckExpressions.size) + assertEquals("(\"PASSPORT_NUMBER\" > CAST(0 AS BIGINT))", schema.tables.filter { it.id.name == "PASSPORTS" }.first().tableCheckExpressions[0].sqlCheckExpression) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt index c0ebd13d7c..f352d964eb 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt @@ -29,17 +29,17 @@ class CatwatchSqlExtractTest : ExtractTestBaseH2(){ assertAll(Executable { assertEquals("db_test", schema.name.lowercase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(5, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "CONTRIBUTOR" }) }, - Executable { assertTrue(schema.tables.any { it.name == "LANGUAGE_LIST" }) }, - Executable { assertTrue(schema.tables.any { it.name == "MAINTAINERS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PROJECT" }) }, - Executable { assertTrue(schema.tables.any { it.name == "STATISTICS" }) } + Executable { assertTrue(schema.tables.any { it.id.name == "CONTRIBUTOR" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "LANGUAGE_LIST" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "MAINTAINERS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PROJECT" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "STATISTICS" }) } ) - assertEquals(listOf("ID", "ORGANIZATION_ID", "SNAPSHOT_DATE"), schema.tables.filter { it.name == "CONTRIBUTOR" }.first().primaryKeySequence) - assertEquals(listOf("ID", "SNAPSHOT_DATE"), schema.tables.filter { it.name == "STATISTICS" }.first().primaryKeySequence) - assertEquals(listOf("ID"), schema.tables.filter { it.name == "PROJECT" }.first().primaryKeySequence) - assertEquals(listOf(), schema.tables.filter { it.name == "MAINTAINERS" }.first().primaryKeySequence) + assertEquals(listOf("ID", "ORGANIZATION_ID", "SNAPSHOT_DATE"), schema.tables.filter { it.id.name == "CONTRIBUTOR" }.first().primaryKeySequence) + assertEquals(listOf("ID", "SNAPSHOT_DATE"), schema.tables.filter { it.id.name == "STATISTICS" }.first().primaryKeySequence) + assertEquals(listOf("ID"), schema.tables.filter { it.id.name == "PROJECT" }.first().primaryKeySequence) + assertEquals(listOf(), schema.tables.filter { it.id.name == "MAINTAINERS" }.first().primaryKeySequence) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckEnumTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckEnumTest.kt index f29aa3b00c..12ab81c0b2 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckEnumTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckEnumTest.kt @@ -22,14 +22,14 @@ class CreateTableCheckEnumTest : ExtractTestBaseH2() { assertEquals("db_test", schema.name.lowercase()) assertEquals(DatabaseType.H2, schema.databaseType) - assertTrue(schema.tables.any { it.name == "X" }) + assertTrue(schema.tables.any { it.id.name == "X" }) - assertEquals(10, schema.tables.first { it.name == "X" }.columns.size) + assertEquals(10, schema.tables.first { it.id.name == "X" }.columns.size) - assertTrue(schema.tables.first { it.name == "X" }.columns.any { it.name == "STATUS" }); + assertTrue(schema.tables.first { it.id.name == "X" }.columns.any { it.name == "STATUS" }); - assertEquals(1, schema.tables.first { it.name == "X" }.tableCheckExpressions.size) - assertEquals("(\"STATUS\" IN('A', 'B'))", schema.tables.first { it.name == "X" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "X" }.tableCheckExpressions.size) + assertEquals("(\"STATUS\" IN('A', 'B'))", schema.tables.first { it.id.name == "X" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckTest.kt index f9e6c9dc6d..ca58b0bd29 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckTest.kt @@ -22,14 +22,14 @@ class CreateTableCheckTest : ExtractTestBaseH2() { assertEquals("db_test", schema.name.lowercase()) assertEquals(DatabaseType.H2, schema.databaseType) - assertTrue(schema.tables.any { it.name == "PEOPLE" }) + assertTrue(schema.tables.any { it.id.name == "PEOPLE" }) - assertEquals(2, schema.tables.first { it.name == "PEOPLE" }.columns.size) + assertEquals(2, schema.tables.first { it.id.name == "PEOPLE" }.columns.size) - assertTrue(schema.tables.first { it.name == "PEOPLE" }.columns.any { it.name == "AGE" }); + assertTrue(schema.tables.first { it.id.name == "PEOPLE" }.columns.any { it.name == "AGE" }); - assertEquals(1, schema.tables.first { it.name == "PEOPLE" }.tableCheckExpressions.size) - assertEquals("(\"AGE\" <= 100)", schema.tables.first { it.name == "PEOPLE" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "PEOPLE" }.tableCheckExpressions.size) + assertEquals("(\"AGE\" <= 100)", schema.tables.first { it.id.name == "PEOPLE" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/FeaturesServiceSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/FeaturesServiceSqlExtractTest.kt index 19a8b03cc6..0f94aa456c 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/FeaturesServiceSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/FeaturesServiceSqlExtractTest.kt @@ -22,15 +22,15 @@ class FeaturesServiceSqlExtractTest : ExtractTestBaseH2() { assertAll(Executable { assertEquals("db_test", schema.name.toLowerCase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(6, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "CONSTRAINT_EXCLUDES" }) }, - Executable { assertTrue(schema.tables.any { it.name == "CONSTRAINT_REQUIRES" }) }, - Executable { assertTrue(schema.tables.any { it.name == "FEATURE" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PRODUCT" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PRODUCT_CONFIGURATION" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PRODUCT_CONFIGURATION_ACTIVED_FEATURES" }) } + Executable { assertTrue(schema.tables.any { it.id.name == "CONSTRAINT_EXCLUDES" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "CONSTRAINT_REQUIRES" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "FEATURE" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PRODUCT" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PRODUCT_CONFIGURATION" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PRODUCT_CONFIGURATION_ACTIVED_FEATURES" }) } ) - assertEquals(listOf("IN_CONFIGURATIONS_ID", "ACTIVED_FEATURES_ID"), schema.tables.filter { it.name == "PRODUCT_CONFIGURATION_ACTIVED_FEATURES" }.first().primaryKeySequence) + assertEquals(listOf("IN_CONFIGURATIONS_ID", "ACTIVED_FEATURES_ID"), schema.tables.filter { it.id.name == "PRODUCT_CONFIGURATION_ACTIVED_FEATURES" }.first().primaryKeySequence) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/NewsSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/NewsSqlExtractTest.kt index d16bffc068..e895ce36e3 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/NewsSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/NewsSqlExtractTest.kt @@ -21,7 +21,7 @@ class NewsSqlExtractTest : ExtractTestBaseH2() { assertAll(Executable { assertEquals("db_test", schema.name.toLowerCase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(1, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "NEWS_ENTITY" }) } + Executable { assertTrue(schema.tables.any { it.id.name == "NEWS_ENTITY" }) } ) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/OcvnExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/OcvnExtractTest.kt index cc79b3252c..94eceab9c0 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/OcvnExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/OcvnExtractTest.kt @@ -64,7 +64,7 @@ class OcvnExtractTest : ExtractTestBaseH2() { ) for (name in tableNames) { - assertTrue(schema.tables.any { it.name.equals(name, true) }, "Missing table $name") + assertTrue(schema.tables.any { it.id.name.equals(name, true) }, "Missing table $name") } } @@ -77,7 +77,7 @@ class OcvnExtractTest : ExtractTestBaseH2() { val builder = SqlInsertBuilder(schema) val tableName = "FILE_CONTENT" - assertTrue(schema.tables.any { it.name.equals(tableName, true) }) + assertTrue(schema.tables.any { it.id.name.equals(tableName, true) }) val columnName = "BYTES" val actions = builder.createSqlInsertionAction(tableName, setOf(columnName)) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ProxyPrintSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ProxyPrintSqlExtractTest.kt index 9b83d1c220..f5b39f5475 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ProxyPrintSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ProxyPrintSqlExtractTest.kt @@ -30,18 +30,18 @@ class ProxyPrintSqlExtractTest : ExtractTestBaseH2() { assertAll(Executable { assertEquals("db_test", schema.name.lowercase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(15, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "ADMIN" }) }, - Executable { assertTrue(schema.tables.any { it.name == "CONSUMERS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "DOCUMENTS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "DOCUMENTS_SPECS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "EMPLOYEES" }) }, - Executable { assertTrue(schema.tables.any { it.name == "MANAGERS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "NOTIFICATION" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PRICETABLES" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PRINT_REQUESTS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "REVIEWS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "ROLES" }) }, - Executable { assertTrue(schema.tables.any { it.name == "USERS" }) } + Executable { assertTrue(schema.tables.any { it.id.name == "ADMIN" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "CONSUMERS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "DOCUMENTS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "DOCUMENTS_SPECS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "EMPLOYEES" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "MANAGERS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "NOTIFICATION" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PRICETABLES" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PRINT_REQUESTS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "REVIEWS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "ROLES" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "USERS" }) } ) @@ -50,38 +50,38 @@ class ProxyPrintSqlExtractTest : ExtractTestBaseH2() { * table USERS has a unique column USERNAME: * alter table users add constraint UK_r43af9ap4edm43mmtq01oddj6 unique (username); */ - assertEquals(true, schema.tables.find { it.name == "USERS" }!!.columns.find { it.name == "USERNAME" }!!.unique) + assertEquals(true, schema.tables.find { it.id.name == "USERS" }!!.columns.find { it.name == "USERNAME" }!!.unique) /** * BIGSERIAL are autoincrement fields */ - assertEquals(true, schema.tables.filter { it.name == "USERS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "DOCUMENTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "NOTIFICATION" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "PRINTING_SCHEMAS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "PRINTSHOPS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "PRINT_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "REGISTER_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "REVIEWS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - - assertEquals(false, schema.tables.filter { it.name == "USERS" }.first().columns.filter { it.name == "PASSWORD" }.first().nullable) - assertEquals(false, schema.tables.filter { it.name == "USERS" }.first().columns.filter { it.name == "USERNAME" }.first().nullable) - - assertEquals(listOf("PRINTSHOP_ID", "ITEM"), schema.tables.filter { it.name == "PRICETABLES" }.first().primaryKeySequence); - - assertEquals(true, schema.tables.filter { it.name == "CONSUMERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); - assertEquals(true, schema.tables.filter { it.name == "ADMIN" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); - assertEquals(true, schema.tables.filter { it.name == "EMPLOYEES" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); assertEquals(true, schema.tables.filter { it.name == "CONSUMERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); - assertEquals(true, schema.tables.filter { it.name == "MANAGERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); - assertEquals(true, schema.tables.filter { it.name == "PRICETABLES" }.first().columns.filter { it.name == "PRINTSHOP_ID" }.first().foreignKeyToAutoIncrement); - - assertEquals(false, schema.tables.filter { it.name == "DOCUMENTS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) - assertEquals(false, schema.tables.filter { it.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) - assertEquals(false, schema.tables.filter { it.name == "NOTIFICATION" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) - assertEquals(false, schema.tables.filter { it.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) - assertEquals(false, schema.tables.filter { it.name == "PRINTING_SCHEMAS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) - assertEquals(false, schema.tables.filter { it.name == "REGISTER_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "USERS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "DOCUMENTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "NOTIFICATION" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "PRINTING_SCHEMAS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "PRINTSHOPS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "PRINT_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "REGISTER_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "REVIEWS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + + assertEquals(false, schema.tables.filter { it.id.name == "USERS" }.first().columns.filter { it.name == "PASSWORD" }.first().nullable) + assertEquals(false, schema.tables.filter { it.id.name == "USERS" }.first().columns.filter { it.name == "USERNAME" }.first().nullable) + + assertEquals(listOf("PRINTSHOP_ID", "ITEM"), schema.tables.filter { it.id.name == "PRICETABLES" }.first().primaryKeySequence); + + assertEquals(true, schema.tables.filter { it.id.name == "CONSUMERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); + assertEquals(true, schema.tables.filter { it.id.name == "ADMIN" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); + assertEquals(true, schema.tables.filter { it.id.name == "EMPLOYEES" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); assertEquals(true, schema.tables.filter { it.id.name == "CONSUMERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); + assertEquals(true, schema.tables.filter { it.id.name == "MANAGERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); + assertEquals(true, schema.tables.filter { it.id.name == "PRICETABLES" }.first().columns.filter { it.name == "PRINTSHOP_ID" }.first().foreignKeyToAutoIncrement); + + assertEquals(false, schema.tables.filter { it.id.name == "DOCUMENTS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(false, schema.tables.filter { it.id.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(false, schema.tables.filter { it.id.name == "NOTIFICATION" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(false, schema.tables.filter { it.id.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(false, schema.tables.filter { it.id.name == "PRINTING_SCHEMAS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(false, schema.tables.filter { it.id.name == "REGISTER_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ScoutApiSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ScoutApiSqlExtractTest.kt index d2c0f226df..b57e7a4655 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ScoutApiSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ScoutApiSqlExtractTest.kt @@ -20,41 +20,41 @@ class ScoutApiSqlExtractTest : ExtractTestBaseH2() { assertAll(Executable { assertEquals("db_test", schema.name.lowercase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(14, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY" }, "missing table ACTIVITY") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_DERIVED" }, "missing table ACTIVITY_DERIVED") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_PROPERTIES" }, "missing table ACTIVITY_PROPERTIES") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_PROPERTIES_MEDIA_FILE" }, "missing table ACTIVITY_PROPERTIES_MEDIA_FILE") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_PROPERTIES_TAG" }, "missing table ACTIVITY_PROPERTIES_TAG") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_RATING" }, "missing table ACTIVITY_RATING") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_RELATION" }, "missing table ACTIVITY_RELATION") }, - Executable { assertTrue(schema.tables.any { it.name == "MEDIA_FILE" }, "missing table MEDIA_FILE") }, - Executable { assertTrue(schema.tables.any { it.name == "MEDIA_FILE_KEYWORDS" }, "missing table MEDIA_FILE_KEYWORDS") }, - Executable { assertTrue(schema.tables.any { it.name == "SYSTEM_MESSAGE" }, "missing table SYSTEM_MESSAGE") }, - Executable { assertTrue(schema.tables.any { it.name == "TAG" }, "missing table TAG") }, - Executable { assertTrue(schema.tables.any { it.name == "TAG_DERIVED" }, "missing table TAG_DERIVED") }, - Executable { assertTrue(schema.tables.any { it.name == "USER_IDENTITY" }, "missing table USER_IDENTITY") }, - Executable { assertTrue(schema.tables.any { it.name == "USERS" }, "missing table USERS") } + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY" }, "missing table ACTIVITY") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_DERIVED" }, "missing table ACTIVITY_DERIVED") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_PROPERTIES" }, "missing table ACTIVITY_PROPERTIES") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_PROPERTIES_MEDIA_FILE" }, "missing table ACTIVITY_PROPERTIES_MEDIA_FILE") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_PROPERTIES_TAG" }, "missing table ACTIVITY_PROPERTIES_TAG") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_RATING" }, "missing table ACTIVITY_RATING") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_RELATION" }, "missing table ACTIVITY_RELATION") }, + Executable { assertTrue(schema.tables.any { it.id.name == "MEDIA_FILE" }, "missing table MEDIA_FILE") }, + Executable { assertTrue(schema.tables.any { it.id.name == "MEDIA_FILE_KEYWORDS" }, "missing table MEDIA_FILE_KEYWORDS") }, + Executable { assertTrue(schema.tables.any { it.id.name == "SYSTEM_MESSAGE" }, "missing table SYSTEM_MESSAGE") }, + Executable { assertTrue(schema.tables.any { it.id.name == "TAG" }, "missing table TAG") }, + Executable { assertTrue(schema.tables.any { it.id.name == "TAG_DERIVED" }, "missing table TAG_DERIVED") }, + Executable { assertTrue(schema.tables.any { it.id.name == "USER_IDENTITY" }, "missing table USER_IDENTITY") }, + Executable { assertTrue(schema.tables.any { it.id.name == "USERS" }, "missing table USERS") } ) - assertTrue(schema.tables.filter { it.name == "ACTIVITY" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "ACTIVITY_PROPERTIES" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "USERS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "SYSTEM_MESSAGE" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "TAG" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "USER_IDENTITY" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "MEDIA_FILE" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "ACTIVITY" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "ACTIVITY_PROPERTIES" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "USERS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "SYSTEM_MESSAGE" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "TAG" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "USER_IDENTITY" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "MEDIA_FILE" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(5, schema.tables.filter { it.name == "USERS" }.first().columns.size) + assertEquals(5, schema.tables.filter { it.id.name == "USERS" }.first().columns.size) - assertEquals(true, schema.tables.filter { it.name == "ACTIVITY_PROPERTIES" }.first().columns.filter { it.name == "publishing_activity_id".uppercase() }.first().unique) - assertEquals(true, schema.tables.filter { it.name == "MEDIA_FILE" }.first().columns.filter { it.name == "uri".uppercase() }.first().unique) - assertEquals(true, schema.tables.filter { it.name == "SYSTEM_MESSAGE" }.first().columns.filter { it.name == "keyColumn".uppercase() }.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "ACTIVITY_PROPERTIES" }.first().columns.filter { it.name == "publishing_activity_id".uppercase() }.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "MEDIA_FILE" }.first().columns.filter { it.name == "uri".uppercase() }.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "SYSTEM_MESSAGE" }.first().columns.filter { it.name == "keyColumn".uppercase() }.first().unique) - assertEquals(2, schema.tables.filter { it.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions.size) - assertEquals("(\"AGE_MAX\" <= 100)", schema.tables.filter { it.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions[0].sqlCheckExpression) - assertEquals("(\"AGE_MIN\" <= 100)", schema.tables.filter { it.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions[1].sqlCheckExpression) + assertEquals(2, schema.tables.filter { it.id.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions.size) + assertEquals("(\"AGE_MAX\" <= 100)", schema.tables.filter { it.id.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions[0].sqlCheckExpression) + assertEquals("(\"AGE_MIN\" <= 100)", schema.tables.filter { it.id.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions[1].sqlCheckExpression) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableExtractCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableExtractCheckTest.kt index 33ebf504f9..56c5d34558 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableExtractCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableExtractCheckTest.kt @@ -18,12 +18,12 @@ class AlterTableExtractCheckTest : ExtractTestBaseMySQL() { assertEquals("test", schema.name.toLowerCase()) assertEquals(DatabaseType.MYSQL, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("people", ignoreCase = true) }) + assertTrue(schema.tables.any { it.id.name.equals("people", ignoreCase = true) }) - assertEquals(2, schema.tables.first { it.name.equals("people", ignoreCase = true) }.columns.size) + assertEquals(2, schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.columns.size) assertTrue(schema.tables.first { - it.name.equals( + it.id.name.equals( "people", ignoreCase = true ) @@ -31,12 +31,12 @@ class AlterTableExtractCheckTest : ExtractTestBaseMySQL() { assertEquals( 1, - schema.tables.first { it.name.equals("people", ignoreCase = true) }.tableCheckExpressions.size + schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.tableCheckExpressions.size ) assertEquals( "(age <= 100)", schema.tables.first { - it.name.equals( + it.id.name.equals( "people", ignoreCase = true ) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableUniqueTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableUniqueTest.kt index 85c5d9a451..9ec7d3a9e7 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableUniqueTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableUniqueTest.kt @@ -19,7 +19,7 @@ class AlterTableUniqueTest : ExtractTestBaseMySQL() { assertEquals("test", schema.name.toLowerCase()) assertEquals(DatabaseType.MYSQL, schema.databaseType) - val tableDto = schema.tables.find { it.name.equals("suppliers", ignoreCase = true) } + val tableDto = schema.tables.find { it.id.name.equals("suppliers", ignoreCase = true) } assertNotNull(tableDto) assertEquals(4, tableDto!!.columns.size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt index d24033f14d..165b124d3b 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt @@ -25,9 +25,9 @@ class CreateTableBoundedNumberTest : ExtractTestBaseMySQL() { assertEquals("test", schema.name) assertEquals(DatabaseType.MYSQL, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("BoundedNumberTable", ignoreCase = true) }) + assertTrue(schema.tables.any { it.id.name.equals("BoundedNumberTable", ignoreCase = true) }) - val columns = schema.tables.first { it.name.equals("BoundedNumberTable", ignoreCase = true) }.columns + val columns = schema.tables.first { it.id.name.equals("BoundedNumberTable", ignoreCase = true) }.columns columns.apply { assertEquals(8, size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckT1Test.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckT1Test.kt index 1fa159a507..3ceb7abd1f 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckT1Test.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckT1Test.kt @@ -18,12 +18,12 @@ class CreateTableCheckT1Test : ExtractTestBaseMySQL() { assertEquals("test", schema.name) assertEquals(DatabaseType.MYSQL, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("t1", ignoreCase = true) }) - assertEquals(3, schema.tables.first { it.name.equals("t1", ignoreCase = true) }.columns.size) + assertTrue(schema.tables.any { it.id.name.equals("t1", ignoreCase = true) }) + assertEquals(3, schema.tables.first { it.id.name.equals("t1", ignoreCase = true) }.columns.size) - assertEquals(listOf("c1", "c2", "c3"),schema.tables.first { it.name.equals("t1", ignoreCase = true) }.columns.map { it.name }) + assertEquals(listOf("c1", "c2", "c3"),schema.tables.first { it.id.name.equals("t1", ignoreCase = true) }.columns.map { it.name }) - schema.tables.first { it.name.equals("t1", ignoreCase = true) }.tableCheckExpressions.apply { + schema.tables.first { it.id.name.equals("t1", ignoreCase = true) }.tableCheckExpressions.apply { assertEquals(6, size) /* https://dev.mysql.com/doc/refman/8.0/en/create-table-check-constraints.html diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckTest.kt index c23ed8bad2..20d48c2d59 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckTest.kt @@ -18,16 +18,16 @@ class CreateTableCheckTest : ExtractTestBaseMySQL() { assertEquals("test", schema.name.toLowerCase()) assertEquals(DatabaseType.MYSQL, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("people", ignoreCase = true) }) + assertTrue(schema.tables.any { it.id.name.equals("people", ignoreCase = true) }) - assertEquals(2, schema.tables.first { it.name.equals("people", ignoreCase = true) }.columns.size) + assertEquals(2, schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.columns.size) - assertTrue(schema.tables.first { it.name.equals("people", ignoreCase = true) }.columns.any { it.name == "age" }); + assertTrue(schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.columns.any { it.name == "age" }); - assertEquals(1, schema.tables.first { it.name.equals("people", ignoreCase = true) }.tableCheckExpressions.size) + assertEquals(1, schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.tableCheckExpressions.size) assertEquals( "(age <= 100)", - schema.tables.first { it.name.equals("people", ignoreCase = true) }.tableCheckExpressions[0].sqlCheckExpression + schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.tableCheckExpressions[0].sqlCheckExpression ) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt index 8c2e02db0f..e4cf633c1f 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt @@ -18,7 +18,7 @@ class CreateTableEnumTest : ExtractTestBaseMySQL() { assertNotNull(schema) - val tableDto = schema.tables.find { it.name.equals("shirts", ignoreCase = true) } + val tableDto = schema.tables.find { it.id.name.equals("shirts", ignoreCase = true) } assertNotNull(tableDto) assertEquals(2, tableDto!!.columns.size) assertEquals(1, tableDto.tableCheckExpressions.size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableUniqueTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableUniqueTest.kt index 080f7dd84f..417b4c8ac3 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableUniqueTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableUniqueTest.kt @@ -19,7 +19,7 @@ class CreateTableUniqueTest : ExtractTestBaseMySQL() { assertEquals("test", schema.name.toLowerCase()) assertEquals(DatabaseType.MYSQL, schema.databaseType) - val tableDto = schema.tables.find { it.name.equals("suppliers", ignoreCase = true) } + val tableDto = schema.tables.find { it.id.name.equals("suppliers", ignoreCase = true) } assertNotNull(tableDto) assertEquals(4, tableDto!!.columns.size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractCheckTest.kt index d882b8d12b..a922e2ba33 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractCheckTest.kt @@ -22,14 +22,14 @@ class AlterTableExtractCheckTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name == "people" }) + assertTrue(schema.tables.any { it.id.name == "people" }) - assertEquals(2, schema.tables.first { it.name == "people" }.columns.size) + assertEquals(2, schema.tables.first { it.id.name == "people" }.columns.size) - assertTrue(schema.tables.first { it.name == "people" }.columns.any { it.name == "age" }); + assertTrue(schema.tables.first { it.id.name == "people" }.columns.any { it.name == "age" }); - assertEquals(1, schema.tables.first { it.name == "people" }.tableCheckExpressions.size) - assertEquals("(age <= 100)", schema.tables.first { it.name == "people" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "people" }.tableCheckExpressions.size) + assertEquals("(age <= 100)", schema.tables.first { it.id.name == "people" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractUniqueTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractUniqueTest.kt index bafe886e2e..33a8ce684a 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractUniqueTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractUniqueTest.kt @@ -20,16 +20,16 @@ class AlterTableExtractUniqueTest : ExtractTestBasePostgres() { assertAll(Executable { assertEquals("postgres", schema.name.toLowerCase()) }, Executable { assertEquals(DatabaseType.POSTGRES, schema.databaseType) }, Executable { assertEquals(2, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "countries" }, "missing table COUNTRIES") }, - Executable { assertTrue(schema.tables.any { it.name == "passports" }, "missing table PASSPORTS") } + Executable { assertTrue(schema.tables.any { it.id.name == "countries" }, "missing table COUNTRIES") }, + Executable { assertTrue(schema.tables.any { it.id.name == "passports" }, "missing table PASSPORTS") } ) - assertEquals(true, schema.tables.filter { it.name == "passports" }.first().columns.filter { it.name == "country_id" }.first().unique) - assertEquals(true, schema.tables.filter { it.name == "passports" }.first().columns.filter { it.name == "passport_number" }.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "passports" }.first().columns.filter { it.name == "country_id" }.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "passports" }.first().columns.filter { it.name == "passport_number" }.first().unique) - assertEquals(1, schema.tables.filter { it.name == "passports" }.first().tableCheckExpressions.size) - assertEquals("(passport_number > 0)", schema.tables.filter { it.name == "passports" }.first().tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.filter { it.id.name == "passports" }.first().tableCheckExpressions.size) + assertEquals("(passport_number > 0)", schema.tables.filter { it.id.name == "passports" }.first().tableCheckExpressions[0].sqlCheckExpression) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ArrayTypesTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ArrayTypesTest.kt index e86bc8142b..81d905d9e6 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ArrayTypesTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ArrayTypesTest.kt @@ -39,8 +39,8 @@ class ArrayTypesTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("ArrayTypes".lowercase()) }) - val table = schema.tables.find { it.name.equals("ArrayTypes".lowercase()) } + assertTrue(schema.tables.any { it.id.name.equals("ArrayTypes".lowercase()) }) + val table = schema.tables.find { it.id.name.equals("ArrayTypes".lowercase()) } assertTrue(table!!.columns.any { it.name.equals("nonArrayColumn".lowercase()) }) val nonArrayColumnDto = table.columns.find { it.name.equals("nonArrayColumn".lowercase()) }!! diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CompositeTypesTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CompositeTypesTest.kt index 3b746e72de..f59dd42312 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CompositeTypesTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CompositeTypesTest.kt @@ -73,8 +73,8 @@ class CompositeTypesTest : ExtractTestBasePostgres() { assertTrue(itemColumn.columnTypeIsComposite) - assertTrue(schema.tables.any { it.name.equals("on_hand".lowercase()) }) - val onHandTable = schema.tables.find { it.name.equals("on_hand".lowercase()) }!! + assertTrue(schema.tables.any { it.id.name.equals("on_hand".lowercase()) }) + val onHandTable = schema.tables.find { it.id.name.equals("on_hand".lowercase()) }!! assertTrue(onHandTable.columns[0].name.equals("item".lowercase())) assertTrue(onHandTable.columns[1].name.equals("count".lowercase())) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CreateTableExtractCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CreateTableExtractCheckTest.kt index 0a6c2c35ac..28ca153b5a 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CreateTableExtractCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CreateTableExtractCheckTest.kt @@ -22,14 +22,14 @@ class CreateTableExtractCheckTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name == "people" }) + assertTrue(schema.tables.any { it.id.name == "people" }) - assertEquals(2, schema.tables.first { it.name == "people" }.columns.size) + assertEquals(2, schema.tables.first { it.id.name == "people" }.columns.size) - assertTrue(schema.tables.first { it.name == "people" }.columns.any { it.name == "age" }); + assertTrue(schema.tables.first { it.id.name == "people" }.columns.any { it.name == "age" }); - assertEquals(1, schema.tables.first { it.name == "people" }.tableCheckExpressions.size) - assertEquals("(age <= 100)", schema.tables.first { it.name == "people" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "people" }.tableCheckExpressions.size) + assertEquals("(age <= 100)", schema.tables.first { it.id.name == "people" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/Ind0ExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/Ind0ExtractTest.kt index 31526c9da3..14f2771625 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/Ind0ExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/Ind0ExtractTest.kt @@ -28,26 +28,26 @@ class Ind0ExtractTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name == "x" }) - assertTrue(schema.tables.any { it.name == "y" }) + assertTrue(schema.tables.any { it.id.name == "x" }) + assertTrue(schema.tables.any { it.id.name == "y" }) - assertEquals(11, schema.tables.first { it.name == "x" }.columns.size) - assertEquals(7, schema.tables.first { it.name == "y" }.columns.size) + assertEquals(11, schema.tables.first { it.id.name == "x" }.columns.size) + assertEquals(7, schema.tables.first { it.id.name == "y" }.columns.size) - assertTrue(schema.tables.first { it.name == "x" }.columns.any { it.type.equals("xml", true) }) - assertTrue(schema.tables.first { it.name == "y" }.columns.any { it.type.equals("jsonb", true) }) + assertTrue(schema.tables.first { it.id.name == "x" }.columns.any { it.type.equals("xml", true) }) + assertTrue(schema.tables.first { it.id.name == "y" }.columns.any { it.type.equals("jsonb", true) }) - assertEquals(4, schema.tables.first { it.name == "x" }.tableCheckExpressions.size) - assertEquals(1, schema.tables.first { it.name == "y" }.tableCheckExpressions.size) + assertEquals(4, schema.tables.first { it.id.name == "x" }.tableCheckExpressions.size) + assertEquals(1, schema.tables.first { it.id.name == "y" }.tableCheckExpressions.size) // constraints on X table - assertTrue(schema.tables.first { it.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(status = ANY (ARRAY['A'::text, 'B'::text]))") }) - assertTrue(schema.tables.first { it.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("((status = 'B'::text) = (p_at IS NOT NULL))") }) - assertTrue(schema.tables.first { it.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("((f_id ~~ 'hi'::text) OR (f_id ~~ '%foo%'::text) OR (f_id ~~ '%foo%x%'::text) OR (f_id ~~ '%bar%'::text) OR (f_id ~~ '%bar%y%'::text) OR (f_id ~~ '%hello%'::text))") }) - assertTrue(schema.tables.first { it.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(w_id ~ similar_to_escape('/foo/__/bar/(left|right)/[0-9]{4}-[0-9]{2}-[0-9]{2}(/[0-9]*)?'::text))") }) + assertTrue(schema.tables.first { it.id.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(status = ANY (ARRAY['A'::text, 'B'::text]))") }) + assertTrue(schema.tables.first { it.id.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("((status = 'B'::text) = (p_at IS NOT NULL))") }) + assertTrue(schema.tables.first { it.id.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("((f_id ~~ 'hi'::text) OR (f_id ~~ '%foo%'::text) OR (f_id ~~ '%foo%x%'::text) OR (f_id ~~ '%bar%'::text) OR (f_id ~~ '%bar%y%'::text) OR (f_id ~~ '%hello%'::text))") }) + assertTrue(schema.tables.first { it.id.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(w_id ~ similar_to_escape('/foo/__/bar/(left|right)/[0-9]{4}-[0-9]{2}-[0-9]{2}(/[0-9]*)?'::text))") }) // constraints on Y table - assertTrue(schema.tables.first { it.name == "y" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(status = ANY (ARRAY['A'::text, 'B'::text, 'C'::text, 'D'::text, 'E'::text]))") }) + assertTrue(schema.tables.first { it.id.name == "y" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(status = ANY (ARRAY['A'::text, 'B'::text, 'C'::text, 'D'::text, 'E'::text]))") }) //TODO check the 3 views diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt index 70c3641859..f91b9bb1d0 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt @@ -24,7 +24,7 @@ class ManySimilarToChecksTest : ExtractTestBasePostgres() { fun testManySimilarToPatternsSchemaExtraction() { val schema = DbInfoExtractor.extract(connection) - assertEquals(2, schema.tables.first { it.name.equals("email_table", ignoreCase = true) }.tableCheckExpressions.size) + assertEquals(2, schema.tables.first { it.id.name.equals("email_table", ignoreCase = true) }.tableCheckExpressions.size) val builder = SqlInsertBuilder(schema) val emailTable = builder.getTable("email_table", useExtraConstraints = true) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ObjectIdentifierTypesTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ObjectIdentifierTypesTest.kt index 8bf843a52d..c03f3ae7cb 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ObjectIdentifierTypesTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ObjectIdentifierTypesTest.kt @@ -44,8 +44,8 @@ class ObjectIdentifierTypesTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("ObjectIdentifierTypes".lowercase()) }) - val table = schema.tables.find { it.name.equals("ObjectIdentifierTypes".lowercase()) } + assertTrue(schema.tables.any { it.id.name.equals("ObjectIdentifierTypes".lowercase()) }) + val table = schema.tables.find { it.id.name.equals("ObjectIdentifierTypes".lowercase()) } assertTrue(table!!.columns.any { it.name.equals("oidColumn".lowercase()) }) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/PgLsnTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/PgLsnTest.kt index 957eb06df6..0f576e4929 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/PgLsnTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/PgLsnTest.kt @@ -28,8 +28,8 @@ class PgLsnTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("PgLsnType".lowercase()) }) - val table = schema.tables.find { it.name.equals("PgLsnType".lowercase()) } + assertTrue(schema.tables.any { it.id.name.equals("PgLsnType".lowercase()) }) + val table = schema.tables.find { it.id.name.equals("PgLsnType".lowercase()) } assertTrue(table!!.columns.any { it.name.equals("pglsnColumn".lowercase()) }) val nonArrayColumnDto = table.columns.find { it.name.equals("pglsnColumn".lowercase()) }!! diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/SqlTextColumnTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/SqlTextColumnTest.kt index 4aa04446a6..89f1d36a0c 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/SqlTextColumnTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/SqlTextColumnTest.kt @@ -28,8 +28,8 @@ class SqlTextColumnTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("people".lowercase()) }) - val peopleTable = schema.tables.find { it.name.equals("people".lowercase()) } + assertTrue(schema.tables.any { it.id.name.equals("people".lowercase()) }) + val peopleTable = schema.tables.find { it.id.name.equals("people".lowercase()) } val idColumn = peopleTable!!.columns[0] val nameColumn = peopleTable.columns[1] diff --git a/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingConstraintsTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingConstraintsTest.kt index 1023919e4c..d18c5a1f15 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingConstraintsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingConstraintsTest.kt @@ -12,10 +12,10 @@ class ConflictingConstraintsTest : MultiDbTestBase() { val info = DbInfoExtractor.extract(connection) assertEquals(name.lowercase(), info.name.lowercase()) assertEquals(2, info.tables.size) - val first = info.tables.find { it.schema.equals("first",true) }!! - val second = info.tables.find { it.schema.equals("other",true) }!! + val first = info.tables.find { it.id.schema.equals("first",true) }!! + val second = info.tables.find { it.id.schema.equals("other",true) }!! - assertEquals(first.name, second.name) + assertEquals(first.id.name, second.id.name) assertEquals(2, first.columns.size) assertTrue(first.columns.any { it.name.equals("x", true) }) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingSchemasTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingSchemasTest.kt index b15e1b5af7..a82442e366 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingSchemasTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingSchemasTest.kt @@ -13,10 +13,10 @@ class ConflictingSchemasTest : MultiDbTestBase() { val info = DbInfoExtractor.extract(connection) assertEquals(name.lowercase(), info.name.lowercase()) assertEquals(2, info.tables.size) - val first = info.tables.find { it.schema.equals("first",true) }!! - val second = info.tables.find { it.schema.equals("other",true) }!! + val first = info.tables.find { it.id.schema.equals("first",true) }!! + val second = info.tables.find { it.id.schema.equals("other",true) }!! - assertEquals(first.name, second.name) + assertEquals(first.id.name, second.id.name) assertEquals(2, first.columns.size) assertEquals(2, second.columns.size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/multidb/SecondSchemaTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/multidb/SecondSchemaTest.kt index 985a70a9a4..047a0c0ca2 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/multidb/SecondSchemaTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/multidb/SecondSchemaTest.kt @@ -14,14 +14,14 @@ class SecondSchemaTest : MultiDbTestBase(){ val info = DbInfoExtractor.extract(connection) assertEquals(name.lowercase(), info.name.lowercase()) assertEquals(2, info.tables.size) - val foo = info.tables.find { it.name.lowercase() == "foo" }!! - val bar = info.tables.find { it.name.lowercase() == "bar" }!! + val foo = info.tables.find { it.id.name.lowercase() == "foo" }!! + val bar = info.tables.find { it.id.name.lowercase() == "bar" }!! if(databaseType == DatabaseType.MYSQL){ - assertEquals(name.lowercase(), foo.schema.lowercase()) + assertEquals(name.lowercase(), foo.id.schema.lowercase()) } else { - assertEquals("public", foo.schema.lowercase()) + assertEquals("public", foo.id.schema.lowercase()) } - assertEquals("other", bar.schema.lowercase()) + assertEquals("other", bar.id.schema.lowercase()) } override fun getSchemaLocation() = "/sql_schema/multidb/secondschema.sql" From ba60dfc23b8cb20e3a3e42a983ea80308015cd90 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 8 Jan 2025 12:34:23 +0100 Subject: [PATCH 15/21] fix for some tests --- .../java/controller/internal/SutController.java | 6 +++++- .../dto/database/schema/ExtraConstraintsDto.java | 2 +- ...HeuristicsCalculatorWithInsertionDtoTest.java | 2 ++ .../sql/internal/QueryResultTransformerTest.java | 2 ++ .../org/evomaster/core/sql/SqlInsertBuilder.kt | 16 +++++++++++----- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 02d621c34a..de2e7e983f 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -24,6 +24,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.ExtraConstraintsDto; import org.evomaster.client.java.controller.api.dto.MockDatabaseDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.controller.api.dto.problem.RPCProblemDto; import org.evomaster.client.java.controller.api.dto.problem.rpc.*; import org.evomaster.client.java.sql.DbCleaner; @@ -1455,7 +1456,10 @@ protected UnitsInfoDto getUnitsInfoDto(UnitsInfoRecorder recorder){ ec.digitsInteger = c.getDigitsInteger(); ec.enumValuesAsStrings = c.getEnumValuesAsStrings() == null ? null : new ArrayList<>(c.getEnumValuesAsStrings()); ExtraConstraintsDto jpa = new ExtraConstraintsDto(); - jpa.tableName = c.getTableName(); + jpa.tableId = new TableIdDto(); + jpa.tableId.name = c.getTableName(); + jpa.tableId.schema = null; //TODO + jpa.tableId.catalog = null; //TODO jpa.columnName = c.getColumnName(); jpa.constraints = ec; return jpa; diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/ExtraConstraintsDto.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/ExtraConstraintsDto.java index d6cf667615..ed4dcc7fab 100644 --- a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/ExtraConstraintsDto.java +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/ExtraConstraintsDto.java @@ -8,7 +8,7 @@ */ public class ExtraConstraintsDto { - public String tableName; + public TableIdDto tableId; public String columnName; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java index b069264b5c..f2e30b8650 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java @@ -4,6 +4,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.sql.DataRow; import org.evomaster.client.java.sql.QueryResult; import org.junit.jupiter.api.Test; @@ -20,6 +21,7 @@ public class HeuristicsCalculatorWithInsertionDtoTest { private DbInfoDto createSchemaDtoWithFooTableAndXColumn(String xDataType){ DbInfoDto schemaDto = new DbInfoDto(); TableDto tableDto = new TableDto(); + tableDto.id = new TableIdDto(); tableDto.id.name = "Foo"; ColumnDto dto = new ColumnDto(); dto.name = "x"; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java index b1994d7b03..b1df973402 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java @@ -4,6 +4,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.sql.QueryResult; import org.junit.jupiter.api.Test; @@ -23,6 +24,7 @@ public class QueryResultTransformerTest { private TableDto createTableDate(List columnTypes, List columnNames, String tableName){ assertEquals(columnTypes.size(), columnNames.size()); TableDto tableDto = new TableDto(); + tableDto.id = new TableIdDto(); tableDto.id.name = tableName; for (int i = 0; i < columnTypes.size(); i++){ ColumnDto dto = new ColumnDto(); diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 49f870a523..6a7517b16f 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -130,11 +130,11 @@ class SqlInsertBuilder( private fun validateExtraConstraintsHandle(schemaDto: DbInfoDto) { schemaDto.extraConstraintDtos.forEach { extraConstraintsDto -> - val table = tables.values.find { matchJpaName(it.name, extraConstraintsDto.tableName) } + val table = tables.values.find { matchJpaName(it.name, extraConstraintsDto.tableId.name) } if (table == null) { LoggingUtil.uniqueWarn( log, "Handling of extra constraints failed." + - " There is no SQL table called ${extraConstraintsDto.tableName}" + " There is no SQL table called ${extraConstraintsDto.tableId.name}" ) assert(false) } else { @@ -157,7 +157,7 @@ class SqlInsertBuilder( table: Table ): Table { val extrasForTable = schemaDto.extraConstraintDtos - .filter { matchJpaName(it.tableName, table.name) } + .filter { matchJpaName(it.tableId.name, table.name) } val columns = table.columns .map { column -> @@ -304,10 +304,16 @@ class SqlInsertBuilder( so, if match fails, we try again without _ */ private fun matchJpaName(original: String, jpaDefaultMadness: String): Boolean { - if (original.equals(jpaDefaultMadness, true)) { + /* + FIXME this will need to be refactored once we introduce TableId + */ + val name = if(original.contains(".")) original.substring(original.lastIndexOf(".") + 1, original.length) + else original + + if (name.equals(jpaDefaultMadness, true)) { return true } - return original.replace("_", "").equals(jpaDefaultMadness.replace("_", ""), true) + return name.replace("_", "").equals(jpaDefaultMadness.replace("_", ""), true) } private fun mergeConstraints(column: Column, extra: ExtraConstraintsDto): Column { From cd98db970a9cb0d9e33aa7aa7d6c026f4f4c86d4 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 8 Jan 2025 12:56:14 +0100 Subject: [PATCH 16/21] introducing TableId --- .../core/solver/SMTLibZ3DbConstraintSolver.kt | 7 +-- .../org/evomaster/core/sql/SqlAction.kt | 48 ++-------------- .../core/sql/SqlActionTransformer.kt | 2 +- .../evomaster/core/sql/SqlInsertBuilder.kt | 5 +- .../org/evomaster/core/sql/schema/Table.kt | 12 ++-- .../org/evomaster/core/sql/schema/TableId.kt | 57 +++++++++++++++++++ .../kotlin/org/evomaster/core/TestUtils.kt | 3 +- 7 files changed, 78 insertions(+), 56 deletions(-) create mode 100644 core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt diff --git a/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt b/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt index 8c44924e46..040cd44a22 100644 --- a/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt +++ b/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt @@ -14,10 +14,7 @@ import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.numeric.LongGene import org.evomaster.core.search.gene.string.StringGene import org.evomaster.core.sql.SqlAction -import org.evomaster.core.sql.schema.Column -import org.evomaster.core.sql.schema.ColumnDataType -import org.evomaster.core.sql.schema.ForeignKey -import org.evomaster.core.sql.schema.Table +import org.evomaster.core.sql.schema.* import org.evomaster.solver.Z3DockerExecutor import org.evomaster.solver.smtlib.SMTLib import org.evomaster.solver.smtlib.value.* @@ -170,7 +167,7 @@ class SMTLibZ3DbConstraintSolver( val tableDto = schema.tables.find { it.id.name.equals(tableName, ignoreCase = true) } ?: throw RuntimeException("Table not found: $tableName") return Table( - tableDto.id.name, + TableId(tableDto.id.name) , //TODO other info, eg schema findColumns(tableDto), // Convert columns from DTO findForeignKeys(tableDto) // TODO: Implement this method ) diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt index 0dffdc43dc..c2e671f9bc 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt @@ -45,39 +45,6 @@ class SqlAction( */ val representExistingData: Boolean = false, - /** - * An id representing the connection to the database process. - * This could be the connecting URL. - * This is needed when dealing with multiple connection to different database processes, - * possibly running on different machines. - * If left null, default connection will be used (ie, assume there is only one). - */ - val connectionId: String? = null, - - /** - * At a high-level this represents a "catalog". - * Ie, a "logical" database instance. - * Note that a database process could have several logical databases. - * We do not use the term "catalog" though as Postgres and MySQL behave differently. - * In MySQL, can access different catalogs on same connection. - * This is not possible on Postgres. - * However, this latter has "schemas", which MySQL does not. - * In other words: - * - MySQL: this will always be empty - * - Postgres: this would be the catalog name. - * However, as in Postgres we cannot change catalog within same connection, the important info is in the - * [connectionId] entry. - * Even if we were to access 2 different catalogs in the same database process, we would need 2 different connections. - * As such, this parameter is technically redundant. - */ - val sealedGroupName: String? = null, - - /** - * In Postgres this represents a "schema", whereas it is a "catalog" for MySQL. - * In other words, this is used to group tables that can be indexed by this group name for disambiguation. - */ - val openGroupName: String? = null, - ) : EnvironmentAction(listOf()) { init { @@ -98,12 +65,6 @@ class SqlAction( } } - fun getFullQualifyingTableName() : String{ - if(openGroupName.isNullOrBlank()){ - return table.name - } - return "$openGroupName.${table.name}" - } private val genes: List = (computedGenes ?: selectedColumns.map { SqlActionGeneBuilder().buildGene(id, table, it) } @@ -136,9 +97,10 @@ class SqlAction( } override fun getName(): String { - val c = if(connectionId.isNullOrEmpty()) "" else "_$connectionId" - val s = if(openGroupName.isNullOrEmpty()) "" else "_$openGroupName" - val t = "_${table.name}" + val id = table.id + val c = if(id.connectionId.isNullOrEmpty()) "" else "_${id.connectionId}" + val s = if(id.openGroupName.isNullOrEmpty()) "" else "_${id.openGroupName}" + val t = "_${id.name}" return "SQL_Insert$c$s${t}_${selectedColumns.map { it.name }.sorted().joinToString("_")}" } @@ -155,7 +117,7 @@ class SqlAction( } override fun copyContent(): Action { - return SqlAction(table, selectedColumns, id, genes.map(Gene::copy), representExistingData, connectionId, sealedGroupName, openGroupName) + return SqlAction(table, selectedColumns, id, genes.map(Gene::copy), representExistingData) } fun geInsertionId(): Long { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt index 29277d4448..5f459d3efd 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt @@ -35,7 +35,7 @@ object SqlActionTransformer { } - val insertion = InsertionDto().apply { targetTable = action.getFullQualifyingTableName() } + val insertion = InsertionDto().apply { targetTable = action.table.id.getFullQualifyingTableName() } for (g in action.seeTopGenes()) { if (g is SqlPrimaryKeyGene) { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 6a7517b16f..9459deb8c3 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -33,7 +33,7 @@ class SqlInsertBuilder( */ /** - * Information about tables, indexed by name + * Information about tables, indexed by id */ private val tables = mutableMapOf() @@ -112,7 +112,8 @@ class SqlInsertBuilder( for (tableDto in schemaDto.tables) { val id = SqlDtoUtils.getId(tableDto) val table = Table( - id, + //FIXME MySQL vs Postgres + TableId(tableDto.id.name, /*TODO*/ null, tableDto.id.catalog, tableDto.id.schema), tableToColumns[id]!!, tableToForeignKeys[id]!!, tableToConstraints[id]!! diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt index 93497032c7..f0192457cf 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt @@ -7,10 +7,8 @@ import org.evomaster.dbconstraint.TableConstraint * Should be immutable */ data class Table( - /** - * This usually would be fully qualified, ie, including schema - */ - val name: String, + + val id: TableId, val columns: Set, @@ -22,6 +20,12 @@ data class Table( */ val tableConstraints: Set = setOf() ){ + constructor(name: String, columns: Set, foreignKeys: Set, tableConstraints: Set = setOf()) + : this(TableId(name), columns, foreignKeys, tableConstraints) + + @Deprecated("Use id directly instead") + val name: String + get() = id.name fun primaryKeys() = columns.filter { it.primaryKey } } \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt new file mode 100644 index 0000000000..23f8169de3 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt @@ -0,0 +1,57 @@ +package org.evomaster.core.sql.schema + +data class TableId( + + /** + * The name of table + */ + val name: String, + + /** + * An id representing the connection to the database process. + * This could be the connecting URL. + * This is needed when dealing with multiple connection to different database processes, + * possibly running on different machines. + * If left null, default connection will be used (ie, assume there is only one). + */ + val connectionId: String? = null, + + /** + * At a high-level this represents a "catalog". + * Ie, a "logical" database instance. + * Note that a database process could have several logical databases. + * We do not use the term "catalog" though as Postgres and MySQL behave differently. + * In MySQL, can access different catalogs on same connection. + * This is not possible on Postgres. + * However, this latter has "schemas", which MySQL does not. + * In other words: + * - MySQL: this will always be empty + * - Postgres: this would be the catalog name. + * However, as in Postgres we cannot change catalog within same connection, the important info is in the + * [connectionId] entry. + * Even if we were to access 2 different catalogs in the same database process, we would need 2 different connections. + * As such, this parameter is technically redundant. + */ + val sealedGroupName: String? = null, + + /** + * In Postgres this represents a "schema", whereas it is a "catalog" for MySQL. + * In other words, this is used to group tables that can be indexed by this group name for disambiguation. + */ + val openGroupName: String? = null, +){ + + init { + if(name.contains(".")){ + throw IllegalArgumentException("Name contains a '.'. " + + "You sure you passed a simple name and not a fully qualified one? Value: $name") + } + } + + fun getFullQualifyingTableName() : String{ + if(openGroupName.isNullOrBlank()){ + return name + } + return "$openGroupName.${name}" + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/TestUtils.kt b/core/src/test/kotlin/org/evomaster/core/TestUtils.kt index b03910bee1..a46946016d 100644 --- a/core/src/test/kotlin/org/evomaster/core/TestUtils.kt +++ b/core/src/test/kotlin/org/evomaster/core/TestUtils.kt @@ -18,6 +18,7 @@ import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.string.StringGene import org.evomaster.core.search.gene.sql.SqlForeignKeyGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene +import org.evomaster.core.sql.schema.TableId object TestUtils { @@ -63,7 +64,7 @@ object TestUtils { // for rest problem fun generateFakeDbAction(pkId : Long, pkGeneUniqueId: Long, tableName : String = "Foo", intValue : Int =0, fkColumn : Column?=null, fkGene: SqlForeignKeyGene? = null) : SqlAction { val fooId = Column("Id", ColumnDataType.INTEGER, 10, primaryKey = true, databaseType = DatabaseType.H2) - val foo = Table(tableName, setOf(fooId), setOf()) + val foo = Table(TableId(tableName), setOf(fooId), setOf()) val integerGene = IntegerGene(fooId.name, intValue) val pkFoo = SqlPrimaryKeyGene(fooId.name, "Foo", integerGene, pkGeneUniqueId) if(fkColumn != null && fkGene != null) From cdb2a64d913c904720d87a2b3ed0a7e6b9885445 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 8 Jan 2025 14:38:59 +0100 Subject: [PATCH 17/21] starting refactoring for TableId --- .../core/output/service/TestSuiteWriter.kt | 9 +++- .../enterprise/service/EnterpriseSampler.kt | 5 +- .../problem/util/inference/SimpleDeriveR2T.kt | 4 +- .../core/search/gene/sql/SqlForeignKeyGene.kt | 5 +- .../core/search/gene/sql/SqlPrimaryKeyGene.kt | 3 +- .../evomaster/core/search/service/Sampler.kt | 6 --- .../core/sql/SqlActionGeneBuilder.kt | 2 +- .../org/evomaster/core/sql/SqlActionUtils.kt | 4 +- .../evomaster/core/sql/SqlInsertBuilder.kt | 50 +++++++++---------- .../evomaster/core/sql/schema/ForeignKey.kt | 2 +- .../org/evomaster/core/sql/schema/TableId.kt | 26 ++++++++++ .../individual/RestIndividualStructureTest.kt | 2 +- .../evomaster/core/sql/DbActionUtilsTest.kt | 21 ++++---- .../DbDirectIntWithSqlEMTest.java | 3 +- .../e2etests/spring/rest/basic/BasicEMTest.kt | 3 +- .../DbDirectIntWithSqlEMTest.java | 3 +- 16 files changed, 87 insertions(+), 61 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 4753b28ae1..8ebd96c9b4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -12,6 +12,7 @@ import org.evomaster.core.output.TestWriterUtils.handleDefaultStubForAsJavaOrKot import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy import org.evomaster.core.output.naming.TestCaseNamingStrategyFactory import org.evomaster.core.problem.api.ApiWsIndividual +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler import org.evomaster.core.problem.externalservice.httpws.HttpWsExternalService import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceAction import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler @@ -21,6 +22,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Solution import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.sql.schema.TableId import org.evomaster.test.utils.EMTestUtils import org.evomaster.test.utils.SeleniumEMUtils import org.evomaster.test.utils.js.JsLoader @@ -191,10 +193,13 @@ class TestSuiteWriter { } private fun handleResetDatabaseInput(solution: Solution<*>): String { + if(sampler !is EnterpriseSampler<*>){ + throw IllegalArgumentException("Not dealing with an enterprise application") + } if (!config.outputFormat.isJavaOrKotlin()) throw IllegalStateException("DO NOT SUPPORT resetDatabased for " + config.outputFormat) - val accessedTable = mutableSetOf() + val accessedTable = mutableSetOf() solution.individuals.forEach { e -> //TODO will need to be refactored when supporting Web Frontend if (e.individual is ApiWsIndividual) { @@ -206,7 +211,7 @@ class TestSuiteWriter { accessedTable.addAll(de.deletedData) } } - val all = sampler.extractFkTables(accessedTable) + val all = (sampler as EnterpriseSampler).extractFkTables(accessedTable) val tableNamesInSchema = remoteController.getCachedSutInfo() ?.sqlSchemaDto diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt index 04e3dd581a..b9d81618b0 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt @@ -11,6 +11,7 @@ import org.evomaster.core.remote.SutProblemException import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Individual import org.evomaster.core.search.service.Sampler +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -53,7 +54,7 @@ abstract class EnterpriseSampler : Sampler() where T : Individual { } } - fun sampleSqlInsertion(tableName: String, columns: Set): List { + fun sampleSqlInsertion(tableName: TableId, columns: Set): List { val extraConstraints = randomness.nextBoolean(apc.getExtraSqlDbConstraintsProbability()) val enableSingleInsertionForTable = randomness.nextBoolean(config.probOfEnablingSingleInsertionForTable) @@ -96,7 +97,7 @@ abstract class EnterpriseSampler : Sampler() where T : Individual { } } - override fun extractFkTables(tables: Set): Set { + fun extractFkTables(tables: Set): Set { if(sqlInsertBuilder == null || tables.isEmpty()) return tables return sqlInsertBuilder!!.extractFkTable(tables) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index 9b91a085b8..f1649a039e 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -220,12 +220,12 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { .map { Pair(it.name, if(ParamUtil.isGeneralName(it.name)) - t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName(paramName, "${f.targetTable}${it.name}")} + t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName(paramName, "${f.targetTableId}${it.name}")} .plus(calculateStringSimilarityScoreWithTableName(paramName, it.name)) .plus(calculateStringSimilarityScoreWithTableName(paramName, "$tableName${it.name}")) .asSequence().sorted().last() else if(ParamUtil.isGeneralName(paramName)) - t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName("${f.targetTable}$paramName", it.name)} + t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName("${f.targetTableId}$paramName", it.name)} .plus(calculateStringSimilarityScoreWithTableName(paramName, it.name)) .plus(calculateStringSimilarityScoreWithTableName("$tableName$paramName", it.name)) .asSequence().sorted().last() diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt index 42883e4ad9..a884f2143e 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt @@ -9,6 +9,7 @@ import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy +import org.evomaster.core.sql.schema.TableId /** * A gene specifically designed to handle Foreign Keys in SQL databases. @@ -25,9 +26,9 @@ class SqlForeignKeyGene( sourceColumn: String, val uniqueId: Long, /** - * The name of the table this FK points to + * The id of the table this FK points to */ - val targetTable: String, + val targetTable: TableId, val nullable: Boolean, /** * A negative value means this FK is not bound yet. diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt index c4d807bc68..c766d173f1 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt @@ -10,6 +10,7 @@ import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -19,7 +20,7 @@ import org.slf4j.LoggerFactory * This is important to check Foreign Keys referencing it. */ class SqlPrimaryKeyGene(name: String, - val tableName: String, + val tableName: TableId, val gene: Gene, /** * Important for the Foreign Keys referencing it. diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt index e19726d391..21b46d654d 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt @@ -195,12 +195,6 @@ abstract class Sampler : TrackOperator where T : Individual { } } - /** - * extract tables with additional FK tables - */ - open fun extractFkTables(tables: Set): Set{ - throw IllegalStateException("FK tables have not been not handled yet") - } /** * Return a list of pre-written individuals that will be added in the final solution. diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt index 4fe6afdd83..d65c7eb55e 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt @@ -54,7 +54,7 @@ class SqlActionGeneBuilder { column.autoIncrement -> SqlAutoIncrementGene(column.name) fk != null -> - SqlForeignKeyGene(column.name, id, fk.targetTable, column.nullable) + SqlForeignKeyGene(column.name, id, fk.targetTableId, column.nullable) else -> when (column.type) { // Man: TODO need to check diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt index 202a5e7da3..769e693bd4 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt @@ -432,8 +432,8 @@ object SqlActionUtils { val sorted = table.sortedWith( Comparator { o1, o2 -> when { - o1.foreignKeys.any { t-> t.targetTable.equals(o2.name,ignoreCase = true) } -> -1 - o2.foreignKeys.any { t-> t.targetTable.equals(o1.name,ignoreCase = true) } -> 1 + o1.foreignKeys.any { t-> t.targetTableId == o2.id } -> -1 + o2.foreignKeys.any { t-> t.targetTableId == o1.id } -> 1 else -> 0 } } diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 9459deb8c3..6653f99dea 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -35,12 +35,12 @@ class SqlInsertBuilder( /** * Information about tables, indexed by id */ - private val tables = mutableMapOf() + private val tables = mutableMapOf() /** * Same as table, but possibly with extended info on constraints, derived from analyzing the SUT */ - private val extendedTables = mutableMapOf() + private val extendedTables = mutableMapOf() private val compositeTypes = mutableMapOf() @@ -87,33 +87,33 @@ class SqlInsertBuilder( compositeTypes[compositeTypeDto.name] = CompositeType(compositeTypeDto.name, columns) } - val tableToColumns = mutableMapOf>() - val tableToForeignKeys = mutableMapOf>() - val tableToConstraints = mutableMapOf>() + val tableToColumns = mutableMapOf>() + val tableToForeignKeys = mutableMapOf>() + val tableToConstraints = mutableMapOf>() // Then load all constraints and columns for (tableDto in schemaDto.tables) { val tableConstraints = parseTableConstraints(tableDto).toMutableList() val columns = generateColumnsFrom(tableDto, tableConstraints, schemaDto) - - tableToConstraints[SqlDtoUtils.getId(tableDto)] = tableConstraints.toSet() - tableToColumns[SqlDtoUtils.getId(tableDto)] = columns + val id = TableId.fromDto(databaseType, tableDto.id) + tableToConstraints[id] = tableConstraints.toSet() + tableToColumns[id] = columns } // After all columns are loaded, we can load foreign keys for (tableDto in schemaDto.tables) { - tableToForeignKeys[SqlDtoUtils.getId(tableDto)] = calculateForeignKeysFrom(tableDto, tableToColumns) + val id = TableId.fromDto(databaseType, tableDto.id) + tableToForeignKeys[id] = calculateForeignKeysFrom(tableDto, tableToColumns) } // Now we can create the tables for (tableDto in schemaDto.tables) { - val id = SqlDtoUtils.getId(tableDto) + val id = TableId.fromDto(schemaDto.databaseType, tableDto.id) val table = Table( - //FIXME MySQL vs Postgres - TableId(tableDto.id.name, /*TODO*/ null, tableDto.id.catalog, tableDto.id.schema), + id, tableToColumns[id]!!, tableToForeignKeys[id]!!, tableToConstraints[id]!! @@ -175,13 +175,14 @@ class SqlInsertBuilder( private fun calculateForeignKeysFrom( tableDto: TableDto, - tableToColumns: MutableMap> + tableToColumns: MutableMap> ): MutableSet { val fks = mutableSetOf() for (fk in tableDto.foreignKeys) { val tableKey = SqlActionUtils.getTableKey(tableToColumns.keys, fk.targetTable) + FIXME if(tableKey == null || tableToColumns[tableKey] == null) { throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}") @@ -641,14 +642,14 @@ class SqlInsertBuilder( - fun getTable(tableName: String, useExtraConstraints: Boolean): Table { + fun getTable(tableName: TableId, useExtraConstraints: Boolean): Table { val data = if (useExtraConstraints) extendedTables else tables - val tableNameKey = SqlActionUtils.getTableKey(data.keys, tableName) - ?: throw IllegalArgumentException("No table called $tableName") +// val tableNameKey = SqlActionUtils.getTableKey(data.keys, tableName) +// ?: throw IllegalArgumentException("No table called $tableName") - return data[tableNameKey] ?: throw IllegalArgumentException("No table called $tableName") + return data[tableName] ?: throw IllegalArgumentException("No table called $tableName") } /** @@ -670,7 +671,7 @@ class SqlInsertBuilder( * test cases manually for EM */ fun createSqlInsertionAction( - tableName: String, + tableName: TableId, /** * Which columns to create data for. Default is all, ie *. * Notice that more columns might be added, eg, to satisfy non-null @@ -680,7 +681,7 @@ class SqlInsertBuilder( /** * used to avoid infinite recursion */ - history: MutableList = mutableListOf(), + history: MutableList = mutableListOf(), /** * When adding new insertions due to FK constraints, specify if * should get all columns for those new insertions, or just the minimal @@ -730,7 +731,7 @@ class SqlInsertBuilder( for (fk in table.foreignKeys) { - val targetTable = fk.targetTable + val targetTable = fk.targetTableId // if we have already generated the sql inserts for the target table more than 3 times and all columns are nullable, then skip it val maxIter = 3 // TODO: as a configurable parameter in EMConfig? @@ -999,7 +1000,7 @@ class SqlInsertBuilder( * @param tables to check * @param all is a complete set of tables with their fk */ - fun extractFkTable(tables: Set, all: MutableSet = mutableSetOf()): Set { + fun extractFkTable(tables: Set, all: MutableSet = mutableSetOf()): Set { tables.forEach { t -> if (!all.contains(t)) all.add(t) @@ -1011,13 +1012,10 @@ class SqlInsertBuilder( return all.toSet() } - private fun extractFkTable(tableName: String): Set { + private fun extractFkTable(tableId: TableId): Set { return tables.filter { t -> t.value.foreignKeys.any { f -> - f.targetTable.equals( - tableName, - ignoreCase = true - ) + f.targetTableId == tableId } }.keys } diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/ForeignKey.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/ForeignKey.kt index b9c6d7c004..6a94623786 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/schema/ForeignKey.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/ForeignKey.kt @@ -9,5 +9,5 @@ data class ForeignKey( val sourceColumns: Set, - val targetTable: String + val targetTableId: TableId ) \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt index 23f8169de3..865b750f76 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt @@ -1,5 +1,11 @@ package org.evomaster.core.sql.schema +import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto + +/** + * MUST be kept immutable + */ data class TableId( /** @@ -48,6 +54,26 @@ data class TableId( } } + companion object{ + + fun fromDto(type: DatabaseType, dto: TableIdDto) : TableId{ + + val connectionId = null //TODO + + return when(type){ + DatabaseType.POSTGRES, DatabaseType.H2 -> + TableId(dto.name, connectionId, dto.catalog, dto.schema) + + DatabaseType.MYSQL, DatabaseType.MARIADB -> + TableId(dto.name, connectionId, null, dto.schema ?: dto.catalog) + + else -> + TableId(dto.name, connectionId, dto.catalog, dto.schema) + } + } + } + + fun getFullQualifyingTableName() : String{ if(openGroupName.isNullOrBlank()){ return name diff --git a/core/src/test/kotlin/org/evomaster/core/search/structuralelement/individual/RestIndividualStructureTest.kt b/core/src/test/kotlin/org/evomaster/core/search/structuralelement/individual/RestIndividualStructureTest.kt index 26226f65c3..5991ee92a9 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/structuralelement/individual/RestIndividualStructureTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/structuralelement/individual/RestIndividualStructureTest.kt @@ -42,7 +42,7 @@ class RestIndividualStructureTest : StructuralElementBaseTest(){ unique = false, databaseType = DatabaseType.H2) - val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTable = foo.name) + val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTableId = foo.id) val bar = Table("Bar", setOf(barIdColumn, fkColumn), setOf(foreignKey)) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt index 06af4490f4..c268540a44 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt @@ -1,10 +1,6 @@ package org.evomaster.core.sql import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType -import org.evomaster.core.sql.schema.Column -import org.evomaster.core.sql.schema.ColumnDataType -import org.evomaster.core.sql.schema.ForeignKey -import org.evomaster.core.sql.schema.Table import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.problem.enterprise.SampleType import org.evomaster.core.search.gene.* @@ -14,6 +10,7 @@ import org.evomaster.core.search.gene.sql.SqlForeignKeyGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.evomaster.core.search.gene.string.StringGene import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.schema.* import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -415,7 +412,7 @@ class DbActionUtilsTest { unique = false, databaseType = DatabaseType.H2) - val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTable = table0.name) + val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTableId = table0.id) val table1 = Table("Table1", setOf(fkColumn), setOf(foreignKey)) @@ -427,7 +424,7 @@ class DbActionUtilsTest { val action0 = SqlAction(table0, setOf(idColumn), insertId0, listOf(pkGeneTable0)) val insertId1 = 1002L - val fkGene = SqlForeignKeyGene("Id", insertId1, "Table0", false, insertId0) + val fkGene = SqlForeignKeyGene("Id", insertId1, TableId("Table0"), false, insertId0) val pkGeneTable1 = SqlPrimaryKeyGene("Id", "Table1", fkGene, insertId1) val action1 = SqlAction(table1, setOf(fkColumn), insertId1, listOf(pkGeneTable1)) @@ -459,7 +456,7 @@ class DbActionUtilsTest { unique = false, databaseType = DatabaseType.H2) - val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTable = table0.name) + val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTableId = table0.id) val table1 = Table("Table1", setOf(fkColumn), setOf(foreignKey)) @@ -470,7 +467,7 @@ class DbActionUtilsTest { val action0 = SqlAction(table0, setOf(idColumn), insertId0, listOf(pkGeneTable0)) val insertId1 = 1002L - val fkGene = SqlForeignKeyGene("Id", insertId1, "Table0", false, insertId0) + val fkGene = SqlForeignKeyGene("Id", insertId1, TableId("Table0"), false, insertId0) val pkGeneTable1 = SqlPrimaryKeyGene("Id", "Table1", fkGene, insertId1) val action1 = SqlAction(table1, setOf(fkColumn), insertId1, listOf(pkGeneTable1)) @@ -501,7 +498,7 @@ class DbActionUtilsTest { unique = false, databaseType = DatabaseType.H2) - val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTable = table0.name) + val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTableId = table0.id) val table1 = Table("Table1", setOf(fkColumn), setOf(foreignKey)) @@ -518,12 +515,12 @@ class DbActionUtilsTest { val insertId2 = 1003L - val fkGene0 = SqlForeignKeyGene("Id", insertId2, "Table0", false, insertId0) + val fkGene0 = SqlForeignKeyGene("Id", insertId2, TableId("Table0"), false, insertId0) val pkGene2 = SqlPrimaryKeyGene("Id", "Table1", fkGene0, insertId2) val action2 = SqlAction(table1, setOf(fkColumn), insertId2, listOf(pkGene2)) val insertId3 = 1003L - val fkGene1 = SqlForeignKeyGene("Id", insertId3, "Table0", false, insertId0) + val fkGene1 = SqlForeignKeyGene("Id", insertId3, TableId("Table0"), false, insertId0) val pkGene3 = SqlPrimaryKeyGene("Id", "Table1", fkGene1, insertId3) val action3 = SqlAction(table1, setOf(fkColumn), insertId3, listOf(pkGene3)) @@ -557,7 +554,7 @@ class DbActionUtilsTest { autoIncrement = false, unique = false, databaseType = DatabaseType.H2) - val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTable = table0.name) + val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTableId = table0.id) val table1 = Table("Table1", setOf(fkColumn), setOf(foreignKey)) val set = listOf(table0, table0, table1) diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java index bc1b658c74..d676f80a09 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java @@ -18,6 +18,7 @@ import org.evomaster.core.search.gene.numeric.IntegerGene; import org.evomaster.core.search.service.FitnessFunction; import org.evomaster.ci.utils.CIUtils; +import org.evomaster.core.sql.schema.TableId; import org.junit.jupiter.api.Test; import java.util.Collections; @@ -139,7 +140,7 @@ public void testSteps() { //now, try to execute an action in which as well we add SQL data - List insertions = sampler.sampleSqlInsertion("DB_DIRECT_INT_ENTITY", Collections.singleton("*")); + List insertions = sampler.sampleSqlInsertion(new TableId("DB_DIRECT_INT_ENTITY",null,null,null), Collections.singleton("*")); assertEquals(1, insertions.size()); //extract the x/y values from the random call diff --git a/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt b/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt index 5209320bd7..ed5ecc319e 100644 --- a/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt +++ b/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt @@ -5,6 +5,7 @@ import org.evomaster.core.problem.rest.HttpVerb import org.evomaster.core.problem.rest.service.RestSampler import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.numeric.LongGene +import org.evomaster.core.sql.schema.TableId import org.evomaster.e2etests.utils.RestTestBase import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeAll @@ -94,7 +95,7 @@ class BasicEMTest : RestTestBase() { val injector = init(args) val sampler = injector.getInstance(RestSampler::class.java) - val dbactions = sampler.sampleSqlInsertion("X", setOf("*")) + val dbactions = sampler.sampleSqlInsertion(TableId("X"), setOf("*")) assertEquals(1, dbactions.size) diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java index 24619b0bba..d056c769fe 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java @@ -19,6 +19,7 @@ import org.evomaster.core.search.gene.numeric.IntegerGene; import org.evomaster.core.search.service.FitnessFunction; import org.evomaster.ci.utils.CIUtils; +import org.evomaster.core.sql.schema.TableId; import org.junit.jupiter.api.Test; import java.util.Collections; @@ -140,7 +141,7 @@ public void testSteps() { //now, try to execute an action in which as well we add SQL data - List insertions = sampler.sampleSqlInsertion("DB_DIRECT_INT_ENTITY", Collections.singleton("*")); + List insertions = sampler.sampleSqlInsertion(new TableId("DB_DIRECT_INT_ENTITY",null,null,null), Collections.singleton("*")); assertEquals(1, insertions.size()); //extract the x/y values from the random call From ec435838ed0fa354865974b854e94e9fbd14a6a0 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 15 Jan 2025 15:15:12 +0100 Subject: [PATCH 18/21] more on TableId --- .../enterprise/EnterpriseIndividual.kt | 5 +-- .../evomaster/core/sql/DatabaseExecution.kt | 33 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/EnterpriseIndividual.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/EnterpriseIndividual.kt index 94e518b14d..20906c1355 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/EnterpriseIndividual.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/EnterpriseIndividual.kt @@ -13,6 +13,7 @@ import org.evomaster.core.search.gene.Gene import org.evomaster.core.search.gene.utils.GeneUtils import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.tracer.TrackOperator +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import java.util.* @@ -391,8 +392,8 @@ abstract class EnterpriseIndividual( /** * @return a list table names which are used to insert data directly */ - open fun getInsertTableNames(): List{ - return sqlInitialization.filterNot { it.representExistingData }.map { it.table.name } + open fun getInsertTableNames(): List{ + return sqlInitialization.filterNot { it.representExistingData }.map { it.table.id } } override fun seeTopGenes(filter: ActionFilter): List { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt index ba9863d954..0f38de640d 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt @@ -1,6 +1,7 @@ package org.evomaster.core.sql import org.evomaster.client.java.controller.api.dto.database.execution.SqlExecutionsDto +import org.evomaster.core.sql.schema.TableId /** * When a test case is executed, and the SUT does access a SQL database, @@ -16,11 +17,11 @@ import org.evomaster.client.java.controller.api.dto.database.execution.SqlExecut * This class MUST be immutable */ class DatabaseExecution( - val queriedData: Map>, - val updatedData: Map>, - val insertedData: Map>, - val failedWhere: Map>, - val deletedData: List, + val queriedData: Map>, + val updatedData: Map>, + val insertedData: Map>, + val failedWhere: Map>, + val deletedData: List, val numberOfSqlCommands: Int, val sqlParseFailureCount: Int, val executionInfo: List @@ -38,15 +39,15 @@ class DatabaseExecution( validateQuotes(deletedData) } - private fun validateQuotes(data: Map>) { - val quoted = data.keys.filter { it.startsWith("\"") } + private fun validateQuotes(data: Map>) { + val quoted = data.keys.filter { it.name.startsWith("\"") } if(quoted.isNotEmpty()) { throw IllegalArgumentException("Following table names are quoted: ${quoted.joinToString(", ")}") } } - private fun validateQuotes(data: List) { - val quoted = data.filter { it.startsWith("\"") } + private fun validateQuotes(data: List) { + val quoted = data.filter { it.name.startsWith("\"") } if(quoted.isNotEmpty()) { throw IllegalArgumentException("Following table names are quoted: ${quoted.joinToString(", ")}") } @@ -66,6 +67,8 @@ class DatabaseExecution( might depend on library used to analyze the SQL commands. */ + FIXME + return DatabaseExecution( cloneData(dealWithQuotes(dto?.queriedData)), cloneData(dealWithQuotes(dto?.updatedData)), @@ -92,11 +95,11 @@ class DatabaseExecution( return data } - private fun dealWithQuotes(data: Map>?): Map>? { + private fun dealWithQuotes(data: Map>?): Map>? { if(data == null) return null return data.entries.associate { - (if(it.key.startsWith("\"")) removeQuotes(it.key) else it.key) to it.value + (if(it.key.name.startsWith("\"")) removeQuotes(it.key) else it.key) to it.value } } @@ -124,8 +127,8 @@ class DatabaseExecution( return data.map { SqlExecutionInfo(it.sqlCommand, it.threwSqlExeception, it.executionTime) } } - private fun cloneData(data: Map>?): Map> { - val clone = mutableMapOf>() + private fun cloneData(data: Map>?): Map> { + val clone = mutableMapOf>() data?.keys?.forEach { clone[it] = data[it]!!.toSet() @@ -133,8 +136,8 @@ class DatabaseExecution( return clone } - private fun merge(current: MutableMap>, - toAdd: Map>) { + private fun merge(current: MutableMap>, + toAdd: Map>) { for (e in toAdd.entries) { val key = e.key From dbd2e903e471673b1b7e9ad3234c6d6e17393b8b Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 17 Jan 2025 15:37:54 +0100 Subject: [PATCH 19/21] more refactoring --- .../core/output/service/TestSuiteWriter.kt | 15 ++-- .../api/service/ApiWsStructureMutator.kt | 44 +++++++----- .../enterprise/service/EnterpriseSampler.kt | 2 +- .../core/problem/rest/RestIndividual.kt | 5 +- .../problem/rest/resource/ResourceCluster.kt | 31 +++++---- .../resource/ResourceImpactOfIndividual.kt | 17 ++--- .../problem/rest/resource/RestResourceNode.kt | 5 +- .../dependency/ResourceRelatedToTable.kt | 68 +++++++++++-------- .../rest/service/ResourceDepManageService.kt | 23 +++++-- .../rest/service/ResourceManageService.kt | 3 +- .../service/ResourceRestStructureMutator.kt | 11 ++- .../util/inference/DeriveResourceBinding.kt | 5 +- .../problem/util/inference/SimpleDeriveR2T.kt | 42 +++++++++--- .../util/inference/model/ParamGeneBindMap.kt | 9 ++- .../org/evomaster/core/search/FitnessValue.kt | 3 +- .../evomaster/core/sql/DatabaseExecution.kt | 6 +- .../core/sql/SqlActionGeneBuilder.kt | 2 +- .../org/evomaster/core/sql/SqlActionUtils.kt | 18 ++--- .../evomaster/core/sql/SqlInsertBuilder.kt | 35 ++++++---- 19 files changed, 215 insertions(+), 129 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 7806630aba..553489aed6 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -22,6 +22,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Solution import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.sql.schema.Table import org.evomaster.core.sql.schema.TableId import org.evomaster.test.utils.EMTestUtils import org.evomaster.test.utils.SeleniumEMUtils @@ -213,14 +214,15 @@ class TestSuiteWriter { } val all = (sampler as EnterpriseSampler).extractFkTables(accessedTable) - val tableNamesInSchema = remoteController.getCachedSutInfo() - ?.sqlSchemaDto + val schema = remoteController.getCachedSutInfo()?.sqlSchemaDto + + val tableNamesInSchema = schema ?.tables - ?.map { SqlDtoUtils.getId(it) } + ?.map { TableId.fromDto(schema.databaseType, it.id) } ?.toSet() ?: setOf() - val missingTables = all.filter { x -> tableNamesInSchema.none { y -> y.equals(x,true) } }.sorted() + val missingTables = all.filter { x -> tableNamesInSchema.none { y -> y == x } }.sortedBy { it.name } if(missingTables.isNotEmpty()){ /* Weird case... but actually seen it in familie-ba-sak, regarding table "task", which is in the migration @@ -232,7 +234,10 @@ class TestSuiteWriter { } val input = if(all.isEmpty()) "" - else all.filter { x -> tableNamesInSchema.any{y -> y.equals(x,true)} }.sorted().joinToString(",") { "\"$it\"" } + else all.filter { x -> tableNamesInSchema.any{y -> y == x} } + .map{it.getFullQualifyingTableName()} + .sorted() + .joinToString(",") { "\"$it\"" } return when { config.outputFormat.isJava() -> "Arrays.asList($input)" diff --git a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt index 7e399ea40c..38d15bbbd0 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt @@ -25,6 +25,7 @@ import org.evomaster.core.search.service.mutator.StructureMutator import org.evomaster.core.sql.SqlAction import org.evomaster.core.sql.SqlActionUtils import org.evomaster.core.sql.SqlInsertBuilder +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import kotlin.math.max @@ -258,7 +259,9 @@ abstract class ApiWsStructureMutator : StructureMutator() { * its phenotype (otherwise the fitness value would be meaningless). */ - val fw = evaluatedIndividual.fitness.getViewOfAggregatedFailedWhere() + val fw = evaluatedIndividual + .fitness + .getViewOfAggregatedFailedWhere() //TODO likely to remove/change once we ll support VIEWs .filter { sampler.canInsertInto(it.key) } @@ -288,7 +291,7 @@ abstract class ApiWsStructureMutator : StructureMutator() { /** * Map of FAILED WHERE clauses. from table name key to column name values */ - fw: Map>, + fw: Map>, mutatedGenes: MutatedGeneSpecification?, sampler: ApiWsSampler ): MutableList>? { @@ -307,7 +310,7 @@ abstract class ApiWsStructureMutator : StructureMutator() { ind: T, sampler: ApiWsSampler, mutatedGenes: MutatedGeneSpecification?, - fw: Map> + fw: Map> ): MutableList>? { /* because there might exist representExistingData in db actions which are in between rest actions, @@ -385,7 +388,7 @@ abstract class ApiWsStructureMutator : StructureMutator() { return addedSqlInsertions } - private fun handleDSE(sampler: ApiWsSampler, fw: Map>): MutableList> { + private fun handleDSE(sampler: ApiWsSampler, fw: Map>): MutableList> { /* TODO: DSE should be plugged in here */ return mutableListOf() } @@ -407,14 +410,17 @@ abstract class ApiWsStructureMutator : StructureMutator() { return addedMongoDbInsertions } - private fun findMissing(fw: Map>, dbactions: List): Map> { + private fun findMissing( + fw: Map>, + dbactions: List + ): Map> { return fw.filter { e -> //shouldn't have already an action adding such SQL data dbactions .filter { !it.representExistingData } .none { a -> - a.table.name.equals(e.key, ignoreCase = true) && e.value.all { c -> + a.table.id == e.key && e.value.all { c -> // either the selected column is already in existing action (c != "*" && a.selectedColumns.any { x -> x.name.equals(c, ignoreCase = true) @@ -440,25 +446,26 @@ abstract class ApiWsStructureMutator : StructureMutator() { note that if there is no any init sql, we randomly select one table to add. */ - val candidatesToMutate = - individual.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData } - val tables = candidatesToMutate.map { it.table.name }.run { - ifEmpty { getSqlInsertBuilder()!!.getTableNames() } - } + val candidatesToMutate = individual.seeInitializingActions() + .filterIsInstance() + .filterNot { it.representExistingData } + val tables = candidatesToMutate + .map { it.table.id } + .ifEmpty { getSqlInsertBuilder()!!.getTableNames() } - val table = randomness.choose(tables) - val total = tables.count { it == table } + val tableId = randomness.choose(tables) + val total = tables.count { it == tableId } if (total == 1 || randomness.nextBoolean()) { // add action val num = randomness.nextInt(1, max(1, getMaxSizeOfMutatingInitAction())) - val add = createInsertSqlAction(table, num) + val add = createInsertSqlAction(tableId, num) handleInitSqlAddition(individual, add, mutatedGenes) } else { // remove action val num = randomness.nextInt(1, max(1, min(total - 1, getMaxSizeOfMutatingInitAction()))) - val candidates = candidatesToMutate.filter { it.table.name == table } + val candidates = candidatesToMutate.filter { it.table.id == tableId } val remove = randomness.choose(candidates, num) handleInitSqlRemoval(individual, remove, mutatedGenes) @@ -520,7 +527,7 @@ abstract class ApiWsStructureMutator : StructureMutator() { * @param name is the table name * @param num is a number of table with [name] to be added */ - fun createInsertSqlAction(name: String, num: Int): List> { + fun createInsertSqlAction(name: TableId, num: Int): List> { getSqlInsertBuilder() ?: throw IllegalStateException("attempt to create resource with SQL but the sqlBuilder is null") if (num <= 0) @@ -537,7 +544,10 @@ abstract class ApiWsStructureMutator : StructureMutator() { } val list = (0 until num) - .map { getSqlInsertBuilder()!!.createSqlInsertionAction(name,chosenColumns, mutableListOf(),true, extraConstraints, enableSingleInsertionForTable=enableSingleInsertionForTable) } + .map { + getSqlInsertBuilder()!! + .createSqlInsertionAction(name,chosenColumns, mutableListOf(),true, extraConstraints, enableSingleInsertionForTable=enableSingleInsertionForTable) + } .toMutableList() if (log.isTraceEnabled) { diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt index b9d81618b0..ee72836a4e 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt @@ -85,7 +85,7 @@ abstract class EnterpriseSampler : Sampler() where T : Individual { return action } - fun canInsertInto(tableName: String) : Boolean { + fun canInsertInto(tableName: TableId) : Boolean { //TODO might need to refactor/remove once we deal with VIEWs return sqlInsertBuilder?.isTable(tableName) ?: false } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/RestIndividual.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/RestIndividual.kt index d49eada1f7..34a409de99 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/RestIndividual.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/RestIndividual.kt @@ -21,6 +21,7 @@ import org.evomaster.core.search.gene.* import org.evomaster.core.search.tracer.Traceable import org.evomaster.core.search.tracer.TraceableElementCopyFilter import org.evomaster.core.search.tracer.TrackOperator +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import kotlin.math.max @@ -402,8 +403,8 @@ class RestIndividual( } - override fun getInsertTableNames(): List { - return seeSqlDbActions().filterNot { it.representExistingData }.map { it.table.name } + override fun getInsertTableNames(): List { + return seeSqlDbActions().filterNot { it.representExistingData }.map { it.table.id } } override fun seeMainExecutableActions(): List { diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt index a2bc2bfe3a..953a0fafff 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt @@ -12,6 +12,7 @@ import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.problem.util.inference.SimpleDeriveResourceBinding import org.evomaster.core.search.action.Action import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.schema.TableId /** * this class is to record the identified resources in the sut, i.e., based on the 'paths' of 'OpenAPI' @@ -29,13 +30,13 @@ class ResourceCluster { * key is table name * value is a list of existing data of PKs in DB */ - private val dataInDB : MutableMap> = mutableMapOf() + private val dataInDB : MutableMap> = mutableMapOf() /** * key is table name * value is the table */ - private val tables : MutableMap = mutableMapOf() + private val tables : MutableMap = mutableMapOf() /** @@ -126,10 +127,10 @@ class ResourceCluster { /** * @return existing rows of the table based on the specified [tableName] */ - fun getDataInDb(tableName: String) : MutableList?{ - val key = SqlActionUtils.getTableKey(dataInDB.keys, tableName) - ?: return null - return dataInDB.getValue(key) + fun getDataInDb(tableName: TableId) : MutableList?{ +// val key = SqlActionUtils.getTableKey(dataInDB.keys, tableName) +// ?: return null + return dataInDB.getValue(tableName) } /** @@ -154,7 +155,7 @@ class ResourceCluster { * if [doNotCreateDuplicatedAction], the returned list would be A and B * else the return list would be A, A and B. the additional A is created for B */ - fun createSqlAction(tables: List, + fun createSqlAction(tables: List, sqlInsertBuilder: SqlInsertBuilder, previous: List, doNotCreateDuplicatedAction: Boolean, @@ -164,15 +165,17 @@ class ResourceCluster { useExtraSqlDbConstraints: Boolean = false, enableSingleInsertionForTable : Boolean = false ) : MutableList{ - val sorted = SqlActionUtils.sortTable(tables.mapNotNull { getTableByName(it) }.run { if (doNotCreateDuplicatedAction) this.distinct() else this }) + val sorted = SqlActionUtils.sortTable( + tables.mapNotNull { this.tables[it] }.run { if (doNotCreateDuplicatedAction) this.distinct() else this } + ) val added = mutableListOf() val preTables = previous.filter { !isInsertion || !it.representExistingData }.map { it.table.name }.toMutableList() - if (!isInsertion && sorted.none { t-> (getDataInDb(t.name)?.size?: 0) > 0 }){ + if (!isInsertion && sorted.none { t-> (getDataInDb(t.id)?.size?: 0) > 0 }){ if (forceSynDataInDb) syncDataInDb(sqlInsertBuilder) - if (sorted.none { t-> (getDataInDb(t.name)?.size?: 0) > 0 }){ + if (sorted.none { t-> (getDataInDb(t.id)?.size?: 0) > 0 }){ return mutableListOf() } } @@ -180,19 +183,19 @@ class ResourceCluster { sorted.forEach { t-> if (!doNotCreateDuplicatedAction || preTables.none { p-> p.equals(t.name, ignoreCase = true) }){ val actions = if (!isInsertion){ - if ((getDataInDb(t.name)?.size ?: 0) == 0) { + if ((getDataInDb(t.id)?.size ?: 0) == 0) { null } else { - val candidates = getDataInDb(t.name)!! + val candidates = getDataInDb(t.id)!! val row = randomness.choose(candidates) - val a = sqlInsertBuilder.extractExistingByCols(t.name, row, useExtraSqlDbConstraints) + val a = sqlInsertBuilder.extractExistingByCols(t.id, row, useExtraSqlDbConstraints) if(a != null) listOf(a) else null } } else{ - sqlInsertBuilder.createSqlInsertionAction(t.name, useExtraSqlDbConstraints = useExtraSqlDbConstraints, enableSingleInsertionForTable=enableSingleInsertionForTable) + sqlInsertBuilder.createSqlInsertionAction(t.id, useExtraSqlDbConstraints = useExtraSqlDbConstraints, enableSingleInsertionForTable=enableSingleInsertionForTable) .onEach { a -> a.doInitialize(randomness) } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceImpactOfIndividual.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceImpactOfIndividual.kt index c6af451554..f5ca92be15 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceImpactOfIndividual.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceImpactOfIndividual.kt @@ -8,6 +8,7 @@ import org.evomaster.core.search.impact.impactinfocollection.ImpactsOfAction import org.evomaster.core.search.impact.impactinfocollection.ImpactsOfIndividual import org.evomaster.core.search.impact.impactinfocollection.InitializationGroupedActionsImpacts import org.evomaster.core.search.impact.impactinfocollection.value.numeric.IntegerGeneImpact +import org.evomaster.core.sql.schema.TableId /** * created by manzhang on 2021/10/21 @@ -32,7 +33,7 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { * key - a name of table * value - impact */ - val sqlTableSizeImpact: MutableMap + val sqlTableSizeImpact: MutableMap /** * impact of changing size of any sql in the individual @@ -45,7 +46,7 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { dynamicMainActionImpacts: MutableList, impactsOfStructure: ActionStructureImpact = ActionStructureImpact("StructureSize"), resourceSizeImpact: MutableMap, - sqlTableImpact: MutableMap, + sqlTableImpact: MutableMap, anyResourceSizeImpact: IntegerGeneImpact, anySqlTableSizeImpact: IntegerGeneImpact @@ -63,9 +64,9 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { putIfAbsent(r, IntegerGeneImpact("size")) } } - sqlTableSizeImpact = mutableMapOf().apply { + sqlTableSizeImpact = mutableMapOf().apply { individual.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }.forEach { d-> - putIfAbsent(d.table.name, IntegerGeneImpact("size")) + putIfAbsent(d.table.id, IntegerGeneImpact("size")) } } anyResourceSizeImpact = IntegerGeneImpact("anyResourceSizeImpact") @@ -84,7 +85,7 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { mutableMapOf().apply { putAll(resourceSizeImpact.map { it.key to it.value.copy() }) }, - mutableMapOf().apply { + mutableMapOf().apply { putAll(sqlTableSizeImpact.map { it.key to it.value.copy() }) }, anyResourceSizeImpact.copy(), @@ -104,7 +105,7 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { mutableMapOf().apply { putAll(resourceSizeImpact.map { it.key to it.value.clone() }) }, - mutableMapOf().apply { + mutableMapOf().apply { putAll(sqlTableSizeImpact.map { it.key to it.value.clone() }) }, anyResourceSizeImpact.clone(), @@ -130,8 +131,8 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { } } - val currentTs = current.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }.map { it.table.name } - val previousTs = previous.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }.map { it.table.name } + val currentTs = current.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }.map { it.table.id } + val previousTs = previous.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }.map { it.table.id } var anySqlChange = false currentTs.toSet().forEach { cr -> val tImpact = sqlTableSizeImpact.getOrPut(cr){IntegerGeneImpact("size")} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt index 3ac9ca4a93..d6fb1e19fa 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt @@ -19,6 +19,7 @@ import org.evomaster.core.search.gene.ObjectGene import org.evomaster.core.search.gene.sql.SqlForeignKeyGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -165,7 +166,7 @@ open class RestResourceNode( } return dbactions.filterNot { it.representExistingData }.flatMap { db-> - val exclude = related.flatMap { r-> r?.getRelatedColumn(db.table.name)?.toList()?:listOf() } + val exclude = related.flatMap { r-> r?.getRelatedColumn(db.table.id)?.toList()?:listOf() } db.seeGenesForInsertion(exclude) }.filter{it.isMutable() && it !is SqlForeignKeyGene && it !is SqlPrimaryKeyGene} } @@ -235,7 +236,7 @@ open class RestResourceNode( /** * @return related table for creating resource for [this] node with sql */ - fun getSqlCreationPoints() : List{ + fun getSqlCreationPoints() : List{ if (resourceToTable.confirmedSet.isNotEmpty()) return resourceToTable.confirmedSet.keys.toList() return resourceToTable.derivedMap.keys.toList() } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt index bc9d513124..e771ddbdaa 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt @@ -6,6 +6,7 @@ import org.evomaster.core.problem.rest.param.BodyParam import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.util.inference.model.MatchedInfo import org.evomaster.core.sql.SqlActionUtils +import org.evomaster.core.sql.schema.TableId /** * related info between resource and tables @@ -27,7 +28,7 @@ class ResourceRelatedToTable(val key: String) { * 2) segments, for instance there are two segments (i.e., /A and /B) regarding the path /A/{a}/B/{b}. for B, the input indicator is 0, for A, the input indicator is 1 * then detect whether there are tables related to C, A and B. */ - val derivedMap : MutableMap> = mutableMapOf() + val derivedMap : MutableMap> = mutableMapOf() /** * key is id of param @@ -44,7 +45,7 @@ class ResourceRelatedToTable(val key: String) { * to bind param of action regarding table, * we need to further detect whether the relationship between resource and table is direct */ - val confirmedSet : MutableMap = mutableMapOf() + val confirmedSet : MutableMap = mutableMapOf() /** * key is verb of the action @@ -89,21 +90,21 @@ class ResourceRelatedToTable(val key: String) { return doesUpdateParamTable } - fun getConfirmedDirectTables() : Set{ + fun getConfirmedDirectTables() : Set{ return derivedMap.keys .filter { t-> - confirmedSet.any { it.key.equals(t, ignoreCase = true) && it.value } + confirmedSet.any { it.key == t && it.value } }.toHashSet() } fun findBestTableForParam( - tables: Set, + tables: Set, simpleP2Table : SimpleParamRelatedToTable, onlyConfirmedColumn : Boolean = false - ) : Pair, Double>? { + ) : Pair, Double>? { val map = simpleP2Table.derivedMap - .filter { d -> tables.any { t-> SqlActionUtils.isMatchingTableName(d.key, t) } } + .filter { d -> tables.any { t-> t == d.key } } .map { Pair(it.key, it.value.similarity) }.toMap() @@ -116,13 +117,13 @@ class ResourceRelatedToTable(val key: String) { * return Pair.first is name of table, Pair.second is column of Table */ fun getSimpleParamToSpecifiedTable( - table: String, + table: TableId, simpleP2Table : SimpleParamRelatedToTable, onlyConfirmedColumn : Boolean = false - ) : Pair? { + ) : Pair? { return simpleP2Table.derivedMap - .filter { table.equals(it.key, ignoreCase = true) } + .filter { table == it.key} .let { map-> if (map.isEmpty()) null else Pair(table, map.values.first().targetMatched) @@ -135,16 +136,20 @@ class ResourceRelatedToTable(val key: String) { * key is field name * value is a pair, the first of pair of related table and the second is similarity */ - fun findBestTableForParam(tables: Set, bodyP2Table : BodyParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : MutableMap, Double>>?{ + fun findBestTableForParam( + tables: Set, + bodyP2Table : BodyParamRelatedToTable, + onlyConfirmedColumn : Boolean = false + ) : MutableMap, Double>>?{ val fmap = bodyP2Table.fieldsMap .filter { - it.value.derivedMap.any { m-> tables.any { t-> t.equals(m.key, ignoreCase = true) } } + it.value.derivedMap.any { m-> tables.any { t-> t == m.key } } } if(fmap.isEmpty()) return null - val result : MutableMap, Double>> = mutableMapOf() + val result : MutableMap, Double>> = mutableMapOf() fmap.forEach { t, u -> - val related = u.derivedMap.filter { m-> tables.any { t-> t.equals(m.key, ignoreCase = true) }} + val related = u.derivedMap.filter { m-> tables.any { t-> t == m.key }} if (related.isNotEmpty()){ val best = related.map { it.value.similarity }.maxOrNull()!! result.put(t, Pair(related.filter { it.value.similarity == best }.keys, best)) @@ -156,10 +161,14 @@ class ResourceRelatedToTable(val key: String) { /** * return Pair.first is name of table, key of the second is field, value is the column */ - fun getBodyParamToSpecifiedTable(table:String, bodyP2Table : BodyParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair>? { + fun getBodyParamToSpecifiedTable( + table:TableId, + bodyP2Table : BodyParamRelatedToTable, + onlyConfirmedColumn : Boolean = false + ) : Pair>? { val fmap = bodyP2Table.fieldsMap .filter { - it.value.derivedMap.any { m->m.key.toLowerCase() == table.toLowerCase() } + it.value.derivedMap.any { m->m.key == table } } if(fmap.isEmpty()) return null else{ @@ -172,15 +181,20 @@ class ResourceRelatedToTable(val key: String) { * * @param onlyConfirmedColumn will be used to only find the field that are confirmed based on feedback from evomaster driver */ - fun getBodyParamToSpecifiedTable(table:String, bodyP2Table : BodyParamRelatedToTable, fieldName : String, onlyConfirmedColumn : Boolean = false) : Pair>? { + fun getBodyParamToSpecifiedTable( + table:TableId, + bodyP2Table : BodyParamRelatedToTable, + fieldName : String, + onlyConfirmedColumn : Boolean = false + ) : Pair>? { val fmap = bodyP2Table.fieldsMap[fieldName]?: return null - fmap.derivedMap.filter { SqlActionUtils.isMatchingTableName(it.key, table)}.let { + fmap.derivedMap.filter { it.key == table}.let { if (it.isEmpty()) return null else return Pair(table, Pair(fieldName, it.values.first().targetMatched)) } } - fun getTablesInDerivedMap() : Set = derivedMap.keys + fun getTablesInDerivedMap() : Set = derivedMap.keys fun getTablesInDerivedMap(segs : List) : Map{ return segs.map { input-> @@ -255,12 +269,12 @@ abstract class ParamRelatedToTable ( * key is table name * value is [MatchedInfo], [MatchInfo.targetMatched] is name of column of the table */ - val derivedMap : MutableMap = mutableMapOf() + val derivedMap : MutableMap = mutableMapOf() val confirmedColumn : MutableSet = mutableSetOf() - open fun getRelatedColumn(table: String) : Set? = - derivedMap.filterKeys { SqlActionUtils.isMatchingTableName(it, table)} + open fun getRelatedColumn(table: TableId) : Set? = + derivedMap.filterKeys { it == table} .values.firstOrNull() .run { if (this == null) null @@ -287,16 +301,16 @@ class BodyParamRelatedToTable(key: String, val referParam: Param): ParamRelatedT */ val fieldsMap : MutableMap = mutableMapOf() - override fun getRelatedColumn(table: String) : Set? = + override fun getRelatedColumn(table: TableId) : Set? = fieldsMap.values .filter { - it.derivedMap.any { m-> SqlActionUtils.isMatchingTableName(m.key, table)} + it.derivedMap.any { m-> m.key == table} }.run { if (this.isEmpty()) return null else this.map { f-> - f.derivedMap.filterKeys { k -> SqlActionUtils.isMatchingTableName(k, table) } + f.derivedMap.filterKeys { k -> k == table } .asSequence().first().value.targetMatched }.toHashSet() } @@ -306,8 +320,8 @@ class BodyParamRelatedToTable(key: String, val referParam: Param): ParamRelatedT */ class ParamFieldRelatedToTable(key: String) : ParamRelatedToTable(key){ - override fun getRelatedColumn(table: String) : Set? { - return derivedMap.filter { SqlActionUtils.isMatchingTableName(it.key, table)} + override fun getRelatedColumn(table: TableId) : Set? { + return derivedMap.filter { it.key == table} .let { if(it.isEmpty()) null else it.values.map {m-> m.targetMatched}.toSet() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt index 6816e974f0..c3272e5261 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt @@ -28,6 +28,7 @@ import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.EvaluatedMutation +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import java.nio.file.Files @@ -99,7 +100,7 @@ class ResourceDepManageService { } - private fun updateParamInfo(action: RestCallAction, tables: Map) { + private fun updateParamInfo(action: RestCallAction, tables: Map) { val r = rm.getResourceNodeFromCluster(action.path.toString()) // skip resource if it is ExcludedResourceNode if (r is ExcludedResourceNode) return @@ -222,8 +223,13 @@ class ResourceDepManageService { } } - private fun updateResourceToTable(action: RestCallAction, dto: SqlExecutionsDto, tables: Map, - addedMap: MutableMap>, removedMap: MutableMap>) { + private fun updateResourceToTable( + action: RestCallAction, + dto: SqlExecutionsDto, + tables: Map, + addedMap: MutableMap>, + removedMap: MutableMap> + ) { dto.insertedData.filter { u -> tables.any { it.key.toLowerCase() == u.key } }.let { added -> updateResourceToTable(action, added, (action.verb == HttpVerb.POST || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) @@ -1001,7 +1007,10 @@ class ResourceDepManageService { */ fun unRelatedSQL(ind: RestIndividual, candidates: List?) : List{ val allrelated = getAllRelatedTables(ind) - return (candidates?:ind.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }).filterNot { allrelated.any { r-> r.equals(it.table.name, ignoreCase = true) } } + return (candidates ?: ind.seeInitializingActions() + .filterIsInstance() + .filterNot { it.representExistingData }) + .filterNot { allrelated.any { r-> r == it.table.id } } } fun identifyUnRelatedSqlTable(ind: RestIndividual, candidates: List?) : List{ @@ -1025,7 +1034,7 @@ class ResourceDepManageService { * @param probability represent a probability of using identified dependent tables, otherwise employ the tables which are * not part of the current dbInitialization */ - fun identifyRelatedSQL(ind: RestIndividual, probability: Double = 1.0): Set{ + fun identifyRelatedSQL(ind: RestIndividual, probability: Double = 1.0): Set{ val allrelated = getAllRelatedTables(ind) @@ -1046,7 +1055,7 @@ class ResourceDepManageService { } } - fun createDbActions(name : String, num : Int) : List>{ + fun createDbActions(name : TableId, num : Int) : List>{ rm.getSqlBuilder() ?:throw IllegalStateException("attempt to create resource with SQL but the sqlBuilder is null") if (num <= 0) throw IllegalArgumentException("invalid num (i.e.,$num) for creating resource") @@ -1128,7 +1137,7 @@ class ResourceDepManageService { ind.addInitializingDbActions(actions = added) } - private fun getAllRelatedTables(ind: RestIndividual) : Set{ + private fun getAllRelatedTables(ind: RestIndividual) : Set{ return ind.getResourceCalls().flatMap { c-> extractRelatedTablesForCall(c, withSql = c.is2POST).values.flatMap { it.map { g->g.tableName } }.toSet() }.toSet() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt index 6a3cfd4fe8..b50473fc35 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt @@ -16,6 +16,7 @@ import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.utils.GeneUtils import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -296,7 +297,7 @@ class ResourceManageService { } - private fun employSelect(tables : List) : Boolean{ + private fun employSelect(tables : List) : Boolean{ return randomness.nextBoolean(config.probOfSelectFromDatabase) && tables.any { cluster.getDataInDb(it)?.size?:0 >= config.minRowOfTable } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt index e800506957..2eed980f41 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt @@ -17,6 +17,7 @@ import org.evomaster.core.search.impact.impactinfocollection.value.numeric.Integ import org.evomaster.core.search.service.mutator.MutatedGeneSpecification import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.ArchiveImpactSelector +import org.evomaster.core.sql.schema.TableId import kotlin.math.max import kotlin.math.min import org.slf4j.Logger @@ -177,7 +178,7 @@ class ResourceRestStructureMutator : ApiWsStructureMutator() { val candidates = if (doesApplyDependencyHeuristics()) dm.identifyRelatedSQL(ind) else - ind.seeInitializingActions().filterIsInstance().map { it.table.name }.toSet() // adding an unrelated table would waste budget, then we add existing ones + ind.seeInitializingActions().filterIsInstance().map { it.table.id }.toSet() // adding an unrelated table would waste budget, then we add existing ones val selectedAdded = if (config.enableAdaptiveResourceStructureMutation){ adaptiveSelectResource(evaluatedIndividual, bySQL = true, candidates.toList(), targets) @@ -190,7 +191,13 @@ class ResourceRestStructureMutator : ApiWsStructureMutator() { } - private fun adaptiveSelectResource(evaluatedIndividual: EvaluatedIndividual?, bySQL: Boolean, candidates: List, targets: Set?): String{ + private fun adaptiveSelectResource( + evaluatedIndividual: EvaluatedIndividual?, + bySQL: Boolean, + candidates: List, + targets: Set? + ): TableId{ + evaluatedIndividual?: throw IllegalStateException("lack of impact with specified evaluated individual") targets?:throw IllegalStateException("targets must be specified if adaptive resource selection is applied") if (evaluatedIndividual.impactInfo == null || evaluatedIndividual.impactInfo !is ResourceImpactOfIndividual) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/DeriveResourceBinding.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/DeriveResourceBinding.kt index a6ae006177..bd6e0249e0 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/DeriveResourceBinding.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/DeriveResourceBinding.kt @@ -7,17 +7,18 @@ import org.evomaster.core.problem.rest.resource.ParamInfo import org.evomaster.core.problem.rest.resource.RestResourceCalls import org.evomaster.core.problem.rest.resource.RestResourceNode import org.evomaster.core.problem.util.inference.model.ParamGeneBindMap +import org.evomaster.core.sql.schema.TableId /** * process inference regarding resource, which can be extended for the different inference technique */ interface DeriveResourceBinding { - fun deriveResourceToTable(resourceCluster: MutableList, allTables : Map) + fun deriveResourceToTable(resourceCluster: MutableList, allTables : Map) /** * derive relationship between a resource and a table */ - fun deriveResourceToTable(resourceNode : RestResourceNode, allTables : Map) + fun deriveResourceToTable(resourceNode : RestResourceNode, allTables : Map) fun generateRelatedTables(paramsInfo: List, calls: RestResourceCalls, sqlActions : List) : MutableMap>? = null diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index f1649a039e..665991cf83 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -17,6 +17,7 @@ import org.evomaster.core.problem.util.StringSimilarityComparator.stringSimilari import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.ObjectGene import org.evomaster.core.sql.SqlActionUtils.SCHEMA_TABLE_SEPARATOR +import org.evomaster.core.sql.schema.TableId import kotlin.math.max /** @@ -31,7 +32,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { * i.e., the action under the resource may manipulate data from the table * e.g., /A/{a}/B{b}, the resource is related to two resource A and B */ - override fun deriveResourceToTable(resourceCluster: MutableList, allTables : Map){ + override fun deriveResourceToTable(resourceCluster: MutableList, allTables : Map){ resourceCluster.forEach { r-> deriveResourceToTable(r, allTables) } @@ -40,7 +41,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { /** * derive a relationship between a resource and a table */ - override fun deriveResourceToTable(resourceNode: RestResourceNode, allTables : Map){ + override fun deriveResourceToTable(resourceNode: RestResourceNode, allTables : Map){ //1. derive resource to table //1.1 derive resource to tables based on segments @@ -52,14 +53,14 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { table name might contain schema, */ val matchedMap = allTables.keys.map { - Pair(it, calculateStringSimilarityScoreWithTableName(it, token)) + Pair(it, calculateStringSimilarityScoreWithTableName(it.getFullQualifyingTableName(), token)) }.asSequence().sortedBy { e->e.second } if(matchedMap.lastOrNull()!= null && matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ matchedMap.filter { it.second == matchedMap.last().second }.forEach { resourceNode.resourceToTable.derivedMap.getOrPut(it.first){ mutableListOf() - }.add(MatchedInfo(seg, it.first, similarity = it.second, inputIndicator = sindex, outputIndicator = 0)) + }.add(MatchedInfo(seg, it.first.getFullQualifyingTableName(), similarity = it.second, inputIndicator = sindex, outputIndicator = 0)) } return@stop } @@ -138,13 +139,13 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { return score } - fun deriveParamsToTable(mapParamInfo : Map, r: RestResourceNode, allTables : Map){ + fun deriveParamsToTable(mapParamInfo : Map, r: RestResourceNode, allTables : Map){ mapParamInfo.forEach { (paramId, paramInfo) -> deriveParamsToTable(paramId, paramInfo, r, allTables) } } - fun deriveParamsToTable(paramId : String, paramInfo : ParamInfo, r: RestResourceNode, allTables : Map){ + fun deriveParamsToTable(paramId : String, paramInfo : ParamInfo, r: RestResourceNode, allTables : Map){ val inputIndicator = paramInfo.segmentLevel @@ -161,7 +162,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { r.resourceToTable.derivedMap.filter { it.value.any { m-> m.input == (ParamUtil.getObjectGene(paramInfo.referParam.gene))!!.refType!!} }.keys.toHashSet() } else null - if(tables == null || tables.isEmpty()) tables = relatedTables + if(tables.isNullOrEmpty()) tables = relatedTables created = deriveRelatedTable(r, paramId, paramInfo, tables, isBodyParam, inputIndicator, alltables = allTables ) } @@ -170,7 +171,15 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { } } - fun deriveRelatedTable(r : RestResourceNode, paramId: String, paramInfo: ParamInfo, relatedToTables: Set, isBodyParam : Boolean, inputIndicator: Int, alltables : Map) : Boolean{ + fun deriveRelatedTable( + r : RestResourceNode, + paramId: String, + paramInfo: ParamInfo, + relatedToTables: Set, + isBodyParam : Boolean, + inputIndicator: Int, + alltables : Map + ) : Boolean{ if(isBodyParam){ val pToTable = BodyParamRelatedToTable(paramId, paramInfo.referParam) ParamUtil.getObjectGene(paramInfo.referParam.gene)?.fields?.forEach { f-> @@ -205,13 +214,24 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { * * Note that same parameter (e.g., id) may related to multiple related tables. */ - private fun deriveParamWithTable(paramName : String, candidateTables : Set, pToTable : MutableMap, inputlevel: Int, tables : Map){ + private fun deriveParamWithTable( + paramName : String, + candidateTables : Set, + pToTable : MutableMap, + inputlevel: Int, + tables : Map){ candidateTables.forEach { tableName -> deriveParamWithTable(paramName, tableName, pToTable, inputlevel, tables) } } - private fun deriveParamWithTable(paramName : String, tableName: String, pToTable : MutableMap, inputlevel: Int, tables : Map){ + private fun deriveParamWithTable( + paramName : String, + tableName: String, + pToTable : MutableMap, + inputlevel: Int, + tables : Map + ){ /* paramName might be \w+id or \w+name, in this case, we compare paramName with table name of current or referred table + column name */ @@ -307,7 +327,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { } - private fun getBindMap(paramId: String, pToTable : ParamRelatedToTable, tables : Set, resourceToTable: ResourceRelatedToTable, result : MutableList) : Boolean{ + private fun getBindMap(paramId: String, pToTable : ParamRelatedToTable, tables : Set, resourceToTable: ResourceRelatedToTable, result : MutableList) : Boolean{ if(pToTable is SimpleParamRelatedToTable){ resourceToTable.findBestTableForParam(tables, pToTable)?.let {pair-> val target = pair.first.toList()[(0..(pair.first.size-1)).shuffled().first()] diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/model/ParamGeneBindMap.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/model/ParamGeneBindMap.kt index 617a4fb4ca..9213363530 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/model/ParamGeneBindMap.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/model/ParamGeneBindMap.kt @@ -1,9 +1,16 @@ package org.evomaster.core.problem.util.inference.model +import org.evomaster.core.sql.schema.TableId + /** * [isElementOfParam] means the gene of Param is one type of ObjectGene, MapGene, ListGene */ -class ParamGeneBindMap(val paramId : String, val isElementOfParam: Boolean, val targetToBind : String, val tableName: String, val column: String){ +class ParamGeneBindMap( + val paramId : String, + val isElementOfParam: Boolean, + val targetToBind : String, + val tableName: TableId, + val column: String){ fun equalWith(p: ParamGeneBindMap) : Boolean{ return paramId == p.paramId && isElementOfParam == p.isElementOfParam && targetToBind == p.targetToBind && tableName == p.tableName && column == p.column diff --git a/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt b/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt index ee737c8876..ac64d7f0e3 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt @@ -11,6 +11,7 @@ import org.evomaster.core.problem.externalservice.httpws.HttpWsExternalService import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceRequest import org.evomaster.core.search.service.IdMapper import org.evomaster.core.search.service.mutator.EvaluatedMutation +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import kotlin.math.max @@ -81,7 +82,7 @@ class FitnessValue( * When SUT does SQL commands using WHERE, keep track of when those "fails" (ie evaluate * to false), in particular the tables and columns in them involved */ - private val aggregatedFailedWhere: MutableMap> = mutableMapOf() + private val aggregatedFailedWhere: MutableMap> = mutableMapOf() /** * When SUT does MONGO commands using FIND, keep track of when those "fails" (ie evaluate diff --git a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt index 0f38de640d..29c0c29f92 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt @@ -83,10 +83,10 @@ class DatabaseExecution( fun mergeData( executions: Collection, - extractor: (DatabaseExecution) -> Map> - ): Map> { + extractor: (DatabaseExecution) -> Map> + ): Map> { - val data: MutableMap> = mutableMapOf() + val data: MutableMap> = mutableMapOf() for (ex in executions) { merge(data, extractor.invoke(ex)) diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt index d65c7eb55e..c95adcafae 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt @@ -388,7 +388,7 @@ class SqlActionGeneBuilder { } if (column.primaryKey) { - gene = SqlPrimaryKeyGene(column.name, table.name, gene, id) + gene = SqlPrimaryKeyGene(column.name, table.id, gene, id) } if (column.nullable && fk == null) { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt index 769e693bd4..a14bef1f70 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt @@ -9,6 +9,7 @@ import org.evomaster.core.search.service.Randomness import org.slf4j.Logger import org.slf4j.LoggerFactory import org.evomaster.core.sql.schema.Table +import org.evomaster.core.sql.schema.TableId object SqlActionUtils { @@ -484,7 +485,7 @@ object SqlActionUtils { throw IllegalStateException("invalid insertion, there exists invalid fk at $index") val pks = sqlActions.subList(0, index).flatMap { it.seeTopGenes() }.filterIsInstance() fks.filter { !it.nullable && !it.isBound() || pks.none { p->p.uniqueId == it.uniqueIdOfPrimaryKey }}.forEach {fk-> - val found = pks.find { pk -> pk.tableName.equals(fk.targetTable, ignoreCase = true) } + val found = pks.find { pk -> pk.tableName == fk.targetTable } ?: throw IllegalStateException("fail to target table ${fk.targetTable} for the fk ${fk.name}") fk.uniqueIdOfPrimaryKey = found.uniqueId } @@ -496,14 +497,13 @@ object SqlActionUtils { /** * @return a list of dbactions from [sqlActions] whose related table is [tableName] */ - fun findDbActionsByTableName(sqlActions: List, tableName : String) : List{ - return sqlActions.filter { it.table.name.equals(tableName, ignoreCase = true) - || it.table.name.endsWith(".$tableName", true)} + fun findDbActionsByTableName(sqlActions: List, tableName : TableId) : List{ + return sqlActions.filter { it.table.id == tableName } } /** - * Are the 2 names matching? This ignore case. + * Are the 2 names matching? This ignores case. * The first [fullName] is a full qualifying name, including schema. * The second [name] "might" be simple, or full qualifying. */ @@ -527,18 +527,18 @@ object SqlActionUtils { * 2) keys having full names (eg, including schemas and possibly catalog) whereas input only having the name. * this latter case is not a problem if names are unique */ - fun getTableKey(keys: Set, tableName: String) : String?{ + fun getTableKey(keys: Set, tableName: String) : TableId?{ /* * SQL is not case sensitivity, table/column must ignore case sensitivity. - * No, this is not really true... + * FIXME: No, this is not really true... * Usually, names are lowered-cased by the DB, unless quoted in "": * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL. * */ - val tableNameKey = keys.find { tableName.equals(it, ignoreCase = true) } + val tableNameKey = keys.find { tableName.equals(it.getFullQualifyingTableName(), ignoreCase = true) } if (!tableName.contains(".") && tableNameKey == null){ //input name might be without schema, so check for partial match - val candidates = keys.filter { it.endsWith(".${tableName}", true) } + val candidates = keys.filter { it.name.equals(tableName, true) } if(candidates.size > 1){ throw IllegalArgumentException("Ambiguity." + " More than one candidate of table called $tableName." + diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 6653f99dea..97a60366f3 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -182,7 +182,6 @@ class SqlInsertBuilder( for (fk in tableDto.foreignKeys) { val tableKey = SqlActionUtils.getTableKey(tableToColumns.keys, fk.targetTable) - FIXME if(tableKey == null || tableToColumns[tableKey] == null) { throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}") @@ -194,7 +193,9 @@ class SqlInsertBuilder( // val c = targetTable.find { it.name.equals(cname, ignoreCase = true) } // ?: throw IllegalArgumentException("Issue in foreign key: table ${f.targetTable} does not have a column called $cname") - val c = tableToColumns[SqlDtoUtils.getId(tableDto)]!!.find { it.name.equals(cname, ignoreCase = true) } + val id = TableId.fromDto(databaseType, tableDto.id) + + val c = tableToColumns[id]!!.find { it.name.equals(cname, ignoreCase = true) } ?: throw IllegalArgumentException("Issue in foreign key: table ${tableDto.id.name} does not have a column called $cname") sourceColumns.add(c) @@ -632,13 +633,17 @@ class SqlInsertBuilder( * * quotes "" can be used to force case-sensitivity */ - fun isTable(tableName: String) = SqlActionUtils.getTableKey(tables.keys, tableName) != null + fun isTable(tableName: TableId) = + //SqlActionUtils.getTableKey(tables.keys, tableName) != null + tables.contains(tableName) - private fun getValueByTableNameKey(map: Map, tableName: String) : T?{ - val key = SqlActionUtils.getTableKey(map.keys, tableName) - return map[key] - } + +// private fun getValueByTableNameKey(map: Map, tableName: String) : T?{ +// +// val key = SqlActionUtils.getTableKey(map.keys, tableName) +// return map[key] +// } @@ -715,7 +720,7 @@ class SqlInsertBuilder( val table = getTable(tableName, useExtraSqlDbConstraints) val takeAll = columnNames.contains("*") - validateColumnNamesInput(takeAll, columnNames, table, tableName) + validateColumnNamesInput(takeAll, columnNames, table, tableName.getFullQualifyingTableName()) // filter only columns that will generate value for val selectedColumns = table.columns @@ -735,7 +740,7 @@ class SqlInsertBuilder( // if we have already generated the sql inserts for the target table more than 3 times and all columns are nullable, then skip it val maxIter = 3 // TODO: as a configurable parameter in EMConfig? - val visitedTableCount = history.count { it.equals(targetTable, ignoreCase = true) } + val visitedTableCount = history.count { it == targetTable } if (visitedTableCount >= maxIter && fk.sourceColumns.all { it.nullable }) { continue } @@ -844,7 +849,7 @@ class SqlInsertBuilder( val pkName = pks[i].name val inQuotes = pks[i].type.shouldBePrintedInQuotes || pks[i].dimension > 0 val data = ImmutableDataHolderGene(pkName, r.columnData[i], inQuotes) - val pk = SqlPrimaryKeyGene(pkName, table.name, data, id) + val pk = SqlPrimaryKeyGene(pkName, table.id, data, id) genes.add(pk) } @@ -872,7 +877,7 @@ class SqlInsertBuilder( * */ fun extractExistingByCols( - tableName: String, + tableName: TableId, pkValues: DataRowDto, useExtraSqlDbConstraints: Boolean, columnIds: List = mutableListOf() @@ -942,7 +947,7 @@ class SqlInsertBuilder( val gene = if (cols[i].primaryKey) { SqlPrimaryKeyGene( colName, - table.name, + table.id, ImmutableDataHolderGene(colName, row.columnData[i], inQuotes), id ) @@ -963,7 +968,7 @@ class SqlInsertBuilder( /** * get existing pks in db */ - fun extractExistingPKs(dataInDB: MutableMap>) { + fun extractExistingPKs(dataInDB: MutableMap>) { if (dbExecutor == null) { throw IllegalStateException("No Database Executor registered for this object") @@ -980,7 +985,7 @@ class SqlInsertBuilder( val result: QueryResultDto = dbExecutor.executeDatabaseCommandAndGetQueryResults(dto) ?: continue - dataInDB.getOrPut(table.name) { result.rows.map { it }.toMutableList() } + dataInDB.getOrPut(table.id) { result.rows.map { it }.toMutableList() } } } @@ -988,7 +993,7 @@ class SqlInsertBuilder( /** * get table info */ - fun extractExistingTables(tablesMap: MutableMap? = null) { + fun extractExistingTables(tablesMap: MutableMap? = null) { if (tablesMap != null) { tablesMap.clear() tablesMap.putAll(tables) From 3483d4ca49300bc9de9c05a2ed24755eb3f31e69 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 17 Jan 2025 23:17:03 +0100 Subject: [PATCH 20/21] yet on the journey to refactor TableId --- .../core/problem/api/service/ApiWsFitness.kt | 2 - .../enterprise/service/EnterpriseFitness.kt | 13 ++- .../problem/rest/resource/RestResourceNode.kt | 5 +- .../dependency/ResourceRelatedToResource.kt | 8 +- .../dependency/ResourceRelatedToTable.kt | 38 ++++---- .../rest/service/ResourceDepManageService.kt | 86 +++++++++++-------- .../rest/service/ResourceManageService.kt | 5 +- .../service/ResourceRestStructureMutator.kt | 10 ++- .../problem/util/inference/SimpleDeriveR2T.kt | 8 +- .../evomaster/core/sql/DatabaseExecution.kt | 75 ++++++++++------ .../evomaster/core/sql/SqlInsertBuilder.kt | 2 +- 11 files changed, 157 insertions(+), 95 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsFitness.kt index 705d26f644..3419e52d80 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsFitness.kt @@ -30,8 +30,6 @@ abstract class ApiWsFitness : EnterpriseFitness() where T : Individual { @Inject protected lateinit var writer: TestSuiteWriter - @Inject - protected lateinit var sampler: Sampler lateinit var infoDto: SutInfoDto diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt index 92a3ea75cf..b7e5e1b10c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt @@ -44,6 +44,8 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual @Inject protected lateinit var searchTimeController: SearchTimeController + @Inject + protected lateinit var sampler: EnterpriseSampler /** * @param allSqlActions specified the db actions to be executed @@ -255,10 +257,13 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual if (configuration.extractSqlExecutionInfo) { for (i in 0 until dto.extraHeuristics.size) { val extra = dto.extraHeuristics[i] - val databaseExecution = DatabaseExecution.fromDto(extra.sqlSqlExecutionsDto) - fv.setDatabaseExecution(i, databaseExecution) - if (databaseExecution.sqlParseFailureCount>0) { - statistics.reportSqlParsingFailures(databaseExecution.sqlParseFailureCount) + val sdto = extra.sqlSqlExecutionsDto + if(sdto != null) { + val databaseExecution = DatabaseExecution.fromDto(sdto, sampler.sqlInsertBuilder!!.getTableNames()) + fv.setDatabaseExecution(i, databaseExecution) + if (databaseExecution.sqlParseFailureCount > 0) { + statistics.reportSqlParsingFailures(databaseExecution.sqlParseFailureCount) + } } } fv.aggregateDatabaseData() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt index d6fb1e19fa..da7033b32c 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt @@ -19,6 +19,7 @@ import org.evomaster.core.search.gene.ObjectGene import org.evomaster.core.search.gene.sql.SqlForeignKeyGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.SqlActionUtils import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -666,7 +667,9 @@ open class RestResourceNode( /** * @return derived tables */ - fun getDerivedTables() : Set = resourceToTable.derivedMap.flatMap { it.value.map { m->m.targetMatched } }.toHashSet() + fun getDerivedTables(all: Set) : Set = resourceToTable.derivedMap + .flatMap { it.value.mapNotNull { m-> SqlActionUtils.getTableKey(all, m.targetMatched) } } + .toHashSet() /** * @return is any POST, GET, PATCH, DELETE, PUT action? diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt index a8382f695d..03f030d35d 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt @@ -1,6 +1,7 @@ package org.evomaster.core.problem.rest.resource.dependency import org.evomaster.core.problem.rest.RestPath +import org.evomaster.core.sql.schema.TableId /** * @param path is a list of path(s), which can be parsed with [RelatedTo.parseMultipleKey] @@ -49,8 +50,11 @@ open class ResourceRelatedToResources( * Note that related table might be derived based on token parser, not confirmed regarding evomaster driver. * [confirmedSet] is used to represent whether the mutual relation is confirmed. */ -class MutualResourcesRelations(mutualResources: List, probability: Double, var referredTables : MutableSet = mutableSetOf()) - : ResourceRelatedToResources(mutualResources, mutualResources.toMutableList(), probability, ""){ +class MutualResourcesRelations( + mutualResources: List, + probability: Double, + var referredTables : MutableSet = mutableSetOf() +) : ResourceRelatedToResources(mutualResources, mutualResources.toMutableList(), probability, ""){ override fun getName(): String { return "MutualRelations:${notateKey()}" diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt index e771ddbdaa..de77cd5b7c 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt @@ -5,6 +5,7 @@ import org.evomaster.core.sql.SQLKey import org.evomaster.core.problem.rest.param.BodyParam import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.util.inference.model.MatchedInfo +import org.evomaster.core.sql.DatabaseExecution import org.evomaster.core.sql.SqlActionUtils import org.evomaster.core.sql.schema.TableId @@ -17,7 +18,7 @@ class ResourceRelatedToTable(val key: String) { companion object { private const val FROM_EVO_DTO = "____FROM_DTO____" - fun generateFromDtoMatchedInfo(table: String) : MatchedInfo = MatchedInfo(FROM_EVO_DTO, table, 1.0, -1, -1) + fun generateFromDtoMatchedInfo(table: TableId) : MatchedInfo = MatchedInfo(FROM_EVO_DTO, table.getFullQualifyingTableName(), 1.0, -1, -1) } /** * key is table name @@ -58,16 +59,17 @@ class ResourceRelatedToTable(val key: String) { private val actionToTables : MutableMap> = mutableMapOf() - fun updateActionRelatedToTable(verb : String, dto: SqlExecutionsDto, existingTables : Set) : Boolean{ + fun updateActionRelatedToTable(verb : String, de: DatabaseExecution, existingTables : Set) : Boolean{ - val tables = mutableListOf() - .plus(dto.deletedData) - .plus(dto.updatedData.keys) - .plus(dto.insertedData.keys) - .plus(dto.queriedData.keys) - .filter { x -> - existingTables.any { x.equals(it, true) } - }.toHashSet() + val tables = mutableListOf() + .plus(de.deletedData) + .plus(de.updatedData.keys) + .plus(de.insertedData.keys) + .plus(de.queriedData.keys) +// .filter { x -> +// existingTables.any { x.equals(it, true) } +// } + .toHashSet() if (tables.isEmpty()) return false @@ -82,10 +84,10 @@ class ResourceRelatedToTable(val key: String) { actionToTables[verb]!!.add(access) } - access.updateTableWithFields(dto.deletedData.associateWith { mutableSetOf() }, SQLKey.DELETE) - access.updateTableWithFields(dto.insertedData, SQLKey.INSERT) - access.updateTableWithFields(dto.queriedData, SQLKey.SELECT) - access.updateTableWithFields(dto.updatedData, SQLKey.UPDATE) + access.updateTableWithFields(de.deletedData.associateWith { mutableSetOf() }, SQLKey.DELETE) + access.updateTableWithFields(de.insertedData, SQLKey.INSERT) + access.updateTableWithFields(de.queriedData, SQLKey.SELECT) + access.updateTableWithFields(de.updatedData, SQLKey.UPDATE) return doesUpdateParamTable } @@ -222,7 +224,7 @@ class ActionRelatedToTable( * key is table name * value is how it accessed by dbaction */ - val tableWithFields: MutableMap> = mutableMapOf() + val tableWithFields: MutableMap> = mutableMapOf() ) { @@ -230,7 +232,7 @@ class ActionRelatedToTable( * key of result is table name * value of result is a set of manipulated columns */ - fun updateTableWithFields(results : Map>, method: SQLKey) { + fun updateTableWithFields(results : Map>, method: SQLKey) { var doesUpdateTarget = false results.forEach { t, u -> doesUpdateTarget = doesUpdateTarget || tableWithFields.containsKey(t) @@ -245,7 +247,7 @@ class ActionRelatedToTable( } } - fun doesSubsume(tables : Set, subsumeThis : Boolean) : Boolean{ + fun doesSubsume(tables : Set, subsumeThis : Boolean) : Boolean{ return if(subsumeThis) tables.toHashSet().containsAll(tableWithFields.keys) else tableWithFields.keys.containsAll(tables) } @@ -255,7 +257,7 @@ class ActionRelatedToTable( * @property table related table * @property field what fields are assess by the sql command */ - class AccessTable(val method : SQLKey, val table : String, val field : MutableSet) + class AccessTable(val method : SQLKey, val table : TableId, val field : MutableSet) } /** diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt index c3272e5261..48c420a02b 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt @@ -28,6 +28,7 @@ import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.EvaluatedMutation +import org.evomaster.core.sql.DatabaseExecution import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -83,16 +84,18 @@ class ResourceDepManageService { /* TODO how to decide to remove relationship between resource and table */ - val addedMap = mutableMapOf>() - val removedMap = mutableMapOf>() + val addedMap = mutableMapOf>() + val removedMap = mutableMapOf>() restIndividual.seeMainExecutableActions().forEachIndexed { index, action -> if (config.doesApplyNameMatching) updateParamInfo(action as RestCallAction, tables) // size of extraHeuristics might be less than size of action due to failure of handling rest action if (index < dto.extraHeuristics.size) { val dbDto = dto.extraHeuristics[index].sqlSqlExecutionsDto - if (dbDto != null) - updateResourceToTable(action as RestCallAction, dbDto, tables, addedMap, removedMap) + if (dbDto != null) { + val de = DatabaseExecution.fromDto(dbDto, tables.keys) + updateResourceToTable(action, de, tables, addedMap, removedMap) + } } } if (addedMap.isNotEmpty() || removedMap.isNotEmpty()) @@ -114,12 +117,20 @@ class ResourceDepManageService { /** * TODO remove false-derived dependencies based on feedback from evomaster driver */ - private fun updateDependencyOnceResourceTableUpdate(addedMap: MutableMap>, removedMap: MutableMap>) { + private fun updateDependencyOnceResourceTableUpdate( + addedMap: MutableMap>, + removedMap: MutableMap> + ) { val groupTable = addedMap.flatMap { it.value }.toHashSet() groupTable.forEach { table -> val newRelatedResource = addedMap.filter { it.value.contains(table) }.keys - val previousResourcesWithTable = dependencies.values.flatMap { l -> l.filter { d->d is MutualResourcesRelations && d.referredTables.contains(table) }.flatMap { it.targets }}.toHashSet() + val previousResourcesWithTable = dependencies.values + .flatMap { + l -> l + .filter { d->d is MutualResourcesRelations && d.referredTables.contains(table) } + .flatMap { it.targets } + }.toHashSet() var find = false dependencies.values.forEach { rlist -> @@ -170,8 +181,14 @@ class ResourceDepManageService { } } - private fun updateResourceToTable(action: RestCallAction, updated: Map>, matchedWithVerb: Boolean, tables: Map, - addedMap: MutableMap>, removedMap: MutableMap>) { + private fun updateResourceToTable( + action: RestCallAction, + updated: Map>, + matchedWithVerb: Boolean, + tables: Map, + addedMap: MutableMap>, + removedMap: MutableMap> + ) { val ar = rm.getResourceNodeFromCluster(action.path.toString()) val rToTable = ar.resourceToTable @@ -179,7 +196,7 @@ class ResourceDepManageService { val derivedTables = rToTable.getTablesInDerivedMap() updated.forEach { (t, u) -> - if (derivedTables.any { it.equals(t, ignoreCase = true) }) { + if (derivedTables.any { it == t }) { if (action.parameters.isNotEmpty() && u.isNotEmpty() && u.none { it == "*" }) { action.parameters.forEach { p -> val paramId = ar.getParamId(action.parameters, p) @@ -191,8 +208,8 @@ class ResourceDepManageService { } } } else { - val matchedInfo = ResourceRelatedToTable.generateFromDtoMatchedInfo(t.toLowerCase()) - ar.resourceToTable.derivedMap.put(t, mutableListOf(matchedInfo)) + val matchedInfo = ResourceRelatedToTable.generateFromDtoMatchedInfo(t) + ar.resourceToTable.derivedMap[t] = mutableListOf(matchedInfo) action.parameters.forEach { p -> val paramId = ar.getParamId(action.parameters, p) val paramInfo = ar.paramsInfo[paramId].run { @@ -204,13 +221,12 @@ class ResourceDepManageService { val hasMatchedParam = SimpleDeriveResourceBinding.deriveRelatedTable(ar, paramId, paramInfo, mutableSetOf(t), p is BodyParam, -1, alltables = tables) ar.resourceToTable.paramToTable[paramId]?.let { paramToTable -> paramToTable.getRelatedColumn(t)?.apply { - paramToTable.confirmedColumn.addAll(this.intersect(u.filter { it != "*" })) + paramToTable.confirmedColumn.addAll(this.intersect(u.filter { it != "*" }.toSet())) } } } addedMap.getOrPut(ar.getName()) { mutableSetOf() }.add(t) - } rToTable.confirmedSet.getOrPut(t) { true } @@ -225,29 +241,25 @@ class ResourceDepManageService { private fun updateResourceToTable( action: RestCallAction, - dto: SqlExecutionsDto, + de: DatabaseExecution, tables: Map, - addedMap: MutableMap>, - removedMap: MutableMap> + addedMap: MutableMap>, + removedMap: MutableMap> ) { - dto.insertedData.filter { u -> tables.any { it.key.toLowerCase() == u.key } }.let { added -> - updateResourceToTable(action, added, (action.verb == HttpVerb.POST || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) - } - - dto.updatedData.filter { u -> tables.any { it.key.toLowerCase() == u.key } }.let { updated -> - updateResourceToTable(action, updated, (action.verb == HttpVerb.PATCH || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) - } + updateResourceToTable(action, de.insertedData, (action.verb == HttpVerb.POST || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) + updateResourceToTable(action, de.updatedData, (action.verb == HttpVerb.PATCH || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) + updateResourceToTable(action, de.deletedData.map { Pair(it, mutableSetOf()) }.toMap(), (action.verb == HttpVerb.PATCH || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) + updateResourceToTable(action, de.queriedData, true, tables, addedMap, removedMap) - dto.deletedData.filter { u -> tables.any { it.key.toLowerCase() == u } }.let { del -> - updateResourceToTable(action, del.map { Pair(it, mutableSetOf()) }.toMap(), (action.verb == HttpVerb.PATCH || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) + val relatedTable = rm.getResourceNodeFromCluster(action.path.toString()) + .resourceToTable - } - dto.queriedData.filter { u -> tables.any { it.key.toLowerCase() == u.key } }.let { get -> - updateResourceToTable(action, get, true, tables, addedMap, removedMap) - } - - rm.getResourceNodeFromCluster(action.path.toString()).resourceToTable.updateActionRelatedToTable(action.verb.toString(), dto, tables.keys) + relatedTable.updateActionRelatedToTable( + action.verb.toString(), + de, + tables.keys + ) } /************************ derive dependency using parser ***********************************/ @@ -257,7 +269,11 @@ class ResourceDepManageService { */ fun initDependencyBasedOnDerivedTables(resourceCluster: ResourceCluster) { resourceCluster.getTableInfo().keys.forEach { table -> - val mutualResources = resourceCluster.getCluster().values.filter { r -> r.getDerivedTables().any { e -> e.equals(table, ignoreCase = true) } }.map { it.getName() }.toList() + val mutualResources = resourceCluster.getCluster().values + .filter { r -> r.getDerivedTables(rm.sqlInsertBuilder!!.getTableNames()).any { e -> e == table } } + .map { it.getName() } + .toList() + if (mutualResources.isNotEmpty() && mutualResources.size > 1) { val mutualRelation = MutualResourcesRelations(mutualResources, StringSimilarityComparator.SimilarityThreshold, mutableSetOf(table)) @@ -267,7 +283,7 @@ class ResourceDepManageService { if (it == null) relations.add(mutualRelation) else - (it as MutualResourcesRelations).referredTables.add(table.toLowerCase()) + (it as MutualResourcesRelations).referredTables.add(table) } } } @@ -1040,7 +1056,7 @@ class ResourceDepManageService { if (allrelated.isNotEmpty() && randomness.nextBoolean(probability)){ val notincluded = allrelated.filterNot { - ind.seeInitializingActions().filterIsInstance().any { d-> it.equals(d.table.name, ignoreCase = true) } + ind.seeInitializingActions().filterIsInstance().any { d-> it == d.table.id } } //prioritize notincluded related ones with a probability 0.8 return if (notincluded.isNotEmpty() && randomness.nextBoolean(0.8)){ @@ -1048,7 +1064,7 @@ class ResourceDepManageService { }else allrelated }else{ val left = rm.getTableInfo().keys.filterNot { - ind.seeInitializingActions().filterIsInstance().any { d-> it.equals(d.table.name, ignoreCase = true) } + ind.seeInitializingActions().filterIsInstance().any { d-> it == d.table.id } } return if (left.isNotEmpty() && randomness.nextBoolean()) left.toSet() else rm.getTableInfo().keys diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt index b50473fc35..92552c0fec 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt @@ -53,7 +53,8 @@ class ResourceManageService { private val excludedCluster : MutableMap = mutableMapOf() - private var sqlInsertBuilder : SqlInsertBuilder? = null + var sqlInsertBuilder : SqlInsertBuilder? = null + private set /** * init resource nodes based on [actionCluster] and [sqlInsertBuilder] @@ -185,7 +186,7 @@ class ResourceManageService { return } - var employSQL = config.shouldGenerateSqlData() && hasDBHandler() && ar.getDerivedTables().isNotEmpty() + var employSQL = config.shouldGenerateSqlData() && hasDBHandler() && ar.getDerivedTables(sqlInsertBuilder!!.getTableNames()).isNotEmpty() && (forceSQLInsert || randomness.nextBoolean(config.probOfApplySQLActionToCreateResources)) var candidate = template diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt index 2eed980f41..0c82c33f3c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt @@ -17,6 +17,7 @@ import org.evomaster.core.search.impact.impactinfocollection.value.numeric.Integ import org.evomaster.core.search.service.mutator.MutatedGeneSpecification import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.ArchiveImpactSelector +import org.evomaster.core.sql.SqlActionUtils import org.evomaster.core.sql.schema.TableId import kotlin.math.max import kotlin.math.min @@ -181,7 +182,9 @@ class ResourceRestStructureMutator : ApiWsStructureMutator() { ind.seeInitializingActions().filterIsInstance().map { it.table.id }.toSet() // adding an unrelated table would waste budget, then we add existing ones val selectedAdded = if (config.enableAdaptiveResourceStructureMutation){ - adaptiveSelectResource(evaluatedIndividual, bySQL = true, candidates.toList(), targets) + val name = adaptiveSelectResource(evaluatedIndividual, bySQL = true, candidates.map { it.getFullQualifyingTableName() }, targets) + SqlActionUtils.getTableKey(this.rm.getTableInfo().keys, name) + ?: randomness.choose(candidates) }else{ randomness.choose(candidates) } @@ -196,7 +199,7 @@ class ResourceRestStructureMutator : ApiWsStructureMutator() { bySQL: Boolean, candidates: List, targets: Set? - ): TableId{ + ): String{ evaluatedIndividual?: throw IllegalStateException("lack of impact with specified evaluated individual") targets?:throw IllegalStateException("targets must be specified if adaptive resource selection is applied") @@ -204,7 +207,8 @@ class ResourceRestStructureMutator : ApiWsStructureMutator() { throw IllegalStateException("lack of impact info or mismatched impact type (type: ${evaluatedIndividual.impactInfo?.javaClass?.simpleName?:"null"})") val impacts = candidates.map { if (bySQL){ - evaluatedIndividual.impactInfo.sqlTableSizeImpact[it] ?:IntegerGeneImpact("size") + val key = SqlActionUtils.getTableKey(rm.getTableInfo().keys, it) + evaluatedIndividual.impactInfo.sqlTableSizeImpact[key] ?:IntegerGeneImpact("size") }else{ evaluatedIndividual.impactInfo.resourceSizeImpact[it] ?:IntegerGeneImpact("size") } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index 665991cf83..d41e7feadf 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -175,7 +175,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { r : RestResourceNode, paramId: String, paramInfo: ParamInfo, - relatedToTables: Set, + relatedToTables: Set, isBodyParam : Boolean, inputIndicator: Int, alltables : Map @@ -269,7 +269,11 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { * @param sqlActions specifies the tables to be analyzed. * if [dbActions] is empty, the tables would be all related tables extracted from its resource node */ - override fun generateRelatedTables(paramsInfo: List, calls: RestResourceCalls, sqlActions : List): MutableMap> { + override fun generateRelatedTables( + paramsInfo: List, + calls: RestResourceCalls, + sqlActions : List + ): MutableMap> { val result = mutableMapOf>() diff --git a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt index 29c0c29f92..dbea7c830b 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt @@ -1,7 +1,9 @@ package org.evomaster.core.sql import org.evomaster.client.java.controller.api.dto.database.execution.SqlExecutionsDto +import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.sql.schema.TableId +import org.slf4j.LoggerFactory /** * When a test case is executed, and the SUT does access a SQL database, @@ -56,7 +58,17 @@ class DatabaseExecution( companion object { - fun fromDto(dto: SqlExecutionsDto?): DatabaseExecution { + private val log = LoggerFactory.getLogger(DatabaseExecution::class.java) + + /** + * From [dto] info on what executed on the SQL database, return an internal representation. + * To achieve this, we need to map it to the existing schema definition from which [tableIds] + * are inferred. + * This is done for few reasons, including: + * 1) collected queries might missing schema info + * 2) there might be cases we don't handle yet (eg views) + */ + fun fromDto(dto: SqlExecutionsDto, tableIds: Set): DatabaseExecution { /* Dealing with quotes in table names is tricky... could be to handle reserved words or case sensitivity. @@ -67,20 +79,44 @@ class DatabaseExecution( might depend on library used to analyze the SQL commands. */ - FIXME - return DatabaseExecution( - cloneData(dealWithQuotes(dto?.queriedData)), - cloneData(dealWithQuotes(dto?.updatedData)), - cloneData(dealWithQuotes(dto?.insertedData)), - cloneData(dealWithQuotes(dto?.failedWhere)), - dealWithQuotes(dto?.deletedData?.toList()) ?: listOf(), - dto?.numberOfSqlCommands ?: 0, - dto?.sqlParseFailureCount ?: 0, - cloneSqlExecutionInfo(dto?.sqlExecutionLogDtoList) + cloneData(convertData(dto.queriedData, tableIds)), + cloneData(convertData(dto.updatedData, tableIds)), + cloneData(convertData(dto.insertedData, tableIds)), + cloneData(convertData(dto.failedWhere, tableIds)), + convertData(dto.deletedData?.toList(), tableIds) ?: listOf(), + dto.numberOfSqlCommands, + dto.sqlParseFailureCount, + cloneSqlExecutionInfo(dto.sqlExecutionLogDtoList) ) } + + private fun convertData(data: List?, tableIds: Set) : List?{ + if(data == null) return null + + return data.mapNotNull { + val id = SqlActionUtils.getTableKey(tableIds, removeQuotes(it)) + if(id == null){ + LoggingUtil.uniqueWarn(log, "Cannot identify table: $it") + } + id + } + } + + private fun convertData(data: Map>?, tableIds: Set) : Map>? { + if(data == null) return null + + return data.entries + .mapNotNull { e -> + val id = SqlActionUtils.getTableKey(tableIds, removeQuotes(e.key)) + if(id == null){ + LoggingUtil.uniqueWarn(log, "Cannot identify table: ${e.key}") + } + id?.let { it to e.value} + }.toMap() + } + fun mergeData( executions: Collection, extractor: (DatabaseExecution) -> Map> @@ -95,21 +131,10 @@ class DatabaseExecution( return data } - private fun dealWithQuotes(data: Map>?): Map>? { - if(data == null) return null - - return data.entries.associate { - (if(it.key.name.startsWith("\"")) removeQuotes(it.key) else it.key) to it.value - } - } - - private fun dealWithQuotes(data: List?) : List?{ - if(data == null) return null - - return data.map { if(it.startsWith("\"")) removeQuotes(it) else it } - } - private fun removeQuotes(s: String) = s.substring(1,s.length-1) + private fun removeQuotes(s: String) = + if(s.startsWith("\"") && s.endsWith("\"")) s.substring(1,s.length-1) + else s private fun cloneData(data: List?): List { if (data == null) { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 97a60366f3..bc925f1407 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -1049,5 +1049,5 @@ class SqlInsertBuilder( /** * get names of all tables */ - fun getTableNames() = tables.keys + fun getTableNames() : Set = tables.keys } From 0f6c7874177bdca94d379a72ab33ce8fb1b138ab Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 20 Jan 2025 14:40:53 +0100 Subject: [PATCH 21/21] some more refactoring --- .../problem/util/inference/SimpleDeriveR2T.kt | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index d41e7feadf..be9216491e 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -65,7 +65,11 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { return@stop } - val matchedPropertyMap = allTables.flatMap { t->t.value.columns.filter { c-> !ParamUtil.isGeneralName(c.name) }.map { c->Pair(t.value.name, c.name) } } + val matchedPropertyMap = allTables + .flatMap { t->t.value.columns + .filter { c-> !ParamUtil.isGeneralName(c.name) } + .map { c->Pair(t.value.id, c.name) } + } .map { p-> Pair(p.first, Pair(p.second, calculateStringSimilarityScoreWithTableName(p.second, token) )) }.asSequence().sortedBy { e->e.second.second } @@ -74,7 +78,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { matchedPropertyMap.filter { it.second.second == matchedPropertyMap.last().second.second }.forEach { resourceNode.resourceToTable.derivedMap.getOrPut(it.first){ mutableListOf() - }.add(MatchedInfo(seg, it.first, similarity = it.second.second, inputIndicator = sindex, outputIndicator = 1)) + }.add(MatchedInfo(seg, it.first.getFullQualifyingTableName(), similarity = it.second.second, inputIndicator = sindex, outputIndicator = 1)) } return@stop } @@ -89,12 +93,12 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { if(reftypes.isNotEmpty()){ reftypes.forEach { type-> if(!resourceNode.isPartOfStaticTokens(type) && allTables.isNotEmpty()){ - val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it, type)) }.asSequence().sortedBy { e->e.second } + val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it.getFullQualifyingTableName(), type)) }.asSequence().sortedBy { e->e.second } if(matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ matchedMap.filter { it.second == matchedMap.last().second }.forEach { resourceNode.resourceToTable.derivedMap.getOrPut(it.first){ mutableListOf() - }.add(MatchedInfo(type, it.first, similarity = it.second, inputIndicator = 0, outputIndicator = 0)) + }.add(MatchedInfo(type, it.first.getFullQualifyingTableName(), similarity = it.second, inputIndicator = 0, outputIndicator = 0)) } } } @@ -103,12 +107,12 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { //1.3 derive resource to tables based on tokens on POST action resourceNode.actions.filter {it.verb == HttpVerb.POST }.forEach { post-> post.tokens.values.filter { !resourceNode.getName().toLowerCase().contains(it.getKey().toLowerCase()) }.forEach { atoken-> - val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it, atoken.getKey())) }.asSequence().sortedBy { e->e.second } + val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it.getFullQualifyingTableName(), atoken.getKey())) }.asSequence().sortedBy { e->e.second } matchedMap.last().apply { if(second >= StringSimilarityComparator.SimilarityThreshold){ resourceNode.resourceToTable.derivedMap.getOrPut(first){ mutableListOf() - }.add(MatchedInfo(atoken.getKey(), first, similarity = second, inputIndicator = 1, outputIndicator = 0)) + }.add(MatchedInfo(atoken.getKey(), first.getFullQualifyingTableName(), similarity = second, inputIndicator = 1, outputIndicator = 0)) } } } @@ -216,7 +220,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { */ private fun deriveParamWithTable( paramName : String, - candidateTables : Set, + candidateTables : Set, pToTable : MutableMap, inputlevel: Int, tables : Map){ @@ -227,7 +231,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { private fun deriveParamWithTable( paramName : String, - tableName: String, + tableName: TableId, pToTable : MutableMap, inputlevel: Int, tables : Map @@ -235,7 +239,8 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { /* paramName might be \w+id or \w+name, in this case, we compare paramName with table name of current or referred table + column name */ - getTable(tableName, tables)?.let { t-> + //getTable(tableName, tables)? + tables[tableName]?.let { t-> val matchedMap = t.columns .map { Pair(it.name,