From 7312ff4e30860c754cf0c332064a1d43afa5fc5d Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Fri, 5 Jan 2024 22:00:08 -0800 Subject: [PATCH] Test fixes (#35) * Update the tests --- .github/workflows/update-gradle-wrapper.yml | 19 ------------ .../application-integrationtest.properties | 2 ++ kafka/build.gradle | 1 + .../application-integrationtest.properties | 2 ++ mysql-persistence/build.gradle | 2 +- postgres-persistence/build.gradle | 1 + test-harness/build.gradle | 6 ++-- .../AbstractResiliencySpecification.groovy | 31 ++++++++++++++++++- .../test/base/AbstractSpecification.groovy | 8 +++-- .../WorkflowAndTaskConfigurationSpec.groovy | 7 +++++ .../integration/AbstractEndToEndTest.java | 8 +++-- .../grpc/AbstractGrpcEndToEndTest.java | 4 ++- .../http/AbstractHttpEndToEndTest.java | 3 +- .../application-integrationtest.properties | 2 ++ test-util/build.gradle | 4 ++- .../test/base/AbstractSpecification.groovy | 3 +- .../grpc/AbstractGrpcEndToEndTest.java | 2 ++ .../application-integrationtest.properties | 2 ++ workflow-event-listener/build.gradle | 2 ++ ...orkflowStatusPublisherIntegrationTest.java | 2 ++ .../application-integrationtest.properties | 2 ++ 21 files changed, 81 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/update-gradle-wrapper.yml diff --git a/.github/workflows/update-gradle-wrapper.yml b/.github/workflows/update-gradle-wrapper.yml deleted file mode 100644 index 831e6d5aa..000000000 --- a/.github/workflows/update-gradle-wrapper.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Update Gradle Wrapper - -on: - schedule: - - cron: "0 0 * * *" - workflow_dispatch: - -jobs: - update-gradle-wrapper: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Zulu JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' - - name: Update Gradle Wrapper - uses: gradle-update/update-gradle-wrapper-action@v1 diff --git a/java-sdk/src/test/resources/application-integrationtest.properties b/java-sdk/src/test/resources/application-integrationtest.properties index 6025e57a9..efb46697d 100644 --- a/java-sdk/src/test/resources/application-integrationtest.properties +++ b/java-sdk/src/test/resources/application-integrationtest.properties @@ -14,6 +14,8 @@ # conductor.db.type=memory +# disable trying to connect to redis and use in-memory +conductor.queue.type=xxx conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=mock conductor.indexing.enabled=false diff --git a/kafka/build.gradle b/kafka/build.gradle index 13339d752..847a9ba5d 100644 --- a/kafka/build.gradle +++ b/kafka/build.gradle @@ -26,5 +26,6 @@ dependencies { testImplementation project(':conductor-test-util') testImplementation project(':conductor-test-util').sourceSets.test.output + testImplementation "redis.clients:jedis:${revJedis}" } diff --git a/kafka/src/test/resources/application-integrationtest.properties b/kafka/src/test/resources/application-integrationtest.properties index 19e39ec47..a6432a1af 100644 --- a/kafka/src/test/resources/application-integrationtest.properties +++ b/kafka/src/test/resources/application-integrationtest.properties @@ -14,6 +14,8 @@ # conductor.db.type=memory +# disable trying to connect to redis and use in-memory +conductor.queue.type=xxx conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=dummy conductor.indexing.enabled=false diff --git a/mysql-persistence/build.gradle b/mysql-persistence/build.gradle index dccdfc32f..267dbee4f 100644 --- a/mysql-persistence/build.gradle +++ b/mysql-persistence/build.gradle @@ -33,7 +33,7 @@ dependencies { testImplementation project(':conductor-test-util').sourceSets.test.output testImplementation project(':conductor-common-persistence').sourceSets.test.output - + testImplementation "redis.clients:jedis:${revJedis}" } test { diff --git a/postgres-persistence/build.gradle b/postgres-persistence/build.gradle index 66155a5d6..acc925b51 100644 --- a/postgres-persistence/build.gradle +++ b/postgres-persistence/build.gradle @@ -29,6 +29,7 @@ dependencies { testImplementation project(':conductor-test-util').sourceSets.test.output testImplementation project(':conductor-common-persistence').sourceSets.test.output + testImplementation "redis.clients:jedis:${revJedis}" } diff --git a/test-harness/build.gradle b/test-harness/build.gradle index ea410db62..c250eddf6 100644 --- a/test-harness/build.gradle +++ b/test-harness/build.gradle @@ -7,7 +7,7 @@ dependencies { testImplementation project(':conductor-core') testImplementation project(':conductor-redis-persistence') testImplementation project(':conductor-cassandra-persistence') - testImplementation project(':conductor-es6-persistence') + testImplementation project(':conductor-es7-persistence') testImplementation project(':conductor-grpc-server') testImplementation project(':conductor-client') testImplementation project(':conductor-grpc-client') @@ -32,8 +32,8 @@ dependencies { testImplementation "org.spockframework:spock-core:${revSpock}" testImplementation "org.spockframework:spock-spring:${revSpock}" - testImplementation "org.elasticsearch.client:elasticsearch-rest-client:6.8.23" - testImplementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:6.8.23" + testImplementation "org.elasticsearch.client:elasticsearch-rest-client:${revElasticSearch7}" + testImplementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${revElasticSearch7}" testImplementation "org.testcontainers:elasticsearch:${revTestContainer}" testImplementation('junit:junit:4.13.2') diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractResiliencySpecification.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractResiliencySpecification.groovy index 0ad2a7f87..de0385f75 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractResiliencySpecification.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractResiliencySpecification.groovy @@ -14,14 +14,42 @@ package com.netflix.conductor.test.base import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.test.context.SpringBootTest import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.ComponentScan import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.FilterType import org.springframework.context.annotation.Primary import org.springframework.test.context.TestPropertySource +import com.netflix.conductor.ConductorTestApp +import com.netflix.conductor.core.config.SchedulerConfiguration +import com.netflix.conductor.core.events.DefaultEventProcessor +import com.netflix.conductor.core.events.DefaultEventQueueManager +import com.netflix.conductor.core.events.queue.ConductorEventQueueProvider +import com.netflix.conductor.core.execution.mapper.DoWhileTaskMapper +import com.netflix.conductor.core.execution.mapper.EventTaskMapper +import com.netflix.conductor.core.execution.mapper.ForkJoinDynamicTaskMapper +import com.netflix.conductor.core.execution.mapper.ForkJoinTaskMapper +import com.netflix.conductor.core.execution.mapper.HumanTaskMapper +import com.netflix.conductor.core.execution.mapper.JoinTaskMapper +import com.netflix.conductor.core.execution.mapper.SimpleTaskMapper +import com.netflix.conductor.core.execution.mapper.SubWorkflowTaskMapper +import com.netflix.conductor.core.execution.mapper.SwitchTaskMapper +import com.netflix.conductor.core.execution.mapper.WaitTaskMapper +import com.netflix.conductor.core.execution.tasks.DoWhile +import com.netflix.conductor.core.execution.tasks.Event +import com.netflix.conductor.core.execution.tasks.ExclusiveJoin +import com.netflix.conductor.core.execution.tasks.Human +import com.netflix.conductor.core.execution.tasks.Inline +import com.netflix.conductor.core.execution.tasks.Join +import com.netflix.conductor.core.execution.tasks.SetVariable +import com.netflix.conductor.core.execution.tasks.SubWorkflow +import com.netflix.conductor.core.execution.tasks.Wait import com.netflix.conductor.dao.QueueDAO import com.netflix.conductor.redis.dao.DynoQueueDAO import com.netflix.conductor.redis.jedis.JedisMock +import com.netflix.conductor.tasks.json.JsonJqTransform import com.netflix.dyno.connectionpool.Host import com.netflix.dyno.queues.ShardSupplier import com.netflix.dyno.queues.redis.RedisQueues @@ -33,7 +61,8 @@ import spock.mock.DetachedMockFactory "conductor.system-task-workers.enabled=false", "conductor.workflow-repair-service.enabled=true", "conductor.workflow-reconciler.enabled=false", - "conductor.integ-test.queue-spy.enabled=true" + "conductor.integ-test.queue-spy.enabled=true", + "conductor.queue.type=xxx" ]) abstract class AbstractResiliencySpecification extends AbstractSpecification { diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy index 974f30001..9aef149d9 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy @@ -16,6 +16,7 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.TestPropertySource +import com.netflix.conductor.ConductorTestApp import com.netflix.conductor.core.execution.AsyncSystemTaskExecutor import com.netflix.conductor.core.execution.StartWorkflowInput import com.netflix.conductor.core.execution.WorkflowExecutor @@ -27,8 +28,11 @@ import com.netflix.conductor.test.util.WorkflowTestUtil import spock.lang.Specification -@SpringBootTest -@TestPropertySource(locations = "classpath:application-integrationtest.properties") +@SpringBootTest(classes = ConductorTestApp.class) +@TestPropertySource(locations = "classpath:application-integrationtest.properties",properties = [ + "conductor.db.type=memory", + "conductor.queue.type=xxx" +]) abstract class AbstractSpecification extends Specification { @Autowired diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/integration/WorkflowAndTaskConfigurationSpec.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/integration/WorkflowAndTaskConfigurationSpec.groovy index d594fbfeb..fd27be739 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/integration/WorkflowAndTaskConfigurationSpec.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/integration/WorkflowAndTaskConfigurationSpec.groovy @@ -13,7 +13,10 @@ package com.netflix.conductor.test.integration import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.TestPropertySource +import com.netflix.conductor.ConductorTestApp import com.netflix.conductor.common.metadata.tasks.Task import com.netflix.conductor.common.metadata.tasks.TaskDef import com.netflix.conductor.common.metadata.tasks.TaskResult @@ -31,6 +34,10 @@ import spock.lang.Shared import static com.netflix.conductor.test.util.WorkflowTestUtil.verifyPolledAndAcknowledgedTask +@TestPropertySource(properties = [ + "conductor.db.type=memory", + "conductor.queue.type=xxx" +]) class WorkflowAndTaskConfigurationSpec extends AbstractSpecification { @Autowired diff --git a/test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java b/test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java index c110e6730..66b883022 100644 --- a/test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java +++ b/test-harness/src/test/java/com/netflix/conductor/test/integration/AbstractEndToEndTest.java @@ -48,7 +48,11 @@ import static org.junit.Assert.assertNotNull; @TestPropertySource( - properties = {"conductor.indexing.enabled=true", "conductor.elasticsearch.version=6"}) + properties = { + "conductor.indexing.enabled=true", + "conductor.elasticsearch.version=7", + "conductor.queue.type=xxx" + }) public abstract class AbstractEndToEndTest { private static final Logger log = LoggerFactory.getLogger(AbstractEndToEndTest.class); @@ -62,7 +66,7 @@ public abstract class AbstractEndToEndTest { private static final ElasticsearchContainer container = new ElasticsearchContainer( DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss") - .withTag("6.8.17")); // this should match the client version + .withTag("7.10.2")); // this should match the client version private static RestClient restClient; diff --git a/test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java b/test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java index 96ea7783f..0c4de90a6 100644 --- a/test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java +++ b/test-harness/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java @@ -22,6 +22,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import com.netflix.conductor.ConductorTestApp; import com.netflix.conductor.client.grpc.EventClient; import com.netflix.conductor.client.grpc.MetadataClient; import com.netflix.conductor.client.grpc.TaskClient; @@ -48,6 +49,7 @@ @RunWith(SpringRunner.class) @SpringBootTest( + classes = ConductorTestApp.class, properties = {"conductor.grpc-server.enabled=true", "conductor.grpc-server.port=8092"}) @TestPropertySource(locations = "classpath:application-integrationtest.properties") public abstract class AbstractGrpcEndToEndTest extends AbstractEndToEndTest { @@ -156,7 +158,7 @@ public void testAll() throws Exception { assertNotNull(polled); assertEquals(0, polled.size()); - polled = taskClient.batchPollTasksByTaskType(t0.getName(), "test", 1, 100); + polled = taskClient.batchPollTasksByTaskType(t0.getName(), "test", 1, 1000); assertNotNull(polled); assertEquals(1, polled.size()); assertEquals(t0.getName(), polled.get(0).getTaskDefName()); diff --git a/test-harness/src/test/java/com/netflix/conductor/test/integration/http/AbstractHttpEndToEndTest.java b/test-harness/src/test/java/com/netflix/conductor/test/integration/http/AbstractHttpEndToEndTest.java index b57066c01..279e38c5f 100644 --- a/test-harness/src/test/java/com/netflix/conductor/test/integration/http/AbstractHttpEndToEndTest.java +++ b/test-harness/src/test/java/com/netflix/conductor/test/integration/http/AbstractHttpEndToEndTest.java @@ -26,6 +26,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import com.netflix.conductor.ConductorTestApp; import com.netflix.conductor.client.exception.ConductorClientException; import com.netflix.conductor.client.http.EventClient; import com.netflix.conductor.client.http.MetadataClient; @@ -55,7 +56,7 @@ import static org.junit.Assert.fail; @RunWith(SpringRunner.class) -@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT) +@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT, classes = ConductorTestApp.class) @TestPropertySource(locations = "classpath:application-integrationtest.properties") public abstract class AbstractHttpEndToEndTest extends AbstractEndToEndTest { diff --git a/test-harness/src/test/resources/application-integrationtest.properties b/test-harness/src/test/resources/application-integrationtest.properties index af1517070..b209c8054 100644 --- a/test-harness/src/test/resources/application-integrationtest.properties +++ b/test-harness/src/test/resources/application-integrationtest.properties @@ -14,6 +14,8 @@ # conductor.db.type=memory +# disable trying to connect to redis and use in-memory +conductor.queue.type=xxx conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=mock conductor.indexing.enabled=false diff --git a/test-util/build.gradle b/test-util/build.gradle index b9fe3754b..b3280ca51 100644 --- a/test-util/build.gradle +++ b/test-util/build.gradle @@ -6,7 +6,7 @@ dependencies { implementation project(':conductor-common') implementation project(':conductor-core') - implementation project(':conductor-server') + compileOnly project(':conductor-server') implementation project(':conductor-client') implementation project(':conductor-rest') implementation project(':conductor-grpc-server') @@ -18,6 +18,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-core" implementation "org.apache.commons:commons-lang3" implementation 'org.springframework.boot:spring-boot-starter-validation' + implementation 'org.springframework.retry:spring-retry' implementation "com.fasterxml.jackson.core:jackson-databind" implementation "com.fasterxml.jackson.core:jackson-core" @@ -46,6 +47,7 @@ dependencies { //In memory implementation "org.rarefiedredis.redis:redis-java:${revRarefiedRedis}" + } test { diff --git a/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy b/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy index 2fde6d4f6..2f59d9d60 100644 --- a/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy +++ b/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy @@ -12,6 +12,7 @@ */ package com.netflix.conductor.test.base +import com.netflix.conductor.ConductorTestApp import com.netflix.conductor.service.WorkflowService import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest @@ -26,7 +27,7 @@ import com.netflix.conductor.test.util.WorkflowTestUtil import spock.lang.Specification -@SpringBootTest +@SpringBootTest(classes = ConductorTestApp.class) @TestPropertySource(locations = "classpath:application-integrationtest.properties") abstract class AbstractSpecification extends Specification { diff --git a/test-util/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java b/test-util/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java index 5f53592bb..5d43ef67d 100644 --- a/test-util/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java +++ b/test-util/src/test/java/com/netflix/conductor/test/integration/grpc/AbstractGrpcEndToEndTest.java @@ -20,6 +20,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import com.netflix.conductor.ConductorTestApp; import com.netflix.conductor.client.grpc.EventClient; import com.netflix.conductor.client.grpc.MetadataClient; import com.netflix.conductor.client.grpc.TaskClient; @@ -46,6 +47,7 @@ @RunWith(SpringRunner.class) @SpringBootTest( + classes = ConductorTestApp.class, properties = {"conductor.grpc-server.enabled=true", "conductor.grpc-server.port=8092"}) @TestPropertySource(locations = "classpath:application-integrationtest.properties") public abstract class AbstractGrpcEndToEndTest extends AbstractEndToEndTest { diff --git a/test-util/src/test/resources/application-integrationtest.properties b/test-util/src/test/resources/application-integrationtest.properties index 3b9e396ae..2b0e07151 100644 --- a/test-util/src/test/resources/application-integrationtest.properties +++ b/test-util/src/test/resources/application-integrationtest.properties @@ -1,4 +1,6 @@ conductor.db.type=memory +# disable trying to connect to redis and use in-memory +conductor.queue.type=xxx conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=dummy conductor.indexing.enabled=true diff --git a/workflow-event-listener/build.gradle b/workflow-event-listener/build.gradle index b30e145ee..bf322be9a 100644 --- a/workflow-event-listener/build.gradle +++ b/workflow-event-listener/build.gradle @@ -22,4 +22,6 @@ dependencies { //In memory implementation "org.rarefiedredis.redis:redis-java:${revRarefiedRedis}" + testImplementation "redis.clients:jedis:${revJedis}" + } \ No newline at end of file diff --git a/workflow-event-listener/src/test/java/com/netflix/conductor/test/listener/WorkflowStatusPublisherIntegrationTest.java b/workflow-event-listener/src/test/java/com/netflix/conductor/test/listener/WorkflowStatusPublisherIntegrationTest.java index 3aa8b072b..33d909a84 100644 --- a/workflow-event-listener/src/test/java/com/netflix/conductor/test/listener/WorkflowStatusPublisherIntegrationTest.java +++ b/workflow-event-listener/src/test/java/com/netflix/conductor/test/listener/WorkflowStatusPublisherIntegrationTest.java @@ -29,6 +29,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import com.netflix.conductor.ConductorTestApp; import com.netflix.conductor.common.metadata.tasks.Task; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskResult; @@ -51,6 +52,7 @@ @RunWith(SpringRunner.class) @SpringBootTest( + classes = ConductorTestApp.class, properties = { "conductor.db.type=memory", "conductor.workflow-status-listener.type=queue_publisher", diff --git a/workflow-event-listener/src/test/resources/application-integrationtest.properties b/workflow-event-listener/src/test/resources/application-integrationtest.properties index 19e39ec47..a6432a1af 100644 --- a/workflow-event-listener/src/test/resources/application-integrationtest.properties +++ b/workflow-event-listener/src/test/resources/application-integrationtest.properties @@ -14,6 +14,8 @@ # conductor.db.type=memory +# disable trying to connect to redis and use in-memory +conductor.queue.type=xxx conductor.workflow-execution-lock.type=local_only conductor.external-payload-storage.type=dummy conductor.indexing.enabled=false