fix: failing integration tests due to not waiting for TAZ cluster to fully failover

.github/workflows/integration_tests.yml

@@ -14,6 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [ "3.8", "3.11" ]
+        db-engine: ["aurora", "multi-az"]
 
     steps:
       - name: 'Clone repository'
@@ -58,7 +59,7 @@ jobs:
 
       - name: 'Run Integration Tests'
         run: |
-          ./gradlew --no-parallel --no-daemon test-python-${{ matrix.python-version }} --info
+          ./gradlew --no-parallel --no-daemon test-python-${{ matrix.python-version }}-${{ matrix.db-engine }} --info
         env:
           RDS_CLUSTER_DOMAIN: ${{ secrets.DB_CONN_SUFFIX }}
           RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

tests/integration/container/utils/rds_test_utility.py

@@ -156,6 +156,7 @@ def failover_cluster_and_wait_until_writer_changed(
             sleep(1)
             cluster_address = socket.gethostbyname(cluster_endpoint)
 
+        self.make_sure_instances_up(self.get_instance_ids())
         self.logger.debug("Testing.FinishedFailover", initial_writer_id, str((perf_counter_ns() - start) / 1_000_000))
 
     def failover_cluster(self, cluster_id: Optional[str] = None) -> None:
@@ -403,7 +404,7 @@ def make_sure_instances_up(self, instances: List[str]) -> None:
             instance_info: TestInstanceInfo = database_info.get_instance(i)
             success: bool = False
             start_time = timeit.default_timer()
-            while (timeit.default_timer() - start_time) < 300:  # 5 min
+            while (timeit.default_timer() - start_time) < 600:  # 10 min
                 try:
                     conn = self._open_connection(instance_info)
                     conn.close()

tests/integration/host/build.gradle.kts

@@ -62,21 +62,43 @@ tasks.withType<Test> {
     systemProperty("java.util.logging.config.file", "${project.buildDir}/resources/test/logging-test.properties")
 }
 
-tasks.register<Test>("test-python-3.11") {
+tasks.register<Test>("test-python-3.11-aurora") {
     group = "verification"
     filter.includeTestsMatching("integration.host.TestRunner.runTests")
     doFirst {
         systemProperty("exclude-performance", "true")
         systemProperty("exclude-python-38", "true")
+        systemProperty("exclude-multi-az", "true")
+    }
+}
+
+tasks.register<Test>("test-python-3.8-aurora") {
+    group = "verification"
+    filter.includeTestsMatching("integration.host.TestRunner.runTests")
+    doFirst {
+        systemProperty("exclude-performance", "true")
+        systemProperty("exclude-python-311", "true")
+        systemProperty("exclude-multi-az", "true")
     }
 }
 
-tasks.register<Test>("test-python-3.8") {
+tasks.register<Test>("test-python-3.11-multi-az") {
+    group = "verification"
+    filter.includeTestsMatching("integration.host.TestRunner.runTests")
+    doFirst {
+        systemProperty("exclude-performance", "true")
+        systemProperty("exclude-python-38", "true")
+        systemProperty("exclude-aurora", "true")
+    }
+}
+
+tasks.register<Test>("test-python-3.8-multi-az") {
     group = "verification"
     filter.includeTestsMatching("integration.host.TestRunner.runTests")
     doFirst {
         systemProperty("exclude-performance", "true")
         systemProperty("exclude-python-311", "true")
+        systemProperty("exclude-aurora", "true")
     }
 }
 

tests/integration/host/src/test/java/integration/host/TestEnvironment.java

@@ -16,8 +16,6 @@
 
 package integration.host;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.rekawek.toxiproxy.ToxiproxyClient;
@@ -44,6 +42,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Logger;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.Network;
@@ -70,6 +69,7 @@ public class TestEnvironment implements AutoCloseable {
   private static final TestEnvironmentConfiguration config = new TestEnvironmentConfiguration();
   private static final boolean USE_OTLP_CONTAINER_FOR_TRACES = false;
 
+  private static final AtomicInteger ipAddressUsageRefCount = new AtomicInteger(0);
   private final TestEnvironmentInfo info =
       new TestEnvironmentInfo(); // only this info is passed to test container
 
@@ -102,12 +102,22 @@ private TestEnvironment(TestEnvironmentRequest request) {
   }
 
   public static TestEnvironment build(TestEnvironmentRequest request) throws IOException {
+      DatabaseEngineDeployment deployment = request.getDatabaseEngineDeployment();
+    if (deployment == DatabaseEngineDeployment.AURORA
+        || deployment == DatabaseEngineDeployment.RDS
+        || deployment == DatabaseEngineDeployment.RDS_MULTI_AZ) {
+      // These environment require creating external database cluster that should be publicly available.
+      // Corresponding AWS Security Groups should be configured and the test task runner IP address
+      // should be whitelisted.
+      ipAddressUsageRefCount.incrementAndGet();
+    }
+
     LOGGER.finest("Building test env: " + request.getEnvPreCreateIndex());
     preCreateEnvironment(request.getEnvPreCreateIndex());
 
     TestEnvironment env;
 
-    switch (request.getDatabaseEngineDeployment()) {
+    switch (deployment) {
       case DOCKER:
         env = new TestEnvironment(request);
         initDatabaseParams(env);
@@ -894,7 +904,11 @@ public void close() throws Exception {
 
   private void deleteDbCluster() {
     if (!this.reuseAuroraDbCluster && !StringUtils.isNullOrEmpty(this.runnerIP)) {
-      auroraUtil.ec2DeauthorizesIP(runnerIP);
+      if (ipAddressUsageRefCount.decrementAndGet() == 0) {
+        // Another test environments are still in use of test task runner IP address.
+        // The last execute tst environment will do the cleanup.
+        auroraUtil.ec2DeauthorizesIP(runnerIP);
+      }
     }
 
     if (!this.reuseAuroraDbCluster) {