add collections calls to PineconeControlPlaneClient, add new integrat…

…ion tests for collections, add new helpers for new collections tests

build.gradle

@@ -1,3 +1,6 @@
+import org.gradle.api.tasks.testing.logging.TestExceptionFormat;
+import org.gradle.api.tasks.testing.logging.TestLogEvent;
+
 plugins {
     id 'com.github.johnrengelman.shadow' version '6.1.0'
     id 'java-library'
@@ -93,13 +96,74 @@ tasks.named('build') {
 
 test {
     useJUnitPlatform()
+
+
+    testLogging {
+        events = [TestLogEvent.FAILED, TestLogEvent.PASSED, TestLogEvent.SKIPPED, TestLogEvent.STANDARD_OUT]
+        exceptionFormat = TestExceptionFormat.FULL
+        showExceptions = true
+        showCauses = true
+        showStackTraces = true
+
+        debugOptions {
+            events = [
+                    TestLogEvent.STARTED,
+                    TestLogEvent.FAILED,
+                    TestLogEvent.PASSED,
+                    TestLogEvent.SKIPPED,
+                    TestLogEvent.STANDARD_ERROR,
+                    TestLogEvent.STANDARD_OUT
+            ]
+            exceptionFormat = TestExceptionFormat.FULL
+        }
+
+        afterSuite { desc, result ->
+            if (!desc.parent) {
+                def output = "Results: ${result.resultType} (${result.testCount}) tests, ${result.successfulTestCount} successful, ${result.failedTestCount} failed, ${result.skippedTestCount} skipped."
+                def startItem = "| ", endItem = " |"
+                def repeatLength = startItem.length() + output.length() + endItem.length()
+                println('\n' + ('-' * repeatLength) + '\n' + startItem + output + endItem + '\n' + ('-' * repeatLength))
+            }
+        }
+    }
 }
 
 task integrationTest(type: Test) {
     useJUnitPlatform()
     testClassesDirs = sourceSets.integrationTest.output.classesDirs
     classpath = sourceSets.integrationTest.runtimeClasspath
     outputs.upToDateWhen { false }
+
+
+    testLogging {
+        events = [TestLogEvent.FAILED, TestLogEvent.PASSED, TestLogEvent.SKIPPED, TestLogEvent.STANDARD_OUT]
+        exceptionFormat = TestExceptionFormat.FULL
+        showExceptions = true
+        showCauses = true
+        showStackTraces = true
+
+        debugOptions {
+            events = [
+                    TestLogEvent.STARTED,
+                    TestLogEvent.FAILED,
+                    TestLogEvent.PASSED,
+                    TestLogEvent.SKIPPED,
+                    TestLogEvent.STANDARD_ERROR,
+                    TestLogEvent.STANDARD_OUT
+            ]
+            exceptionFormat = TestExceptionFormat.FULL
+
+        }
+
+        afterSuite { desc, result ->
+            if (!desc.parent) {
+                def output = "Results: ${result.resultType} (${result.testCount}) tests, ${result.successfulTestCount} successful, ${result.failedTestCount} failed, ${result.skippedTestCount} skipped."
+                def startItem = "| ", endItem = " |"
+                def repeatLength = startItem.length() + output.length() + endItem.length()
+                println('\n' + ('-' * repeatLength) + '\n' + startItem + output + endItem + '\n' + ('-' * repeatLength))
+            }
+        }
+    }
 }
 
 // Configure Auto Relocation

src/integration/java/io/pinecone/helpers/AssertRetry.java

@@ -1,18 +1,21 @@
 package io.pinecone.helpers;
 
+import io.pinecone.exceptions.PineconeException;
+
 import java.io.IOException;
 import java.util.concurrent.ExecutionException;
 
 public class AssertRetry {
     private static final int maxRetry = 4;
-    private static int delay = 1500;
+    private static final int delay = 1500;
 
-    public static void assertWithRetry(AssertionRunnable assertionRunnable) throws InterruptedException {
+    public static void assertWithRetry(AssertionRunnable assertionRunnable) throws InterruptedException, PineconeException {
         assertWithRetry(assertionRunnable, 2);
     }
 
-    public static void assertWithRetry(AssertionRunnable assertionRunnable, int backOff) throws InterruptedException {
+    public static void assertWithRetry(AssertionRunnable assertionRunnable, int backOff) throws InterruptedException, PineconeException {
         int retryCount = 0;
+        int delayCount = delay;
         boolean success = false;
 
         while (retryCount < maxRetry && !success) {
@@ -21,14 +24,14 @@ public static void assertWithRetry(AssertionRunnable assertionRunnable, int back
                 success = true;
             } catch (AssertionError | ExecutionException | IOException e) {
                 retryCount++;
-                delay*=backOff;
-                Thread.sleep(delay);
+                Thread.sleep(delayCount);
+                delayCount*=backOff;
             }
         }
     }
 
     @FunctionalInterface
     public interface AssertionRunnable {
-        void run() throws AssertionError, ExecutionException, InterruptedException, IOException;
+        void run() throws AssertionError, ExecutionException, InterruptedException, IOException, PineconeException;
     }
 }

src/integration/java/io/pinecone/helpers/IndexManager.java

@@ -1,12 +1,15 @@
 package io.pinecone.helpers;
 
 import io.pinecone.*;
+import io.pinecone.exceptions.PineconeException;
 import org.openapitools.client.model.*;
 
 import java.io.IOException;
 import java.util.List;
 
 import static io.pinecone.helpers.AssertRetry.assertWithRetry;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class IndexManager {
     private static PineconeClientConfig config;
@@ -79,6 +82,72 @@ private static String createNewIndex(PineconeControlPlaneClient controlPlaneClie
         return indexName;
     }
 
+    public static IndexModel waitUntilIndexIsReady(PineconeControlPlaneClient controlPlaneClient, String indexName, Integer totalMsToWait) throws InterruptedException {
+        IndexModel index = controlPlaneClient.describeIndex(indexName);
+        int waitedTimeMs = 0;
+        int intervalMs = 1500;
+
+        while (!index.getStatus().getReady()) {
+            index = controlPlaneClient.describeIndex(indexName);
+            if (waitedTimeMs >= totalMsToWait) {
+                System.out.println("Index " + indexName + " not ready after " + waitedTimeMs + "ms");
+                break;
+            }
+            if (index.getStatus().getReady()) {
+                Thread.sleep(3000);
+                System.out.println("Index " + indexName + " is ready after " + waitedTimeMs + 3000 + "ms");
+                break;
+            }
+            Thread.sleep(intervalMs);
+            waitedTimeMs += intervalMs;
+        }
+        return index;
+    }
+
+    public static IndexModel waitUntilIndexIsReady(PineconeControlPlaneClient controlPlaneClient, String indexName) throws InterruptedException {
+        return waitUntilIndexIsReady(controlPlaneClient, indexName, 120000);
+    }
+
+    public static PineconeConnection createNewIndexAndConnect(PineconeControlPlaneClient controlPlaneClient, String indexName, int dimension, IndexMetric metric, CreateIndexRequestSpec spec) throws InterruptedException, PineconeException {
+        CreateIndexRequest createIndexRequest = new CreateIndexRequest().name(indexName).dimension(dimension).metric(metric).spec(spec);
+        controlPlaneClient.createIndex(createIndexRequest);
+
+        // Wait until index is ready
+        waitUntilIndexIsReady(controlPlaneClient, indexName, 200000);
+        String host = controlPlaneClient.describeIndex(indexName).getHost();
+
+        PineconeClientConfig specificConfig = new PineconeClientConfig().withApiKey(System.getenv("PINECONE_API_KEY"));
+        PineconeClient dataPlaneClient = new PineconeClient(specificConfig);
+
+        return dataPlaneClient.connect(
+                new PineconeConnectionConfig()
+                        .withConnectionUrl("https://" + host));
+    }
+
+    public static CollectionModel createCollection(PineconeControlPlaneClient controlPlaneClient, String collectionName, String indexName, boolean waitUntilReady) throws InterruptedException {
+        CreateCollectionRequest createCollectionRequest = new CreateCollectionRequest().name(collectionName).source(indexName);
+        CollectionModel collection = controlPlaneClient.createCollection(createCollectionRequest);
+
+        assertEquals(collection.getStatus(), CollectionModel.StatusEnum.INITIALIZING);
+
+        // Wait until collection is ready
+        int timeWaited = 0;
+        CollectionModel.StatusEnum collectionReady = collection.getStatus();
+        while (collectionReady != CollectionModel.StatusEnum.READY && timeWaited < 120000) {
+            System.out.println("Waiting for collection" + collectionName + " to be ready. Waited " + timeWaited + " milliseconds...");
+            Thread.sleep(5000);
+            timeWaited += 5000;
+            collection = controlPlaneClient.describeCollection(collectionName);
+            collectionReady = collection.getStatus();
+        }
+
+        if (timeWaited > 120000) {
+            fail("Collection: " + collectionName + " is not ready after 120 seconds");
+        }
+
+        return collection;
+    }
+
     public static IndexModel isIndexReady(String indexName, PineconeControlPlaneClient controlPlaneClient)
             throws InterruptedException {
         final IndexModel[] indexModels = new IndexModel[1];

src/integration/java/io/pinecone/integration/controlPlane/pod/CollectionErrorTest.java

@@ -0,0 +1,143 @@
+package io.pinecone.integration.controlPlane.pod;
+
+import io.pinecone.PineconeConnection;
+import io.pinecone.PineconeControlPlaneClient;
+import io.pinecone.exceptions.PineconeException;
+import io.pinecone.helpers.RandomStringBuilder;
+import io.pinecone.proto.VectorServiceGrpc;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Disabled;
+import org.openapitools.client.model.*;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
+import static io.pinecone.helpers.IndexManager.createNewIndexAndConnect;
+import static io.pinecone.helpers.IndexManager.createCollection;
+import static io.pinecone.helpers.IndexManager.waitUntilIndexIsReady;
+import static io.pinecone.helpers.BuildUpsertRequest.*;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class CollectionErrorTest {
+    private static final String apiKey = System.getenv("PINECONE_API_KEY");
+    private static final String environment = System.getenv("PINECONE_ENVIRONMENT");
+    private static final String indexName = RandomStringBuilder.build("collection-error-test", 8);
+    private static final List<String> upsertIds = Arrays.asList("v1", "v2", "v3");
+    private static final int dimension = 3;
+    private static final String collectionName = RandomStringBuilder.build("reusable-coll", 8);
+    private static PineconeControlPlaneClient controlPlaneClient;
+
+    @BeforeAll
+    public static void setUpIndexAndCollection() throws InterruptedException {
+        controlPlaneClient = new PineconeControlPlaneClient(apiKey);
+        CreateIndexRequestSpecPod podSpec = new CreateIndexRequestSpecPod().pods(1).podType("p1.x1").replicas(1).environment(environment);
+        CreateIndexRequestSpec spec = new CreateIndexRequestSpec().pod(podSpec);
+        PineconeConnection dataPlaneConnection = createNewIndexAndConnect(controlPlaneClient, indexName, dimension, IndexMetric.COSINE, spec);
+        VectorServiceGrpc.VectorServiceBlockingStub blockingStub = dataPlaneConnection.getBlockingStub();
+
+        // Upsert vectors to index and sleep for freshness
+        blockingStub.upsert(buildRequiredUpsertRequest(upsertIds, ""));
+        dataPlaneConnection.close();
+
+        Thread.sleep(3500);
+
+        // Create collection from index
+        createCollection(controlPlaneClient, collectionName, indexName, true);
+    }
+
+    @AfterAll
+    public static void cleanUp() {
+        controlPlaneClient.deleteIndex(indexName);
+        controlPlaneClient.deleteCollection(collectionName);
+    }
+
+    @Test
+    public void testCreateCollectionFromInvalidIndex() {
+        try {
+            CreateCollectionRequest createCollectionRequest = new CreateCollectionRequest().name(RandomStringBuilder.build("coll1", 8)).source("invalid-index");
+            controlPlaneClient.createCollection(createCollectionRequest);
+        } catch (PineconeException exception) {
+            assertTrue(exception.getMessage().contains("Resource invalid-index not found"));
+        }
+    }
+    @Test
+    public void testIndexFromNonExistentCollection() {
+        try {
+            CreateIndexRequestSpecPod podSpec = new CreateIndexRequestSpecPod().environment(environment).sourceCollection("non-existent-collection");
+            CreateIndexRequestSpec spec = new CreateIndexRequestSpec().pod(podSpec);
+            CreateIndexRequest newCreateIndexRequest = new CreateIndexRequest().name(RandomStringBuilder.build("from-nonexistent-coll", 8)).dimension(3).metric(IndexMetric.COSINE).spec(spec);
+            controlPlaneClient.createIndex(newCreateIndexRequest);
+        } catch (PineconeException exception) {
+            assertTrue(exception.getMessage().contains("Resource non-existent-collection not found"));
+        }
+    }
+
+    @Test
+    public void testCreateIndexInMismatchedEnvironment() {
+        try {
+            List<String> environments = new LinkedList<>(Arrays.asList(
+                    "eastus-azure",
+                    "eu-west4-gcp",
+                    "northamerica-northeast1-gcp",
+                    "us-central1-gcp",
+                    "us-west4-gcp",
+                    "asia-southeast1-gcp",
+                    "us-east-1-aws",
+                    "asia-northeast1-gcp",
+                    "eu-west1-gcp",
+                    "eu-east1-gcp",
+                    "eu-east4-gcp",
+                    "us-west1-gcp"
+            ));
+            CollectionModel collection = controlPlaneClient.describeCollection(collectionName);
+            environments.remove(collection.getEnvironment());
+            String mismatchedEnv = environments.get(new Random().nextInt(environments.size()));
+
+            CreateIndexRequestSpecPod podSpec = new CreateIndexRequestSpecPod().sourceCollection(collection.getName()).environment(mismatchedEnv);
+            CreateIndexRequestSpec spec = new CreateIndexRequestSpec().pod(podSpec);
+            CreateIndexRequest createIndexRequest = new CreateIndexRequest().name(RandomStringBuilder.build("from-coll-", 8)).dimension(dimension).metric(IndexMetric.COSINE).spec(spec);
+            controlPlaneClient.createIndex(createIndexRequest);
+        } catch (PineconeException exception) {
+            assertTrue(exception.getMessage().contains("Source collection must be in the same environment as the index"));
+        }
+    }
+
+    @Test
+    @Disabled("Bug reported in #global-cps")
+    public void testCreateIndexWithMismatchedDimension() {
+        try {
+            CollectionModel collection = controlPlaneClient.describeCollection(collectionName);
+            CreateIndexRequestSpecPod podSpec = new CreateIndexRequestSpecPod().sourceCollection(collection.getName()).environment(collection.getEnvironment());
+            CreateIndexRequestSpec spec = new CreateIndexRequestSpec().pod(podSpec);
+            CreateIndexRequest createIndexRequest = new CreateIndexRequest().name(RandomStringBuilder.build("from-coll-", 8)).dimension(dimension + 1).metric(IndexMetric.COSINE).spec(spec);
+            controlPlaneClient.createIndex(createIndexRequest);
+        } catch (PineconeException exception) {
+            assertTrue(exception.getMessage().contains("Index and collection must have the same dimension"));
+        }
+    }
+
+    @Test
+    public void testCreateCollectionFromNotReadyIndex() throws InterruptedException {
+        String notReadyIndexName = RandomStringBuilder.build("from-coll4", 8);
+        try {
+            CreateIndexRequestSpecPod specPod = new CreateIndexRequestSpecPod().pods(1).podType("p1.x1").replicas(1).environment(environment);
+            CreateIndexRequestSpec spec = new CreateIndexRequestSpec().pod(specPod);
+            CreateIndexRequest createIndexRequest = new CreateIndexRequest().name(notReadyIndexName).dimension(dimension).metric(IndexMetric.COSINE).spec(spec);
+            controlPlaneClient.createIndex(createIndexRequest);
+
+            CreateCollectionRequest createCollectionRequest = new CreateCollectionRequest().name(RandomStringBuilder.build("coll4-", 8)).source(notReadyIndexName);
+            controlPlaneClient.createCollection(createCollectionRequest);
+        } catch (PineconeException exception) {
+            assertTrue(exception.getMessage().contains("Source index is not ready"));
+
+            // Wait for index to initialize and clean up
+            waitUntilIndexIsReady(controlPlaneClient, notReadyIndexName);
+            controlPlaneClient.deleteIndex(notReadyIndexName);
+        }
+    }
+}