Add source_collection to indexMetaDatabase object and ignore newly added fields (#58)

## Problem

`source_collection` is a newly added field in `indexMetaDatabase` object
of describeIndex response body. But without adding it to the response
body of describe index call, it breaks.

## Solution

Added the field as well as added support to ignore newly added fields.

## Type of Change

- [X] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update
- [ ] Infrastructure change (CI configs, etc)
- [ ] Non-code change (docs, etc)
- [ ] None of the above: (explain here)

## Test Plan
Ran integration tests and added unit test for the same

Author: rohanshah18

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 [comment]: <> (When bumping [pc:VERSION_LATEST_RELEASE] create a new entry below)
 ### Unreleased version
+### v0.7.4
+- Add source_collection and support to ignore newly added fields to the response body for describeIndex's indexMetaDatabase object
+
 ### v0.7.3
 - Add assert with retry mechanism
 - Add deprecation warning for queries parameter

README.md

@@ -15,20 +15,20 @@ Maven:
 <dependency>
   <groupId>io.pinecone</groupId>
   <artifactId>pinecone-client</artifactId>
-  <version>0.7.3</version>
+  <version>0.7.4</version>
 </dependency>
 ```
 
 [comment]: <> (^ [pc:VERSION_LATEST_RELEASE])
 
 Gradle:
 ```
-implementation "io.pinecone:pinecone-client:0.7.3"
+implementation "io.pinecone:pinecone-client:0.7.4"
 ```
 
 [comment]: <> (^ [pc:VERSION_LATEST_RELEASE])
 
-Alternatively, you can use our standalone uberjar [pinecone-client-0.7.3-all.jar](https://repo1.maven.org/maven2/io/pinecone/pinecone-client/0.7.3/pinecone-client-0.7.3-all.jar), which bundles the pinecone client and all dependencies together inside a single jar. You can include this on your classpath like any 3rd party JAR without having to obtain the *pinecone-client* dependencies separately.
+Alternatively, you can use our standalone uberjar [pinecone-client-0.7.4-all.jar](https://repo1.maven.org/maven2/io/pinecone/pinecone-client/0.7.4/pinecone-client-0.7.4-all.jar), which bundles the pinecone client and all dependencies together inside a single jar. You can include this on your classpath like any 3rd party JAR without having to obtain the *pinecone-client* dependencies separately.
 
 [comment]: <> (^ [pc:VERSION_LATEST_RELEASE])
 

gradle.properties

@@ -1 +1 @@
-pineconeClientVersion = 0.7.3
+pineconeClientVersion = 0.7.4

src/integration/java/io/pinecone/helpers/RetryAssert.java renamed to src/integration/java/io/pinecone/helpers/AssertRetry.java

@@ -1,8 +1,9 @@
 package io.pinecone.helpers;
 
+import java.io.IOException;
 import java.util.concurrent.ExecutionException;
 
-public class RetryAssert {
+public class AssertRetry {
     private static final int maxRetry = 4;
     private static int delay = 1500;
 
@@ -14,7 +15,7 @@ public static void assertWithRetry(AssertionRunnable assertionRunnable) throws I
             try {
                 assertionRunnable.run();
                 success = true;
-            } catch (AssertionError | ExecutionException e) {
+            } catch (AssertionError | ExecutionException | IOException e) {
                 retryCount++;
                 delay*=2;
                 Thread.sleep(delay);
@@ -24,6 +25,6 @@ public static void assertWithRetry(AssertionRunnable assertionRunnable) throws I
 
     @FunctionalInterface
     public interface AssertionRunnable {
-        void run() throws AssertionError, ExecutionException, InterruptedException;
+        void run() throws AssertionError, ExecutionException, InterruptedException, IOException;
     }
 }

src/integration/java/io/pinecone/helpers/IndexManager.java

@@ -7,6 +7,8 @@
 import java.io.IOException;
 import java.util.List;
 
+import static io.pinecone.helpers.AssertRetry.assertWithRetry;
+
 public class IndexManager {
     private static PineconeClientConfig config;
 
@@ -67,15 +69,12 @@ private static String createNewIndex(int dimension, PineconeIndexOperationClient
 
     public static IndexMeta isIndexReady(String indexName, PineconeIndexOperationClient indexOperationClient)
             throws IOException, InterruptedException {
-        IndexMeta indexMeta;
-        while (true) {
-            indexMeta = indexOperationClient.describeIndex(indexName);
-            if (indexMeta.getStatus().getState().equalsIgnoreCase("ready")) {
-                break;
-            }
-            Thread.sleep(1000);
-        }
+        final IndexMeta[] indexMeta = new IndexMeta[1];
+        assertWithRetry(() -> {
+            indexMeta[0] = indexOperationClient.describeIndex(indexName);
+            assert (indexMeta[0].getStatus().getState().equalsIgnoreCase("ready"));
+        });
 
-        return indexMeta;
+        return indexMeta[0];
     }
 }

src/integration/java/io/pinecone/integration/controlPlane/ConfigureIndexTest.java

@@ -43,8 +43,8 @@ public void configureIndexWithInvalidIndexName() {
         try {
             isIndexReady(indexName, indexOperationClient);
             indexOperationClient.configureIndex("non-existent-index", configureIndexRequest);
-        } catch (PineconeNotFoundException exception) {
-            assert (exception.getLocalizedMessage().contains("404: Not Found"));
+        } catch (PineconeNotFoundException notFoundException) {
+            assert (notFoundException.getLocalizedMessage().toLowerCase().contains("not found"));
         } catch (IOException | InterruptedException exception) {
             logger.error(exception.toString());
         }
@@ -57,10 +57,8 @@ public void configureIndexExceedingQuota() {
         try {
             isIndexReady(indexName, indexOperationClient);
             indexOperationClient.configureIndex(indexName, configureIndexRequest);
-        } catch (PineconeBadRequestException exception) {
-            assert (exception.getLocalizedMessage().contains("The index exceeds the project quota"));
-            assert (exception.getLocalizedMessage().contains("Upgrade your account or change" +
-                    " the project settings to increase the quota."));
+        } catch (PineconeBadRequestException badRequestException) {
+            assert (badRequestException.getLocalizedMessage().contains("Capacity Reached. Increase your quota or upgrade to create more indexes."));
         } catch (Exception exception) {
             logger.error(exception.toString());
         }
@@ -110,8 +108,8 @@ public void changingBasePodType() {
 
             isIndexReady(indexName, indexOperationClient);
             indexOperationClient.configureIndex(indexName, configureIndexRequest);
-        } catch (PineconeBadRequestException pineconeBadRequestException) {
-            assertEquals(pineconeBadRequestException.getMessage(), "updating base pod type is not supported");
+        } catch (PineconeBadRequestException badRequestException) {
+            assertEquals(badRequestException.getMessage(), "Bad request: Cannot change pod type.");
         } catch (Exception exception) {
             logger.error(exception.getLocalizedMessage());
         }
@@ -166,7 +164,7 @@ public void sizeDown() throws IOException, InterruptedException {
             Thread.sleep(3500);
         } catch (Exception exception) {
             assertEquals(exception.getClass(), PineconeBadRequestException.class);
-            assertEquals(exception.getMessage(), "scaling down pod type is not supported");
+            assert(exception.getMessage().contains("Bad request: Cannot scale down an index, only up."));
         } finally {
             // Delete this index since it'll be unused for other tests
             indexOperationClient.deleteIndex(indexName);

src/integration/java/io/pinecone/integration/dataplane/PineconeClientLiveIntegTest.java

@@ -18,10 +18,7 @@
 import java.util.List;
 
 import static io.pinecone.helpers.IndexManager.createIndexIfNotExistsDataPlane;
-import static io.pinecone.helpers.RetryAssert.assertWithRetry;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
+import static io.pinecone.helpers.AssertRetry.assertWithRetry;
 
 public class PineconeClientLiveIntegTest {
 
@@ -155,8 +152,7 @@ public void sanity() throws InterruptedException {
 
         assertWithRetry(() -> {
             QueryResponse queryResponse = blockingStub.query(queryByIdRequest);
-            assertThat(queryResponse, notNullValue());
-            assertThat(queryResponse.getMatchesCount(), equalTo(1));
+            Assertions.assertEquals(queryResponse.getMatchesCount(),1);
             assert (queryResponse.getMatches(0).getValuesList().containsAll(updateValueList));
         });
 

src/integration/java/io/pinecone/integration/dataplane/UpsertAndDescribeIndexStatsTest.java

@@ -6,7 +6,7 @@
 
 import static io.pinecone.helpers.BuildUpsertRequest.*;
 import static io.pinecone.helpers.IndexManager.createIndexIfNotExistsDataPlane;
-import static io.pinecone.helpers.RetryAssert.assertWithRetry;
+import static io.pinecone.helpers.AssertRetry.assertWithRetry;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import java.io.IOException;

src/main/java/io/pinecone/PineconeClientConfig.java

@@ -139,7 +139,7 @@ private String maskedApiKey() {
     }
 
     public String getUserAgent() {
-        String userAgentLanguage = "lang=java; pineconeClientVersion = v0.7.3";
+        String userAgentLanguage = "lang=java; pineconeClientVersion = v0.7.4";
         return (this.getUsageContext() != null) ?
                 userAgentLanguage + "; usageContext=" + this.getUsageContext() : userAgentLanguage;
     }

src/main/java/io/pinecone/model/CreateIndexRequest.java

@@ -12,7 +12,7 @@ public class CreateIndexRequest {
 
     private String metric = "cosine";
 
-    private Integer pods = 1;
+    private Integer pods;
 
     private Integer replicas = 1;
 

src/main/java/io/pinecone/model/IndexMetaDatabase.java

@@ -1,7 +1,9 @@
 package io.pinecone.model;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class IndexMetaDatabase {
     private String name;
 
@@ -18,6 +20,9 @@ public class IndexMetaDatabase {
     @JsonProperty("pod_type")
     private String podType;
 
+    @JsonProperty("source_collection")
+    private String sourceCollection;
+
     private IndexMetadataConfig metadataConfig;
 
     public IndexMetaDatabase () {
@@ -87,6 +92,15 @@ public IndexMetaDatabase withPodType(String podType) {
         return this;
     }
 
+    public String getSourceCollection() {
+        return sourceCollection;
+    }
+
+    public IndexMetaDatabase withSourceCollection(String sourceCollection) {
+        this.sourceCollection = sourceCollection;
+        return this;
+    }
+
     public IndexMetadataConfig getMetadataConfig() {
         return metadataConfig;
     }
@@ -107,6 +121,7 @@ public String toString() {
                 ", shards=" + shards +
                 ", podType='" + podType + '\'' +
                 ", metadataConfig=" + metadataConfig +
+                ", sourceCollection=" + sourceCollection +
                 '}';
     }
 }

src/test/java/io/pinecone/PineconeIndexOperationClientTest.java

@@ -10,6 +10,8 @@
 import org.junit.jupiter.api.BeforeAll;
 
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.List;
 
@@ -176,10 +178,59 @@ public void testCreateIndexWithAllFields() throws IOException {
     @Test
     public void testDescribeIndex() throws IOException {
         String indexName = "testIndex";
-        String indexJsonString = "{\"database\":{\"name\":\"testIndex\",\"metric\":\"cosine\",\"dimension\":3," +
-                "\"replicas\":1,\"shards\":1,\"pods\":1,\"pod_type\":\"p1.x1\"}," +
-                "\"status\":{\"waiting\":[],\"crashed\":[],\"host\":\"url\",\"port\":433," +
-                "\"state\":\"Ready\",\"ready\":true}}";
+        String indexString = "";
+        String filePath = "src/test/resources/indexJsonString.json";
+        String indexJsonString = new String(Files.readAllBytes(Paths.get(filePath)));
+
+        PineconeClientConfig clientConfig = new PineconeClientConfig()
+                .withApiKey("testApiKey")
+                .withEnvironment("testEnvironment");
+        IndexMetaDatabase metaDatabase = new IndexMetaDatabase()
+                .withName("testIndex")
+                .withMetric("cosine")
+                .withDimension(3)
+                .withReplicas(1)
+                .withShards(1)
+                .withPods(1)
+                .withPodType("p1.x1")
+                .withSourceCollection("randomCollection");
+        IndexStatus indexStatus = new IndexStatus()
+                .withHost("url")
+                .withState("Ready")
+                .withReady(true)
+                .withPort("433");
+
+        IndexMeta expectedIndexMeta = new IndexMeta()
+                .withMetaDatabase(metaDatabase)
+                .withStatus(indexStatus);
+
+        Call mockCall = mock(Call.class);
+        Response mockResponse = new Response.Builder()
+                .request(new Request.Builder().url("http://localhost").build())
+                .protocol(Protocol.HTTP_1_1)
+                .code(200)
+                .message("OK")
+                .body(ResponseBody.create(indexJsonString, MediaType.parse("application/json")))
+                .build();
+
+        when(mockCall.execute()).thenReturn(mockResponse);
+
+        OkHttpClient mockClient = mock(OkHttpClient.class);
+        when(mockClient.newCall(any(Request.class))).thenReturn(mockCall);
+        when(mockCall.execute()).thenReturn(mockResponse);
+
+        PineconeIndexOperationClient indexOperationClient = new PineconeIndexOperationClient(clientConfig, mockClient);
+        IndexMeta actualIndexMeta = indexOperationClient.describeIndex(indexName);
+
+        assertEquals(expectedIndexMeta.toString(), actualIndexMeta.toString());
+    }
+
+    @Test
+    public void testDescribeIndexWithExtraFields() throws IOException {
+        String indexName = "testIndex";
+        String filePath = "src/test/resources/indexJsonStringWithExtraFields.json";
+        String indexJsonString = new String(Files.readAllBytes(Paths.get(filePath)));
+
         PineconeClientConfig clientConfig = new PineconeClientConfig()
                 .withApiKey("testApiKey")
                 .withEnvironment("testEnvironment");
@@ -191,6 +242,7 @@ public void testDescribeIndex() throws IOException {
                 .withShards(1)
                 .withPods(1)
                 .withPodType("p1.x1");
+
         IndexStatus indexStatus = new IndexStatus()
                 .withHost("url")
                 .withState("Ready")

src/test/resources/indexJsonString.json

@@ -0,0 +1 @@
+{"database":{"name":"testIndex","metric":"cosine","dimension":3,"replicas":1,"shards":1,"pods":1,"pod_type":"p1.x1", "source_collection":"randomCollection"},"status":{"waiting":[],"crashed":[],"host":"url","port":433,"state":"Ready","ready":true}}

src/test/resources/indexJsonStringWithExtraFields.json

@@ -0,0 +1 @@
+{"database":{"name":"testIndex","metric":"cosine","dimension":3,"replicas":1,"shards":1,"pods":1,"pod_type":"p1.x1","extra_field":"random1"},"status":{"waiting":[],"crashed":[],"host":"url","port":433,"state":"Ready","ready":true, "extra_field":"random2"}}