Change to use rest/api/latest/search/jql

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.3.2 - 2025-04-29
+* [enhancement] Use new endpoint rest/api/latest/search/jql
+* PR [#88](https://github.com/treasure-data/embulk-input-jira/pull/88)
+
 ## 0.3.1 - 2023-05-19
 * [enhancement] Update library, minor code refactoring
 * PR [#78](https://github.com/treasure-data/embulk-input-jira/pull/78)

build.gradle

@@ -12,7 +12,7 @@ repositories {
     mavenCentral()
 }
 
-version = "0.3.1"
+version = "0.3.2"
 group = "com.treasuredata.embulk.plugins"
 description = "JIRA Embulk input plugin."
 

src/main/java/org/embulk/input/jira/Constant.java

@@ -1,5 +1,7 @@
 package org.embulk.input.jira;
 
+import com.google.gson.Gson;
+
 public final class Constant
 {
     public static final int MAX_RESULTS = 50;
@@ -12,7 +14,8 @@ public final class Constant
     public static final String DEFAULT_TIMESTAMP_PATTERN = "%Y-%m-%dT%H:%M:%S.%L%z";
 
     public static final String CREDENTIAL_URI_PATH = "rest/api/latest/myself";
-    public static final String SEARCH_URI_PATH = "rest/api/latest/search";
+    public static final String SEARCH_URI_PATH = "rest/api/latest/search/jql";
+    public static final Gson GSON = new Gson();
 
     private Constant(){}
 }

src/main/java/org/embulk/input/jira/JiraInputPlugin.java

@@ -37,7 +37,6 @@
 import java.util.SortedSet;
 import java.util.TreeSet;
 
-import static org.embulk.input.jira.Constant.GUESS_RECORDS_COUNT;
 import static org.embulk.input.jira.Constant.MAX_RESULTS;
 import static org.embulk.input.jira.Constant.PREVIEW_RECORDS_COUNT;
 
@@ -147,20 +146,22 @@ public TaskReport run(final TaskSource taskSource,
         jiraClient.checkUserCredentials(task);
         try (final PageBuilder pageBuilder = getPageBuilder(schema, output)) {
             if (isPreview()) {
-                final List<Issue> issues = jiraClient.searchIssues(task, 0, PREVIEW_RECORDS_COUNT);
+                final SearchResult result = jiraClient.searchIssues(task, null, PREVIEW_RECORDS_COUNT);
+                final List<Issue> issues = JiraUtil.fromSearchResult(result);
                 issues.forEach(issue -> JiraUtil.addRecord(issue, schema, task, pageBuilder));
             }
             else {
-                int currentPage = 0;
-                final int totalCount = jiraClient.getTotalCount(task);
-                final int totalPage = JiraUtil.calculateTotalPage(totalCount, MAX_RESULTS);
-                LOGGER.info(String.format("Total pages (%d)", totalPage));
-                while (currentPage < totalPage) {
-                    LOGGER.info(String.format("Fetching page %d/%d", (currentPage + 1), totalPage));
-                    final List<Issue> issues = jiraClient.searchIssues(task, (currentPage * MAX_RESULTS), MAX_RESULTS);
+                int totalRecords = 0;
+                String nextPageToken = null;
+                do {
+                    final SearchResult result = jiraClient.searchIssues(task, nextPageToken, MAX_RESULTS);
+                    nextPageToken = result.getNextPageToken();
+                    final List<Issue> issues = JiraUtil.fromSearchResult(result);
+                    totalRecords += issues.size();
                     issues.forEach(issue -> JiraUtil.addRecord(issue, schema, task, pageBuilder));
-                    currentPage++;
+                    LOGGER.info(String.format("Ingested %d items", totalRecords));
                 }
+                while (nextPageToken != null);
             }
             pageBuilder.finish();
         }
@@ -181,7 +182,8 @@ public ConfigDiff guess(final ConfigSource config)
 
     private List<ConfigDiff> getGuessedColumns(final JiraClient jiraClient, final PluginTask task)
     {
-        final List<Issue> issues = jiraClient.searchIssues(task, 0, GUESS_RECORDS_COUNT);
+        final SearchResult result = jiraClient.searchIssues(task, null, PREVIEW_RECORDS_COUNT);
+        final List<Issue> issues = JiraUtil.fromSearchResult(result);
         if (issues.isEmpty()) {
             throw new ConfigException("Could not guess schema due to empty data set");
         }

src/main/java/org/embulk/input/jira/SearchResult.java

@@ -0,0 +1,23 @@
+package org.embulk.input.jira;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+
+public class SearchResult
+{
+    private JsonElement issues;
+    private String nextPageToken;
+
+    public JsonElement getIssues()
+    {
+        if (issues == null || issues.isJsonNull()) {
+            issues = new JsonArray();
+        }
+        return issues;
+    }
+
+    public String getNextPageToken()
+    {
+        return nextPageToken;
+    }
+}

src/main/java/org/embulk/input/jira/client/JiraClient.java

@@ -6,6 +6,8 @@
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 import com.google.gson.JsonPrimitive;
+
+import org.apache.commons.lang3.StringUtils;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
@@ -18,8 +20,8 @@
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.util.EntityUtils;
 import org.embulk.config.ConfigException;
-import org.embulk.input.jira.Issue;
 import org.embulk.input.jira.JiraInputPlugin.PluginTask;
+import org.embulk.input.jira.SearchResult;
 import org.embulk.input.jira.util.JiraException;
 import org.embulk.input.jira.util.JiraUtil;
 import org.embulk.util.retryhelper.RetryExecutor;
@@ -31,18 +33,13 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map.Entry;
 import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-
 import static java.util.Base64.getEncoder;
 import static org.apache.http.HttpHeaders.ACCEPT;
 import static org.apache.http.HttpHeaders.AUTHORIZATION;
 import static org.apache.http.HttpHeaders.CONTENT_TYPE;
+import static org.embulk.input.jira.Constant.GSON;
 import static org.embulk.input.jira.Constant.HTTP_TIMEOUT;
-import static org.embulk.input.jira.Constant.MIN_RESULTS;
 
 public class JiraClient
 {
@@ -64,31 +61,12 @@ public void checkUserCredentials(final PluginTask task)
         }
     }
 
-    public List<Issue> searchIssues(final PluginTask task, final int startAt, final int maxResults)
-    {
-        final String response = searchJiraAPI(task, startAt, maxResults);
-        final JsonObject result = new JsonParser().parse(response).getAsJsonObject();
-        return StreamSupport.stream(result.get("issues").getAsJsonArray().spliterator(), false)
-                            .map(jsonElement -> {
-                                final JsonObject json = jsonElement.getAsJsonObject();
-                                final JsonObject fields = json.get("fields").getAsJsonObject();
-                                final Set<Entry<String, JsonElement>> entries = fields.entrySet();
-                                json.remove("fields");
-                                // Merged all properties in fields to the object
-                                for (final Entry<String, JsonElement> entry : entries) {
-                                    json.add(entry.getKey(), entry.getValue());
-                                }
-                                return new Issue(json);
-                            })
-                            .collect(Collectors.toList());
-    }
-
-    public int getTotalCount(final PluginTask task)
+    public SearchResult searchIssues(final PluginTask task, final String nextPageToken, final int maxResults)
     {
-        return new JsonParser().parse(searchJiraAPI(task, 0, MIN_RESULTS)).getAsJsonObject().get("total").getAsInt();
+        return GSON.fromJson(searchJiraAPI(task, nextPageToken, maxResults), SearchResult.class);
     }
 
-    private String searchJiraAPI(final PluginTask task, final int startAt, final int maxResults)
+    private String searchJiraAPI(final PluginTask task, final String nextPageToken, final int maxResults)
     {
         try {
             return RetryExecutor.builder()
@@ -101,7 +79,7 @@ private String searchJiraAPI(final PluginTask task, final int startAt, final int
                 @Override
                 public String call() throws Exception
                 {
-                    return authorizeAndRequest(task, JiraUtil.buildSearchUrl(task.getUri()), createSearchIssuesBody(task, startAt, maxResults));
+                    return authorizeAndRequest(task, JiraUtil.buildSearchUrl(task.getUri()), createSearchIssuesBody(task, nextPageToken, maxResults));
                 }
 
                 @Override
@@ -243,12 +221,14 @@ private HttpRequestBase createGetRequest(final PluginTask task, final String url
         return request;
     }
 
-    private String createSearchIssuesBody(final PluginTask task, final int startAt, final int maxResults)
+    private String createSearchIssuesBody(final PluginTask task, final String nextPageToken, final int maxResults)
     {
         final JsonObject body = new JsonObject();
         final Optional<String> jql = task.getJQL();
         body.add("jql", new JsonPrimitive(jql.orElse("")));
-        body.add("startAt", new JsonPrimitive(startAt));
+        if (StringUtils.isNotEmpty(nextPageToken)) {
+            body.add("nextPageToken", new JsonPrimitive(nextPageToken));
+        }
         body.add("maxResults", new JsonPrimitive(maxResults));
         final JsonArray fields = new JsonArray();
         fields.add("*all");

src/main/java/org/embulk/input/jira/util/JiraUtil.java

@@ -13,6 +13,7 @@
 import org.embulk.config.ConfigSource;
 import org.embulk.input.jira.Issue;
 import org.embulk.input.jira.JiraInputPlugin.PluginTask;
+import org.embulk.input.jira.SearchResult;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageBuilder;
@@ -27,6 +28,8 @@
 import java.time.Instant;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
@@ -277,4 +280,21 @@ public static LinkedHashMap<String, Object> toLinkedHashMap(final JsonObject flt
         }
         return result;
     }
+
+    public static List<Issue> fromSearchResult(final SearchResult result)
+    {
+      return StreamSupport.stream(result.getIssues().getAsJsonArray().spliterator(), false)
+          .map(jsonElement -> {
+              final JsonObject json = jsonElement.getAsJsonObject();
+              final JsonObject fields = json.get("fields").getAsJsonObject();
+              final Set<Entry<String, JsonElement>> entries = fields.entrySet();
+              json.remove("fields");
+              // Merged all properties in fields to the object
+              for (final Entry<String, JsonElement> entry : entries) {
+                  json.add(entry.getKey(), entry.getValue());
+              }
+              return new Issue(json);
+          })
+          .collect(Collectors.toList());
+    }
 }

src/test/java/org/embulk/input/jira/JiraInputPluginTest.java

@@ -117,8 +117,8 @@ public void test_runDynamicSchema_withResult() throws IOException
                 .thenReturn(new StringEntity(searchResponse.get("body").toString()));
 
         plugin.transaction(TestHelpers.dynamicSchemaConfig(), new Control());
-        // Check credential 1 + getTotal 1 + loadData 2
-        verify(jiraClient, times(4)).createHttpClient();
+        // Check credential 1 + loadData 2
+        verify(jiraClient, times(3)).createHttpClient();
         verify(pageBuilder, times(1)).addRecord();
         verify(pageBuilder, times(1)).finish();
     }
@@ -137,8 +137,8 @@ public void test_run_with1RecordsResult() throws IOException
                 .thenReturn(new StringEntity(searchResponse.get("body").toString()));
 
         plugin.transaction(config, new Control());
-        // Check credential 1 + getTotal 1 + loadData 1
-        verify(jiraClient, times(3)).createHttpClient();
+        // Check credential 1 + loadData 1
+        verify(jiraClient, times(2)).createHttpClient();
         verify(pageBuilder, times(1)).addRecord();
         verify(pageBuilder, times(1)).finish();
     }
@@ -148,18 +148,21 @@ public void test_run_with2PagesResult() throws IOException
     {
         final JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject();
         final JsonObject searchResponse = data.get("2PagesResult").getAsJsonObject();
+        final JsonObject emptyResponse = data.get("emptyResult").getAsJsonObject();
 
         when(statusLine.getStatusCode())
                 .thenReturn(authorizeResponse.get("statusCode").getAsInt())
-                .thenReturn(searchResponse.get("statusCode").getAsInt());
+                .thenReturn(searchResponse.get("statusCode").getAsInt())
+                .thenReturn(emptyResponse.get("statusCode").getAsInt());
         when(response.getEntity())
                 .thenReturn(new StringEntity(authorizeResponse.get("body").toString()))
-                .thenReturn(new StringEntity(searchResponse.get("body").toString()));
+                .thenReturn(new StringEntity(searchResponse.get("body").toString()))
+                .thenReturn(new StringEntity(emptyResponse.get("body").toString()));
 
         plugin.transaction(config, new Control());
-        // Check credential 1 + getTotal 1 + loadData 2
-        verify(jiraClient, times(4)).createHttpClient();
-        verify(pageBuilder, times(2)).addRecord();
+        // Check credential 1 + loadData 2
+        verify(jiraClient, times(3)).createHttpClient();
+        verify(pageBuilder, times(1)).addRecord();
         verify(pageBuilder, times(1)).finish();
     }