diff --git a/microservices/configcheck/src/test/resources/input/microservice/QueryLogicFactory.xml b/microservices/configcheck/src/test/resources/input/microservice/QueryLogicFactory.xml
index e6233a4fac1..49e272ecc5f 100644
--- a/microservices/configcheck/src/test/resources/input/microservice/QueryLogicFactory.xml
+++ b/microservices/configcheck/src/test/resources/input/microservice/QueryLogicFactory.xml
@@ -114,6 +114,10 @@
+
+
+
+
diff --git a/microservices/configcheck/src/test/resources/input/webservice/QueryLogicFactory.xml b/microservices/configcheck/src/test/resources/input/webservice/QueryLogicFactory.xml
index 8c40461dcdf..f7d81628ee7 100644
--- a/microservices/configcheck/src/test/resources/input/webservice/QueryLogicFactory.xml
+++ b/microservices/configcheck/src/test/resources/input/webservice/QueryLogicFactory.xml
@@ -284,6 +284,8 @@
+
+
@@ -324,6 +326,10 @@
+
+ ${local.ivarator.cache.dir.config}
+
+
${ivarator.cache.dir.config}
diff --git a/microservices/configcheck/src/test/resources/input/webservice/properties/default.properties b/microservices/configcheck/src/test/resources/input/webservice/properties/default.properties
index 2adeacbcc0e..e76380a8030 100644
--- a/microservices/configcheck/src/test/resources/input/webservice/properties/default.properties
+++ b/microservices/configcheck/src/test/resources/input/webservice/properties/default.properties
@@ -451,6 +451,10 @@ hdfs.site.config.urls=file:///etc/hadoop/conf/core-site.xml,file:///etc/hadoop/c
## - minAvailableStoragePercent - (Optional, Default: 0.0) A double, between 0.0 and 1.0, which specifies the
## minimum percent of available storage space required to persist to this ivarator path. If less than this
## percent is available, we will not use this ivarator path.
+local.ivarator.cache.dir.config= \
+ \
+\n \
+\n
ivarator.cache.dir.config= \
\
\n \
diff --git a/microservices/configcheck/src/test/resources/rendered/microservice/QueryLogicFactory.xml b/microservices/configcheck/src/test/resources/rendered/microservice/QueryLogicFactory.xml
index bb07d23456d..8de5a940ece 100644
--- a/microservices/configcheck/src/test/resources/rendered/microservice/QueryLogicFactory.xml
+++ b/microservices/configcheck/src/test/resources/rendered/microservice/QueryLogicFactory.xml
@@ -114,6 +114,10 @@
+
+
+
+
diff --git a/microservices/configcheck/src/test/resources/rendered/webservice/QueryLogicFactory.xml b/microservices/configcheck/src/test/resources/rendered/webservice/QueryLogicFactory.xml
index 558d4b05073..595e52252fe 100644
--- a/microservices/configcheck/src/test/resources/rendered/webservice/QueryLogicFactory.xml
+++ b/microservices/configcheck/src/test/resources/rendered/webservice/QueryLogicFactory.xml
@@ -276,6 +276,8 @@
+
+
@@ -316,6 +318,12 @@
+
+
+
+
+
+
diff --git a/microservices/starters/query b/microservices/starters/query
index 165530a2b38..368802eee72 160000
--- a/microservices/starters/query
+++ b/microservices/starters/query
@@ -1 +1 @@
-Subproject commit 165530a2b3802a696ec665412516d9d43b850ba7
+Subproject commit 368802eee7271080798af60c05160a032a85484e
diff --git a/properties/bamboo.properties b/properties/bamboo.properties
index 16a50283f28..961ffa466af 100644
--- a/properties/bamboo.properties
+++ b/properties/bamboo.properties
@@ -1,4 +1,5 @@
#This is for the unit testing of spring configuration files by bamboo
accumulo.user.password=testdummy
+local.ivarator.cache.dir.config=
ivarator.cache.dir.config=
ivarator.fst.hdfs.base.uris=
diff --git a/properties/default.properties b/properties/default.properties
index c4ca9644aab..b7630b05cd8 100644
--- a/properties/default.properties
+++ b/properties/default.properties
@@ -497,6 +497,10 @@ hdfs.site.config.urls=file:///etc/hadoop/conf/core-site.xml,file:///etc/hadoop/c
## - minAvailableStoragePercent - (Optional, Default: 0.0) A double, between 0.0 and 1.0, which specifies the
## minimum percent of available storage space required to persist to this ivarator path. If less than this
## percent is available, we will not use this ivarator path.
+local.ivarator.cache.dir.config= \
+ \
+\n \
+\n
ivarator.cache.dir.config= \
\
\n \
diff --git a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
index 9807df08346..9268a093166 100644
--- a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
+++ b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
@@ -351,7 +351,10 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
private String hdfsSiteConfigURLs = null;
private String hdfsFileCompressionCodec = null;
private String zookeeperConfig = null;
+ // tserver side ivarator cache dir configs
private List ivaratorCacheDirConfigs = Collections.emptyList();
+ // webserver side ivarator cache dir configs (e.g. for unique transform on webserver)
+ private List localIvaratorCacheDirConfigs = Collections.emptyList();
private String ivaratorFstHdfsBaseURIs = null;
private int ivaratorCacheBufferSize = 10000;
@@ -671,6 +674,8 @@ public void copyFrom(ShardQueryConfiguration other) {
this.setHdfsSiteConfigURLs(other.getHdfsSiteConfigURLs());
this.setHdfsFileCompressionCodec(other.getHdfsFileCompressionCodec());
this.setZookeeperConfig(other.getZookeeperConfig());
+ this.setLocalIvaratorCacheDirConfigs(
+ null == other.getLocalIvaratorCacheDirConfigs() ? null : Lists.newArrayList(other.getLocalIvaratorCacheDirConfigs()));
this.setIvaratorCacheDirConfigs(null == other.getIvaratorCacheDirConfigs() ? null : Lists.newArrayList(other.getIvaratorCacheDirConfigs()));
this.setIvaratorFstHdfsBaseURIs(other.getIvaratorFstHdfsBaseURIs());
this.setIvaratorCacheBufferSize(other.getIvaratorCacheBufferSize());
@@ -766,6 +771,8 @@ public ShardQueryConfiguration(ShardQueryConfiguration other, Collection ivaratorCach
this.ivaratorCacheDirConfigs = ivaratorCacheDirConfigs;
}
+ public void setLocalIvaratorCacheDirConfigs(List localIvaratorCacheDirConfigs) {
+ this.localIvaratorCacheDirConfigs = localIvaratorCacheDirConfigs;
+ }
+
+ public List getLocalIvaratorCacheDirConfigs() {
+ return localIvaratorCacheDirConfigs;
+ }
+
public String getIvaratorFstHdfsBaseURIs() {
return ivaratorFstHdfsBaseURIs;
}
@@ -2951,6 +2966,7 @@ public boolean equals(Object o) {
Objects.equals(getHdfsSiteConfigURLs(), that.getHdfsSiteConfigURLs()) &&
Objects.equals(getHdfsFileCompressionCodec(), that.getHdfsFileCompressionCodec()) &&
Objects.equals(getZookeeperConfig(), that.getZookeeperConfig()) &&
+ Objects.equals(getLocalIvaratorCacheDirConfigs(), that.getLocalIvaratorCacheDirConfigs()) &&
Objects.equals(getIvaratorCacheDirConfigs(), that.getIvaratorCacheDirConfigs()) &&
Objects.equals(getIvaratorFstHdfsBaseURIs(), that.getIvaratorFstHdfsBaseURIs()) &&
Objects.equals(getQueryModel(), that.getQueryModel()) &&
@@ -3130,6 +3146,7 @@ public int hashCode() {
getHdfsSiteConfigURLs(),
getHdfsFileCompressionCodec(),
getZookeeperConfig(),
+ getLocalIvaratorCacheDirConfigs(),
getIvaratorCacheDirConfigs(),
getIvaratorFstHdfsBaseURIs(),
getIvaratorCacheBufferSize(),
diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java
index 23954939d0c..fc5d367269d 100644
--- a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java
+++ b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java
@@ -1548,7 +1548,7 @@ protected UniqueTransform getUniqueTransform() throws IOException {
.withUniqueFields(getUniqueFields())
.withQueryExecutionForPageTimeout(getResultTimeout())
.withBufferPersistThreshold(getUniqueCacheBufferSize())
- .withIvaratorCacheDirConfigs(getIvaratorCacheDirConfigs())
+ .withLocalIvaratorCacheDirConfigs(getIvaratorCacheDirConfigs())
.withHdfsSiteConfigURLs(getHdfsSiteConfigURLs())
.withSubDirectory(getQueryId() + "-" + getScanId())
.withMaxOpenFiles(getIvaratorMaxOpenFiles())
diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
index 32c1c611055..8803d927104 100644
--- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
+++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
@@ -714,7 +714,7 @@ private void addConfigBasedTransformers() throws QueryException {
.withQueryExecutionForPageTimeout(this.getQueryExecutionForPageTimeout())
.withModel(getQueryModel())
.withBufferPersistThreshold(getUniqueCacheBufferSize())
- .withIvaratorCacheDirConfigs(getIvaratorCacheDirConfigs())
+ .withLocalIvaratorCacheDirConfigs(getLocalIvaratorCacheDirConfigs())
.withHdfsSiteConfigURLs(getHdfsSiteConfigURLs())
.withSubDirectory(getConfig().getQuery().getId().toString())
.withMaxOpenFiles(getIvaratorMaxOpenFiles())
@@ -1963,6 +1963,14 @@ public void setFullTableScanEnabled(boolean fullTableScanEnabled) {
getConfig().setFullTableScanEnabled(fullTableScanEnabled);
}
+ public void setLocalIvaratorCacheDirConfigs(List localIvaratorCacheDirConfigs) {
+ getConfig().setLocalIvaratorCacheDirConfigs(localIvaratorCacheDirConfigs);
+ }
+
+ public List getLocalIvaratorCacheDirConfigs() {
+ return getConfig().getLocalIvaratorCacheDirConfigs();
+ }
+
public List getIvaratorCacheDirConfigs() {
return getConfig().getIvaratorCacheDirConfigs();
}
diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java b/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java
index d68ae473dbc..cf46ccee418 100644
--- a/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java
+++ b/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java
@@ -488,7 +488,7 @@ public Builder withBufferPersistThreshold(int bufferPersistThreshold) {
return this;
}
- public Builder withIvaratorCacheDirConfigs(List ivaratorCacheDirConfigs) {
+ public Builder withLocalIvaratorCacheDirConfigs(List ivaratorCacheDirConfigs) {
this.ivaratorCacheDirConfigs = ivaratorCacheDirConfigs;
return this;
}
diff --git a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
index e6aaaa47893..51e5212d78c 100644
--- a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
@@ -389,6 +389,8 @@ public void setUp() throws Exception {
updatedValues.put("hdfsFileCompressionCodec", "sunny");
defaultValues.put("zookeeperConfig", null);
updatedValues.put("zookeeperConfig", "file://etc/zookeeper/conf");
+ defaultValues.put("localIvaratorCacheDirConfigs", Collections.emptyList());
+ updatedValues.put("localIvaratorCacheDirConfigs", Lists.newArrayList(new IvaratorCacheDirConfig("file:///tmp/ivarators")));
defaultValues.put("ivaratorCacheDirConfigs", Collections.emptyList());
updatedValues.put("ivaratorCacheDirConfigs", Lists.newArrayList(new IvaratorCacheDirConfig("hdfs://instance-a/ivarators")));
defaultValues.put("ivaratorFstHdfsBaseURIs", null);
diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/UniqueTransformMostRecentTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/UniqueTransformMostRecentTest.java
index 5487ae7162c..14e52cb46ad 100644
--- a/warehouse/query-core/src/test/java/datawave/query/transformer/UniqueTransformMostRecentTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/transformer/UniqueTransformMostRecentTest.java
@@ -35,6 +35,7 @@ public void setUp() throws IOException {
// setup a directory for cache results
File tmpDir = temporaryFolder.newFolder();
IvaratorCacheDirConfig config = new IvaratorCacheDirConfig(tmpDir.toURI().toString());
+ logic.setLocalIvaratorCacheDirConfigs(Collections.singletonList(config));
logic.setIvaratorCacheDirConfigs(Collections.singletonList(config));
QueryImpl query = new QueryImpl();
@@ -50,7 +51,7 @@ protected UniqueTransform getUniqueTransform() {
.withUniqueFields(uniqueFields)
.withQueryExecutionForPageTimeout(Long.MAX_VALUE)
.withBufferPersistThreshold(logic.getUniqueCacheBufferSize())
- .withIvaratorCacheDirConfigs(logic.getIvaratorCacheDirConfigs())
+ .withLocalIvaratorCacheDirConfigs(logic.getLocalIvaratorCacheDirConfigs())
.withHdfsSiteConfigURLs(logic.getHdfsSiteConfigURLs())
.withSubDirectory(logic.getConfig().getQuery().getId().toString())
.withMaxOpenFiles(logic.getIvaratorMaxOpenFiles())
diff --git a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml
index ffa8088b8e0..3f85bd48e87 100644
--- a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml
+++ b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml
@@ -114,6 +114,7 @@
+
diff --git a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
index fe08b21c663..9a823e57144 100644
--- a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
+++ b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
@@ -232,6 +232,7 @@
+
diff --git a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml
index 5adce1647d0..418458a4154 100644
--- a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml
+++ b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml
@@ -285,6 +285,8 @@
+
+
@@ -325,6 +327,10 @@
+
+ ${local.ivarator.cache.dir.config}
+
+
${ivarator.cache.dir.config}