From 9b7efb3cc8d7ee3c61acda1a53732b7dec9798ca Mon Sep 17 00:00:00 2001 From: edunn Date: Thu, 18 Jul 2024 11:21:08 -0700 Subject: [PATCH 01/10] implement ProductClasses.java --- .../model/identifiers/PdsProductClasses.java | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 service/src/main/java/gov/nasa/pds/api/registry/model/identifiers/PdsProductClasses.java diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/identifiers/PdsProductClasses.java b/service/src/main/java/gov/nasa/pds/api/registry/model/identifiers/PdsProductClasses.java new file mode 100644 index 00000000..b33fed06 --- /dev/null +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/identifiers/PdsProductClasses.java @@ -0,0 +1,72 @@ +package gov.nasa.pds.api.registry.model.identifiers; + +/** + * An enumeration of the valid product_class values from the PDS4 Data Dictionary + * ... + */ +public enum PdsProductClasses { + Product_AIP("Product_AIP"), + Product_Ancillary("Product_Ancillary"), + Product_Attribute_Definition("Product_Attribute_Definition"), + Product_Browse("Product_Browse"), + Product_Bundle("Product_Bundle"), + Product_Class_Definition("Product_Class_Definition"), + Product_Collection("Product_Collection"), + Product_Context("Product_Context"), + Product_DIP("Product_DIP"), + Product_DIP_Deep_Archive("Product_DIP_Deep_Archive"), + Product_Data_Set_PDS3("Product_Data_Set_PDS3"), + Product_Document("Product_Document"), + Product_External("Product_External"), + Product_File_Repository("Product_File_Repository"), + Product_File_Text("Product_File_Text"), + Product_Instrument_Host_PDS3("Product_Instrument_Host_PDS3"), + Product_Instrument_PDS3("Product_Instrument_PDS3"), + Product_Metadata_Supplemental("Product_Metadata_Supplemental"), + Product_Mission_PDS3("Product_Mission_PDS3"), + Product_Native("Product_Native"), + Product_Observational("Product_Observational"), + Product_Proxy_PDS3("Product_Proxy_PDS3"), + Product_SIP("Product_SIP"), + Product_SIP_Deep_Archive("Product_SIP_Deep_Archive"), + Product_SPICE_Kernel("Product_SPICE_Kernel"), + Product_Service("Product_Service"), + Product_Software("Product_Software"), + Product_Subscription_PDS3("Product_Subscription_PDS3"), + Product_Target_PDS3("Product_Target_PDS3"), + Product_Thumbnail("Product_Thumbnail"), + Product_Update("Product_Update"), + Product_Volume_PDS3("Product_Volume_PDS3"), + Product_Volume_Set_PDS3("Product_Volume_Set_PDS3"), + Product_XML_Schema("Product_XML_Schema"), + Product_Zipped("Product_Zipped"); + + private final String value; + + PdsProductClasses(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + /** + * @return The database property/field these string values appear in. Provided for convenience. + */ + public static String getPropertyName() { + return "product_class"; + } + + public Boolean isBundle() { + return this == Product_Bundle; + } + + public Boolean isCollection() { + return this == Product_Collection; + } + + public Boolean isBasicProduct() { + return !(isBundle() || isCollection()); + } +} From dd809bbb152c11515630a131fc7acad27202e3cc Mon Sep 17 00:00:00 2001 From: edunn Date: Thu, 18 Jul 2024 11:55:47 -0700 Subject: [PATCH 02/10] implement MiscellaneousBadRequestException --- .../MiscellaneousBadRequestException.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/MiscellaneousBadRequestException.java diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/MiscellaneousBadRequestException.java b/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/MiscellaneousBadRequestException.java new file mode 100644 index 00000000..b5904293 --- /dev/null +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/MiscellaneousBadRequestException.java @@ -0,0 +1,20 @@ +package gov.nasa.pds.api.registry.model.exceptions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Serial; + +/** + * Use as a catch-all for one-off errors where the request is bad and specific handling is not required + */ +public class MiscellaneousBadRequestException extends RegistryApiException { + private static final Logger log = LoggerFactory.getLogger(MiscellaneousBadRequestException.class); + @Serial + private static final long serialVersionUID = 2026697251322082840L; + + public MiscellaneousBadRequestException(String msg) { + super("MiscellaneousBadRequestException: " + msg); + } + +} From 5d0fa0f42f3ccc473b974a0abf399adf3ed9f665 Mon Sep 17 00:00:00 2001 From: edunn Date: Thu, 18 Jul 2024 12:34:31 -0700 Subject: [PATCH 03/10] implement members and members/members products subroutes --- .../controllers/ProductsController.java | 100 +++++++++++++++++- .../search/RegistrySearchRequestBuilder.java | 8 ++ 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 8980e1ce..f1703104 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -5,12 +5,16 @@ import java.util.List; import java.util.Map; import java.util.HashMap; + +import gov.nasa.pds.api.registry.model.exceptions.*; +import gov.nasa.pds.api.registry.model.identifiers.PdsProductClasses; import jakarta.servlet.http.HttpServletRequest; import com.fasterxml.jackson.databind.ObjectMapper; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.client.opensearch._types.OpenSearchException; import org.opensearch.client.opensearch.core.SearchRequest; import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.http.HttpException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -22,10 +26,6 @@ import gov.nasa.pds.api.base.ProductsApi; import gov.nasa.pds.api.registry.ConnectionContext; import gov.nasa.pds.api.registry.model.ErrorMessageFactory; -import gov.nasa.pds.api.registry.model.exceptions.AcceptFormatNotSupportedException; -import gov.nasa.pds.api.registry.model.exceptions.SortSearchAfterMismatchException; -import gov.nasa.pds.api.registry.model.exceptions.NotFoundException; -import gov.nasa.pds.api.registry.model.exceptions.UnhandledException; import gov.nasa.pds.api.registry.model.api_responses.PdsProductBusinessObject; import gov.nasa.pds.api.registry.model.api_responses.ProductBusinessLogic; import gov.nasa.pds.api.registry.model.api_responses.ProductBusinessLogicImpl; @@ -331,6 +331,98 @@ private RawMultipleProductResponse getAllLidVid(PdsProductIdentifier identifier, } + private PdsProductClasses resolveProductClass(PdsProductIdentifier identifier) + throws OpenSearchException, IOException, NotFoundException{ + String productClassKey = "product_class"; + SearchRequest searchRequest = new RegistrySearchRequestBuilder(this.connectionContext) + .matchLid(identifier) + .fieldsFromStrings(List.of(productClassKey)) + .onlyLatest() + .build(); + + SearchResponse searchResponse = this.openSearchClient.search(searchRequest, HashMap.class); + + if (searchResponse.hits().total().value() == 0) { + throw new NotFoundException("No product found with identifier " + identifier.toString()); + } + + String productClassStr = searchResponse.hits().hits().get(0).source().get(productClassKey).toString(); + return PdsProductClasses.valueOf(productClassStr); + } + + @Override + public ResponseEntity productMembers( + String identifier, List fields, Integer limit, List sort, List searchAfter) + throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, MiscellaneousBadRequestException, + AcceptFormatNotSupportedException{ + + try{ + PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); + PdsProductClasses productClass = resolveProductClass(pdsIdentifier); + + RegistrySearchRequestBuilder searchRequestBuilder = new RegistrySearchRequestBuilder(this.connectionContext); + + if (productClass.isBundle()) { + searchRequestBuilder.matchMembersOfBundle(pdsIdentifier); + } else if (productClass.isCollection()) { + searchRequestBuilder.matchMembersOfCollection(pdsIdentifier); + } else { + throw new MiscellaneousBadRequestException("productMembers endpoint is only valid for products with Product_Class '" + + PdsProductClasses.Product_Bundle + "' or '" + PdsProductClasses.Product_Collection + + "' (got '" + productClass + "')"); + } + + SearchRequest searchRequest = searchRequestBuilder + .fieldsFromStrings(fields) + .paginate(limit, sort, searchAfter) + .build(); + + SearchResponse searchResponse = + this.openSearchClient.search(searchRequest, HashMap.class); + + RawMultipleProductResponse products = new RawMultipleProductResponse(searchResponse); + return formatMultipleProducts(products, fields); + + } catch (IOException | OpenSearchException e) { + throw new UnhandledException(e); + } + } + + @Override + public ResponseEntity productMembersMembers( + String identifier, List fields, Integer limit, List sort, List searchAfter) + throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, MiscellaneousBadRequestException, + AcceptFormatNotSupportedException{ + + try{ + PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); + PdsProductClasses productClass = resolveProductClass(pdsIdentifier); + + RegistrySearchRequestBuilder searchRequestBuilder = new RegistrySearchRequestBuilder(this.connectionContext); + + if (productClass.isBundle()) { + searchRequestBuilder.matchMembersOfBundle(pdsIdentifier); + } else { + throw new MiscellaneousBadRequestException("productMembers endpoint is only valid for products with Product_Class '" + + PdsProductClasses.Product_Bundle + "' (got '" + productClass + "')"); + } + + SearchRequest searchRequest = searchRequestBuilder + .fieldsFromStrings(fields) + .paginate(limit, sort, searchAfter) + .build(); + + SearchResponse searchResponse = + this.openSearchClient.search(searchRequest, HashMap.class); + + RawMultipleProductResponse products = new RawMultipleProductResponse(searchResponse); + + return formatMultipleProducts(products, fields); + + } catch (IOException | OpenSearchException e) { + throw new UnhandledException(e); + } + } } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index 812eb227..36e94b7e 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -142,6 +142,14 @@ public RegistrySearchRequestBuilder matchLid(PdsProductIdentifier identifier) { return this.matchField("lid", identifier); } + public RegistrySearchRequestBuilder matchMembersOfBundle(PdsProductIdentifier identifier) { + return this.matchField("ops:Provenance/ops:parent_bundle_identifier", identifier); + } + + public RegistrySearchRequestBuilder matchMembersOfCollection(PdsProductIdentifier identifier) { + return this.matchField("ops:Provenance/ops:parent_collection_identifier", identifier); + } + public RegistrySearchRequestBuilder paginate(Integer pageSize, List sortFieldNames, List searchAfterFieldValues) throws SortSearchAfterMismatchException { if ((sortFieldNames != null) && (!sortFieldNames.isEmpty())) { From d93984dced4e13372ef5ab2d9b490d215a204606 Mon Sep 17 00:00:00 2001 From: edunn Date: Thu, 18 Jul 2024 17:09:02 -0700 Subject: [PATCH 04/10] implement product_class filter helper methods --- .../search/RegistrySearchRequestBuilder.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index 36e94b7e..0f9883f7 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.List; +import gov.nasa.pds.api.registry.model.identifiers.PdsProductClasses; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CodePointCharStream; @@ -300,5 +301,34 @@ public RegistrySearchRequestBuilder onlyLatest() { return this; } + /** + * Limit results to bundle products + */ + public RegistrySearchRequestBuilder onlyBundles() { + return this.matchField(PdsProductClasses.getPropertyName(), PdsProductClasses.Product_Bundle.toString()); + } + + + /** + * Limit results to collection products + */public RegistrySearchRequestBuilder onlyCollections() { + return this.matchField(PdsProductClasses.getPropertyName(), PdsProductClasses.Product_Collection.toString()); + } + + + /** + * Limit results to basic (non-aggregate) products, i.e. exclude bundles/collections + */ + public RegistrySearchRequestBuilder onlyBasicProducts() { + List excludeValues = Arrays.stream(PdsProductClasses.values()) + .filter(cls -> !cls.isBasicProduct()) + .map(value -> new FieldValue.Builder().stringValue(value.toString()).build()).toList(); + TermsQueryField termsQueryField = new TermsQueryField.Builder().value(excludeValues).build(); + TermsQuery query = new TermsQuery.Builder().field(PdsProductClasses.getPropertyName()).terms(termsQueryField).build(); + + this.queryBuilder.mustNot(query.toQuery()); + return this; + } + } From 37738895c7a6056dcfadaa16ca2cd4dd5abd9c88 Mon Sep 17 00:00:00 2001 From: edunn Date: Thu, 18 Jul 2024 13:27:52 -0700 Subject: [PATCH 05/10] [partial] re-implement ancestry endpoints and related helper functions - implement member-of and member-of/member-of endpoints - implement RegistrySearchRequestBuilder methods matchFieldAnyOf() and matchFieldAnyOfIdentifiers() --- .../controllers/ProductsController.java | 82 ++++++++++++++++++- .../search/RegistrySearchRequestBuilder.java | 25 ++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index f1703104..f8cc6480 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -2,11 +2,13 @@ import java.lang.reflect.InvocationTargetException; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.HashMap; import gov.nasa.pds.api.registry.model.exceptions.*; +import gov.nasa.pds.api.registry.model.identifiers.PdsLidVid; import gov.nasa.pds.api.registry.model.identifiers.PdsProductClasses; import jakarta.servlet.http.HttpServletRequest; import com.fasterxml.jackson.databind.ObjectMapper; @@ -14,7 +16,6 @@ import org.opensearch.client.opensearch._types.OpenSearchException; import org.opensearch.client.opensearch.core.SearchRequest; import org.opensearch.client.opensearch.core.SearchResponse; -import org.opensearch.http.HttpException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -425,4 +426,83 @@ public ResponseEntity productMembersMembers( } } + /** + * Given a PdsProductIdentifier and the name of a document field which is expected to contain an array of LIDVID + * strings, return the chained contents of that field from all documents matching the identifier (multiple docs are + * possible if the identifier is a LID). + * @param identifier the LID/LIDVID for which to retrieve documents + * @param fieldName the name of the document _source property/field from which to extract results + * @return a deduplicated list of the aggregated property/field contents, converted to PdsProductLidvids + */ + private List resolveLidVidsFromProductField(PdsProductIdentifier identifier, String fieldName) + throws OpenSearchException, IOException, NotFoundException, UnhandledException { + + RegistrySearchRequestBuilder searchRequestBuilder = new RegistrySearchRequestBuilder(this.connectionContext); + + if (identifier.isLid()) { + searchRequestBuilder.matchLid(identifier); + } else if (identifier.isLidvid()) { + searchRequestBuilder.matchLidvid(identifier); + } else { + throw new UnhandledException("PdsProductIdentifier identifier is neither LID nor LIDVID. This should never occur"); + } + + SearchRequest searchRequest = searchRequestBuilder + .matchLid(identifier) + .fieldsFromStrings(List.of(fieldName)) + .onlyLatest() + .build(); + + SearchResponse searchResponse = this.openSearchClient.search(searchRequest, HashMap.class); + + if (searchResponse.hits().total().value() == 0) { + throw new NotFoundException("No product found with identifier " + identifier); + } + +// TODO: Remove these debug lines, and test this function + List> fieldListsFromDocs = searchResponse.hits().hits().stream().map(hit -> (List) hit.source().get(fieldName)).toList(); + List flatList = searchResponse.hits().hits().stream().map(hit -> (List) hit.source().get(fieldName)).flatMap(Collection::stream).map(PdsLidVid::fromString).toList();; + + return searchResponse.hits().hits().stream().map(hit -> (List) hit.source().get(fieldName)).flatMap(Collection::stream).map(PdsLidVid::fromString).toList(); + } + + + @Override + public ResponseEntity productMemberOf( + String identifier, List fields, Integer limit, List sort, List searchAfter) + throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, MiscellaneousBadRequestException, + AcceptFormatNotSupportedException{ + + try{ + PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); + PdsProductClasses productClass = resolveProductClass(pdsIdentifier); + + List parentIds; + if (productClass.isCollection()) { + parentIds = resolveLidVidsFromProductField(pdsIdentifier, "ops:Provenance/ops:parent_bundle_identifier"); + } else if (productClass.isBasicProduct()) { + parentIds = resolveLidVidsFromProductField(pdsIdentifier, "ops:Provenance/ops:parent_collection_identifier"); + } else { + throw new MiscellaneousBadRequestException("productMembersOf endpoint is not valid for products with Product_Class '" + + PdsProductClasses.Product_Bundle + "' (got '" + productClass + "')"); + } + + SearchRequest searchRequest = new RegistrySearchRequestBuilder(this.connectionContext) + .matchFieldAnyOfIdentifiers("_id", parentIds) + .fieldsFromStrings(fields) + .paginate(limit, sort, searchAfter) + .build(); + + SearchResponse searchResponse = + this.openSearchClient.search(searchRequest, HashMap.class); + + RawMultipleProductResponse products = new RawMultipleProductResponse(searchResponse); + + return formatMultipleProducts(products, fields); + + } catch (IOException | OpenSearchException e) { + throw new UnhandledException(e); + } + } + } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index 0f9883f7..cfd7b9e5 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -135,6 +135,31 @@ public RegistrySearchRequestBuilder matchField(String fieldName, PdsProductIdent return this.matchField(fieldName, identifier.toString()); } + + /** + * Add a constraint that a given field name must match at least one of the given field values + * @param fieldName the name of the field in OpenSearch format + * @param values the values, one of which must be present in the given field + */ + public RegistrySearchRequestBuilder matchFieldAnyOf(String fieldName, List values) { + List fieldValues = values.stream().map(value -> new FieldValue.Builder().stringValue(value).build()).toList(); + TermsQueryField termsQueryField = new TermsQueryField.Builder().value(fieldValues).build(); + TermsQuery query = new TermsQuery.Builder().field(fieldName).terms(termsQueryField).build(); + + this.queryBuilder.must(query.toQuery()); + + return this; + } + + /** + * Add a constraint that a given field name must match at least one of the given field values + * @param fieldName the name of the field in OpenSearch format + * @param identifiers the PDS identifiers, one of whose string representation must be present in the given field + */ + public RegistrySearchRequestBuilder matchFieldAnyOfIdentifiers(String fieldName, List identifiers) { + return this.matchFieldAnyOf(fieldName, identifiers.stream().map(PdsProductIdentifier::toString).toList()); + } + public RegistrySearchRequestBuilder matchLidvid(PdsProductIdentifier identifier) { return this.matchField("_id", identifier); } From bac4a52e1edbccacadf900420b3a88a1dc0762ad Mon Sep 17 00:00:00 2001 From: edunn Date: Thu, 18 Jul 2024 15:39:31 -0700 Subject: [PATCH 06/10] fix bug --- .../pds/api/registry/search/RegistrySearchRequestBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index cfd7b9e5..5fb256b1 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -165,7 +165,7 @@ public RegistrySearchRequestBuilder matchLidvid(PdsProductIdentifier identifier) } public RegistrySearchRequestBuilder matchLid(PdsProductIdentifier identifier) { - return this.matchField("lid", identifier); + return this.matchField("lid", identifier.getLid()); } public RegistrySearchRequestBuilder matchMembersOfBundle(PdsProductIdentifier identifier) { From 7c9af2aadfb278e2e2d8c5b43749427bd8beff4f Mon Sep 17 00:00:00 2001 From: edunn Date: Thu, 18 Jul 2024 16:05:02 -0700 Subject: [PATCH 07/10] implement ProductsController.resolveExtantLidvids() --- .../controllers/ProductsController.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index f8cc6480..703e3fac 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -8,6 +8,7 @@ import java.util.HashMap; import gov.nasa.pds.api.registry.model.exceptions.*; +import gov.nasa.pds.api.registry.model.identifiers.PdsLid; import gov.nasa.pds.api.registry.model.identifiers.PdsLidVid; import gov.nasa.pds.api.registry.model.identifiers.PdsProductClasses; import jakarta.servlet.http.HttpServletRequest; @@ -351,6 +352,26 @@ private PdsProductClasses resolveProductClass(PdsProductIdentifier identifier) return PdsProductClasses.valueOf(productClassStr); } + + private List resolveExtantLidvids(PdsLid lid) + throws OpenSearchException, IOException, NotFoundException{ + + String lidvidKey = "_id"; + + SearchRequest searchRequest = new RegistrySearchRequestBuilder(this.connectionContext) + .matchLid(lid) + .fieldsFromStrings(List.of(lidvidKey)) + .build(); + + SearchResponse searchResponse = this.openSearchClient.search(searchRequest, HashMap.class); + + if (searchResponse.hits().total().value() == 0) { + throw new NotFoundException("No lidvids found with lid " + lid.toString()); + } + + return searchResponse.hits().hits().stream().map(hit -> hit.source().get(lidvidKey).toString()).map(PdsLidVid::fromString).toList(); + } + @Override public ResponseEntity productMembers( String identifier, List fields, Integer limit, List sort, List searchAfter) From 973ad53601de760d15ca7ebbc8eaea34efa40820 Mon Sep 17 00:00:00 2001 From: edunn Date: Thu, 18 Jul 2024 16:13:47 -0700 Subject: [PATCH 08/10] implement ProductsController.resolveLatestLidvid() --- .../controllers/ProductsController.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 703e3fac..b40adc73 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -353,6 +353,27 @@ private PdsProductClasses resolveProductClass(PdsProductIdentifier identifier) } + private PdsLidVid resolveLatestLidvid(PdsProductIdentifier identifier) + throws OpenSearchException, IOException, NotFoundException { + + SearchRequest searchRequest = new RegistrySearchRequestBuilder(this.connectionContext) + .matchLid(identifier.getLid()) + .fieldsFromStrings(List.of()) + .onlyLatest() + .build(); + + SearchResponse searchResponse = this.openSearchClient.search(searchRequest, HashMap.class); + + if (searchResponse.hits().total().value() == 0) { + throw new NotFoundException("No lidvids found with lid " + identifier.getLid().toString()); + } + + // TODO: Determine how to handle multiple hits due to sweepers lag + + return PdsLidVid.fromString(searchResponse.hits().hits().get(0).id()); + } + + private List resolveExtantLidvids(PdsLid lid) throws OpenSearchException, IOException, NotFoundException{ @@ -382,6 +403,8 @@ public ResponseEntity productMembers( PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); PdsProductClasses productClass = resolveProductClass(pdsIdentifier); + + RegistrySearchRequestBuilder searchRequestBuilder = new RegistrySearchRequestBuilder(this.connectionContext); if (productClass.isBundle()) { From c00412be29fc9ca61dfe3111a65ece9d79d20b6a Mon Sep 17 00:00:00 2001 From: edunn Date: Thu, 18 Jul 2024 16:39:28 -0700 Subject: [PATCH 09/10] [partial] implement ancestry-related endpoints and associated helper functions - apply default behaviour of resolving LID to latest-LIDVID and only allowing non-superseded results in output data - implement ProductsController.resolveIdentifierToLidvid() - implement member-of and member-of/member-of endpoints - important explanatory comment - implement members and members/members products subroutes --- .../controllers/ProductsController.java | 83 +++++++++++++++---- .../search/RegistrySearchRequestBuilder.java | 10 ++- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index b40adc73..8ce58a71 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.HashMap; +import java.util.Objects; import gov.nasa.pds.api.registry.model.exceptions.*; import gov.nasa.pds.api.registry.model.identifiers.PdsLid; @@ -335,10 +336,9 @@ private RawMultipleProductResponse getAllLidVid(PdsProductIdentifier identifier, private PdsProductClasses resolveProductClass(PdsProductIdentifier identifier) throws OpenSearchException, IOException, NotFoundException{ - String productClassKey = "product_class"; SearchRequest searchRequest = new RegistrySearchRequestBuilder(this.connectionContext) .matchLid(identifier) - .fieldsFromStrings(List.of(productClassKey)) + .fieldsFromStrings(List.of(PdsProductClasses.getPropertyName())) .onlyLatest() .build(); @@ -348,7 +348,7 @@ private PdsProductClasses resolveProductClass(PdsProductIdentifier identifier) throw new NotFoundException("No product found with identifier " + identifier.toString()); } - String productClassStr = searchResponse.hits().hits().get(0).source().get(productClassKey).toString(); + String productClassStr = searchResponse.hits().hits().get(0).source().get(PdsProductClasses.getPropertyName()).toString(); return PdsProductClasses.valueOf(productClassStr); } @@ -393,6 +393,16 @@ private List resolveExtantLidvids(PdsLid lid) return searchResponse.hits().hits().stream().map(hit -> hit.source().get(lidvidKey).toString()).map(PdsLidVid::fromString).toList(); } + /** + * Resolve a PdsProductIdentifier to a PdsLidVid according to the common rules of the API. + * The rules are currently trivial, but may incorporate additional behaviour later + * @param identifier a LID or LIDVID + * @return a LIDVID + */ + private PdsLidVid resolveIdentifierToLidvid(PdsProductIdentifier identifier) throws NotFoundException, IOException { + return identifier.isLidvid() ? (PdsLidVid) identifier : resolveLatestLidvid(identifier); + } + @Override public ResponseEntity productMembers( String identifier, List fields, Integer limit, List sort, List searchAfter) @@ -402,15 +412,16 @@ public ResponseEntity productMembers( try{ PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); PdsProductClasses productClass = resolveProductClass(pdsIdentifier); - - + PdsLidVid lidvid = resolveIdentifierToLidvid(pdsIdentifier); RegistrySearchRequestBuilder searchRequestBuilder = new RegistrySearchRequestBuilder(this.connectionContext); if (productClass.isBundle()) { - searchRequestBuilder.matchMembersOfBundle(pdsIdentifier); + searchRequestBuilder.matchMembersOfBundle(lidvid); + searchRequestBuilder.onlyCollections(); } else if (productClass.isCollection()) { - searchRequestBuilder.matchMembersOfCollection(pdsIdentifier); + searchRequestBuilder.matchMembersOfCollection(lidvid); + searchRequestBuilder.onlyBasicProducts(); } else { throw new MiscellaneousBadRequestException("productMembers endpoint is only valid for products with Product_Class '" + PdsProductClasses.Product_Bundle + "' or '" + PdsProductClasses.Product_Collection + @@ -420,6 +431,7 @@ public ResponseEntity productMembers( SearchRequest searchRequest = searchRequestBuilder .fieldsFromStrings(fields) .paginate(limit, sort, searchAfter) + .onlyLatest() .build(); SearchResponse searchResponse = @@ -443,11 +455,13 @@ public ResponseEntity productMembersMembers( try{ PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); PdsProductClasses productClass = resolveProductClass(pdsIdentifier); + PdsLidVid lidvid = resolveIdentifierToLidvid(pdsIdentifier); RegistrySearchRequestBuilder searchRequestBuilder = new RegistrySearchRequestBuilder(this.connectionContext); if (productClass.isBundle()) { - searchRequestBuilder.matchMembersOfBundle(pdsIdentifier); + searchRequestBuilder.matchMembersOfBundle(lidvid); + searchRequestBuilder.onlyBasicProducts(); } else { throw new MiscellaneousBadRequestException("productMembers endpoint is only valid for products with Product_Class '" + PdsProductClasses.Product_Bundle + "' (got '" + productClass + "')"); @@ -456,6 +470,7 @@ public ResponseEntity productMembersMembers( SearchRequest searchRequest = searchRequestBuilder .fieldsFromStrings(fields) .paginate(limit, sort, searchAfter) + .onlyLatest() .build(); SearchResponse searchResponse = @@ -494,7 +509,6 @@ private List resolveLidVidsFromProductField(PdsProductIdentifier iden SearchRequest searchRequest = searchRequestBuilder .matchLid(identifier) .fieldsFromStrings(List.of(fieldName)) - .onlyLatest() .build(); SearchResponse searchResponse = this.openSearchClient.search(searchRequest, HashMap.class); @@ -503,11 +517,7 @@ private List resolveLidVidsFromProductField(PdsProductIdentifier iden throw new NotFoundException("No product found with identifier " + identifier); } -// TODO: Remove these debug lines, and test this function - List> fieldListsFromDocs = searchResponse.hits().hits().stream().map(hit -> (List) hit.source().get(fieldName)).toList(); - List flatList = searchResponse.hits().hits().stream().map(hit -> (List) hit.source().get(fieldName)).flatMap(Collection::stream).map(PdsLidVid::fromString).toList();; - - return searchResponse.hits().hits().stream().map(hit -> (List) hit.source().get(fieldName)).flatMap(Collection::stream).map(PdsLidVid::fromString).toList(); + return searchResponse.hits().hits().stream().map(hit -> (List) hit.source().get(fieldName)).filter(Objects::nonNull).flatMap(Collection::stream).map(PdsLidVid::fromString).toList(); } @@ -520,12 +530,13 @@ public ResponseEntity productMemberOf( try{ PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); PdsProductClasses productClass = resolveProductClass(pdsIdentifier); + PdsLidVid lidvid = resolveIdentifierToLidvid(pdsIdentifier); List parentIds; if (productClass.isCollection()) { - parentIds = resolveLidVidsFromProductField(pdsIdentifier, "ops:Provenance/ops:parent_bundle_identifier"); + parentIds = resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:parent_bundle_identifier"); } else if (productClass.isBasicProduct()) { - parentIds = resolveLidVidsFromProductField(pdsIdentifier, "ops:Provenance/ops:parent_collection_identifier"); + parentIds = resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:parent_collection_identifier"); } else { throw new MiscellaneousBadRequestException("productMembersOf endpoint is not valid for products with Product_Class '" + PdsProductClasses.Product_Bundle + "' (got '" + productClass + "')"); @@ -535,6 +546,46 @@ public ResponseEntity productMemberOf( .matchFieldAnyOfIdentifiers("_id", parentIds) .fieldsFromStrings(fields) .paginate(limit, sort, searchAfter) + .onlyLatest() + .build(); + + SearchResponse searchResponse = + this.openSearchClient.search(searchRequest, HashMap.class); + + RawMultipleProductResponse products = new RawMultipleProductResponse(searchResponse); + + return formatMultipleProducts(products, fields); + + } catch (IOException | OpenSearchException e) { + throw new UnhandledException(e); + } + } + + @Override + public ResponseEntity productMemberOfOf( + String identifier, List fields, Integer limit, List sort, List searchAfter) + throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, MiscellaneousBadRequestException, + AcceptFormatNotSupportedException{ + + try{ + PdsProductIdentifier pdsIdentifier = PdsProductIdentifier.fromString(identifier); + PdsProductClasses productClass = resolveProductClass(pdsIdentifier); + PdsLidVid lidvid = resolveIdentifierToLidvid(pdsIdentifier); + + List parentIds; + if (productClass.isBasicProduct()) { + parentIds = resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:parent_bundle_identifier"); + } else { +// TODO: replace with enumeration of acceptable values later + throw new MiscellaneousBadRequestException("productMembersOf endpoint is not valid for products with Product_Class '" + + PdsProductClasses.Product_Bundle + "' or '" + PdsProductClasses.Product_Collection + "' (got '" + productClass + "')"); + } + + SearchRequest searchRequest = new RegistrySearchRequestBuilder(this.connectionContext) + .matchFieldAnyOfIdentifiers("_id", parentIds) + .fieldsFromStrings(fields) + .paginate(limit, sort, searchAfter) + .onlyLatest() .build(); SearchResponse searchResponse = diff --git a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java index 5fb256b1..c7c5cca6 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/search/RegistrySearchRequestBuilder.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.List; +import gov.nasa.pds.api.registry.model.identifiers.PdsLidVid; import gov.nasa.pds.api.registry.model.identifiers.PdsProductClasses; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -168,11 +169,11 @@ public RegistrySearchRequestBuilder matchLid(PdsProductIdentifier identifier) { return this.matchField("lid", identifier.getLid()); } - public RegistrySearchRequestBuilder matchMembersOfBundle(PdsProductIdentifier identifier) { + public RegistrySearchRequestBuilder matchMembersOfBundle(PdsLidVid identifier) { return this.matchField("ops:Provenance/ops:parent_bundle_identifier", identifier); } - public RegistrySearchRequestBuilder matchMembersOfCollection(PdsProductIdentifier identifier) { + public RegistrySearchRequestBuilder matchMembersOfCollection(PdsLidVid identifier) { return this.matchField("ops:Provenance/ops:parent_collection_identifier", identifier); } @@ -315,6 +316,11 @@ public RegistrySearchRequestBuilder addKeywordsParam(List keywords) { return this; } + /** + * Limit results to the latest version of each LID in the result-set. + * N.B. this does *not* mean the latest version which satisfies other constraints, so application of this constraint + * can result in no hits being returned despite valid results existing. + */ public RegistrySearchRequestBuilder onlyLatest() { ExistsQuery supersededByExists = new ExistsQuery.Builder() From 0547f293ef310b0004a6b044d680e3b7b9111d80 Mon Sep 17 00:00:00 2001 From: edunn Date: Fri, 19 Jul 2024 12:52:18 -0700 Subject: [PATCH 10/10] implement BadRequestException handling --- .../controllers/ProductsController.java | 16 +++++++-------- ...stryApiResponseEntityExceptionHandler.java | 15 +++++++------- .../model/exceptions/BadRequestException.java | 17 ++++++++++++++++ .../MiscellaneousBadRequestException.java | 20 ------------------- 4 files changed, 33 insertions(+), 35 deletions(-) create mode 100644 service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/BadRequestException.java delete mode 100644 service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/MiscellaneousBadRequestException.java diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java index 8ce58a71..0c31e496 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/ProductsController.java @@ -406,7 +406,7 @@ private PdsLidVid resolveIdentifierToLidvid(PdsProductIdentifier identifier) thr @Override public ResponseEntity productMembers( String identifier, List fields, Integer limit, List sort, List searchAfter) - throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, MiscellaneousBadRequestException, + throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, BadRequestException, AcceptFormatNotSupportedException{ try{ @@ -423,7 +423,7 @@ public ResponseEntity productMembers( searchRequestBuilder.matchMembersOfCollection(lidvid); searchRequestBuilder.onlyBasicProducts(); } else { - throw new MiscellaneousBadRequestException("productMembers endpoint is only valid for products with Product_Class '" + + throw new BadRequestException("productMembers endpoint is only valid for products with Product_Class '" + PdsProductClasses.Product_Bundle + "' or '" + PdsProductClasses.Product_Collection + "' (got '" + productClass + "')"); } @@ -449,7 +449,7 @@ public ResponseEntity productMembers( @Override public ResponseEntity productMembersMembers( String identifier, List fields, Integer limit, List sort, List searchAfter) - throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, MiscellaneousBadRequestException, + throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, BadRequestException, AcceptFormatNotSupportedException{ try{ @@ -463,7 +463,7 @@ public ResponseEntity productMembersMembers( searchRequestBuilder.matchMembersOfBundle(lidvid); searchRequestBuilder.onlyBasicProducts(); } else { - throw new MiscellaneousBadRequestException("productMembers endpoint is only valid for products with Product_Class '" + + throw new BadRequestException("productMembers endpoint is only valid for products with Product_Class '" + PdsProductClasses.Product_Bundle + "' (got '" + productClass + "')"); } @@ -524,7 +524,7 @@ private List resolveLidVidsFromProductField(PdsProductIdentifier iden @Override public ResponseEntity productMemberOf( String identifier, List fields, Integer limit, List sort, List searchAfter) - throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, MiscellaneousBadRequestException, + throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, BadRequestException, AcceptFormatNotSupportedException{ try{ @@ -538,7 +538,7 @@ public ResponseEntity productMemberOf( } else if (productClass.isBasicProduct()) { parentIds = resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:parent_collection_identifier"); } else { - throw new MiscellaneousBadRequestException("productMembersOf endpoint is not valid for products with Product_Class '" + + throw new BadRequestException("productMembersOf endpoint is not valid for products with Product_Class '" + PdsProductClasses.Product_Bundle + "' (got '" + productClass + "')"); } @@ -564,7 +564,7 @@ public ResponseEntity productMemberOf( @Override public ResponseEntity productMemberOfOf( String identifier, List fields, Integer limit, List sort, List searchAfter) - throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, MiscellaneousBadRequestException, + throws NotFoundException, UnhandledException, SortSearchAfterMismatchException, BadRequestException, AcceptFormatNotSupportedException{ try{ @@ -577,7 +577,7 @@ public ResponseEntity productMemberOfOf( parentIds = resolveLidVidsFromProductField(lidvid, "ops:Provenance/ops:parent_bundle_identifier"); } else { // TODO: replace with enumeration of acceptable values later - throw new MiscellaneousBadRequestException("productMembersOf endpoint is not valid for products with Product_Class '" + + throw new BadRequestException("productMembersOf endpoint is not valid for products with Product_Class '" + PdsProductClasses.Product_Bundle + "' or '" + PdsProductClasses.Product_Collection + "' (got '" + productClass + "')"); } diff --git a/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java b/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java index 4caf77f4..2de03b14 100644 --- a/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java +++ b/service/src/main/java/gov/nasa/pds/api/registry/controllers/RegistryApiResponseEntityExceptionHandler.java @@ -2,6 +2,8 @@ import java.util.Set; + +import gov.nasa.pds.api.registry.model.exceptions.*; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -9,13 +11,6 @@ import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.context.request.WebRequest; import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; -import gov.nasa.pds.api.registry.model.exceptions.AcceptFormatNotSupportedException; -import gov.nasa.pds.api.registry.model.exceptions.SortSearchAfterMismatchException; -import gov.nasa.pds.api.registry.model.exceptions.NotFoundException; -import gov.nasa.pds.api.registry.model.exceptions.RegistryApiException; -import gov.nasa.pds.api.registry.model.exceptions.UnhandledException; -import gov.nasa.pds.api.registry.model.exceptions.UnparsableQParamException; - @ControllerAdvice @@ -49,6 +44,12 @@ protected ResponseEntity notFound(NotFoundException ex, WebRequest reque } + @ExceptionHandler(value = {BadRequestException.class}) + protected ResponseEntity badRequest(BadRequestException ex, WebRequest request) { + return genericExceptionHandler(ex, request, "", HttpStatus.BAD_REQUEST); + + } + @ExceptionHandler(value = {UnhandledException.class}) protected ResponseEntity unhandled(UnhandledException ex, WebRequest request) { return genericExceptionHandler(ex, request, "", HttpStatus.INTERNAL_SERVER_ERROR); diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/BadRequestException.java b/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/BadRequestException.java new file mode 100644 index 00000000..55345724 --- /dev/null +++ b/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/BadRequestException.java @@ -0,0 +1,17 @@ +package gov.nasa.pds.api.registry.model.exceptions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Serial; + +public class BadRequestException extends RegistryApiException { + private static final Logger log = LoggerFactory.getLogger(BadRequestException.class); + @Serial + private static final long serialVersionUID = 2026697251322082840L; + + public BadRequestException(String msg) { + super("BadRequestException: " + msg); + } + +} diff --git a/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/MiscellaneousBadRequestException.java b/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/MiscellaneousBadRequestException.java deleted file mode 100644 index b5904293..00000000 --- a/service/src/main/java/gov/nasa/pds/api/registry/model/exceptions/MiscellaneousBadRequestException.java +++ /dev/null @@ -1,20 +0,0 @@ -package gov.nasa.pds.api.registry.model.exceptions; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Serial; - -/** - * Use as a catch-all for one-off errors where the request is bad and specific handling is not required - */ -public class MiscellaneousBadRequestException extends RegistryApiException { - private static final Logger log = LoggerFactory.getLogger(MiscellaneousBadRequestException.class); - @Serial - private static final long serialVersionUID = 2026697251322082840L; - - public MiscellaneousBadRequestException(String msg) { - super("MiscellaneousBadRequestException: " + msg); - } - -}