do not default to keyword

Defaulting to keyword is causing problems in AOSS. The desire is to not add field to mapping when not present in LDD but to continue processing. This makes it non searchable until it is added and reindexed.
NASA-PDS · Nov 27, 2024 · 3b86ee9 · 3b86ee9
1 parent f78b2f1
commit 3b86ee9
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 14 deletions.
diff --git a/src/main/java/gov/nasa/pds/registry/common/dd/LddEsJsonWriter.java b/src/main/java/gov/nasa/pds/registry/common/dd/LddEsJsonWriter.java
@@ -7,6 +7,7 @@
 import org.apache.logging.log4j.Logger;
 
 import gov.nasa.pds.registry.common.dd.parser.DDAttribute;
+import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException;
 
 
 /**
@@ -125,18 +126,19 @@ private void writeRecord(String classNs, String className, DDAttribute dda) thro
         ddRec.attrName = dda.attrName;
 
         ddRec.dataType = dda.dataType;
-        ddRec.esDataType = dtMap.getEsDataType(dda.dataType);
-
-        ddRec.description = dda.description;
-
-        // Write
-        writer.write(ddRec.esFieldNameFromComponents(), ddRec);
+        try {
+          ddRec.esDataType = dtMap.getEsDataType(dda.dataType);
+          ddRec.description = dda.description;
+          writer.write(ddRec.esFieldNameFromComponents(), ddRec);
 
-        // Fix wrong attribute namespace
-        if(!classNs.equals(dda.attrNs))
-        {
+          // Fix wrong attribute namespace
+          if(!classNs.equals(dda.attrNs))
+          {
             ddRec.attrNs = classNs;
             writer.write(ddRec.esFieldNameFromComponents(), ddRec);
+          }
+        } catch (DataTypeNotFoundException e) {
+          log.error("The field '" + dda.attrName + "' will not be searchable because LDD does not contain a type for it (see harvest#204).");
         }
     }
 

diff --git a/src/main/java/gov/nasa/pds/registry/common/dd/Pds2EsDataTypeMap.java b/src/main/java/gov/nasa/pds/registry/common/dd/Pds2EsDataTypeMap.java
@@ -8,7 +8,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-
+import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException;
 import gov.nasa.pds.registry.common.util.CloseUtils;
 
 
@@ -45,7 +45,7 @@ public Pds2EsDataTypeMap()
      * @param pdsType PDS LDD data type
      * @return Elasticsearch data type
      */
-    public String getEsDataType(String pdsType)
+    public String getEsDataType(String pdsType) throws DataTypeNotFoundException
     {
         String esType = map.get(pdsType);
         if(esType != null) return esType;
@@ -64,7 +64,7 @@ public String getEsDataType(String pdsType)
      * @param pdsType PDS data type, e.g., "UTF8_Text_Preserved"
      * @return Elasticsearch data type, e.g., "text".
      */
-    private String guessEsDataType(String pdsType)
+    private String guessEsDataType(String pdsType) throws DataTypeNotFoundException
     {
         pdsType = pdsType.toLowerCase();
         if(pdsType.contains("_real")) return "double";
@@ -74,7 +74,7 @@ private String guessEsDataType(String pdsType)
         if(pdsType.contains("_date")) return "date";
         if(pdsType.contains("_boolean")) return "boolean";        
 
-        return "keyword";
+        throw new DataTypeNotFoundException("Could not guess type and not more defaulting to keyword (harvest#204)");
     }
 
 
@@ -100,7 +100,8 @@ public void load(File file) throws Exception
         }
         catch(Exception ex)
         {
-            throw new Exception("Could not open data type configuration file '" + file.getAbsolutePath());
+          if (rd != null) rd.close();
+          throw new Exception("Could not open data type configuration file '" + file.getAbsolutePath());
         }
 
         try