Skip to content

Commit

Permalink
Merge pull request #187 from NASA-PDS/dsview43
Browse files Browse the repository at this point in the history 
Update handling of missing references
  • Loading branch information
@jordanpadams
jordanpadams authored Feb 6, 2025
2 parents bc214e1 + 2081447 commit 0541c1d
Show file tree
Hide file tree
Showing 4 changed files with 58 additions and 45 deletions.
32 changes: 18 additions & 14 deletions catalog-legacy/src/main/java/gov/nasa/pds/citool/CIToolIngester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,8 @@ private CatalogVolumeIngester createCatalogVolumeIngester(List<Label> catLabels)
catIngester.setVolumeId(id);

isVolumeCatalog = true;

checkPointerFiles(pointerFiles, lbl, catIngester);
}

catIngester.addCatalogObject(catObj);
Expand All @@ -169,20 +171,6 @@ private CatalogVolumeIngester createCatalogVolumeIngester(List<Label> catLabels)
System.exit(1);
}

for(String ptrFile : pointerFiles)
{
// Ignore the reference file when it's N/A
if(ptrFile.equals("N/A"))
{
continue;
}

if(!catIngester.labelExists(ptrFile))
{
log.warning("Missing catalog file " + ptrFile);
}
}

return catIngester;
}

Expand Down Expand Up @@ -279,4 +267,20 @@ public boolean compare(URL source, URL target) {
return false;
}
}

private void checkPointerFiles(List<String> pointerFiles, Label label,
CatalogVolumeIngester catIngester) {
for (String ptrFile : pointerFiles) {
// Ignore the reference file when it's N/A
if (ptrFile.equals("N/A")) {
continue;
}

if (!catIngester.labelExists(ptrFile)) {
LabelParserException lp = new LabelParserException(label.getLabelURI(), null, null,
"parser.warning.missingPointer", ProblemType.UNKNOWN_FILE, ptrFile);
label.addProblem(lp);
}
}
}
}
15 changes: 8 additions & 7 deletions catalog-legacy/src/main/java/gov/nasa/pds/citool/ingestor/CatalogVolumeIngester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@

public class CatalogVolumeIngester
{
public static int fileObjCount = 0;
public static int registryCount = 0;
public static int solrDocCount = 0;
public static int failCount = 0;
public static int fileObjCount = 0;
public static int registryCount = 0;
public static int solrDocCount = 0;
public static int failCount = 0;

private String archiveStatus = null;
private String volumeId;
Expand Down Expand Up @@ -410,11 +410,12 @@ private Map<String, List<String>> getDSRefs(List<CatalogObject> catObjs, String

public void publishObject(CatalogObject obj)
{
RegistryObject ro = obj.getExtrinsicObject();
if(ro == null) return;
if (obj.getExtrinsicObject() == null) {
return;
}

try {
DocGenerator.getInstance().addDoc(ro);
DocGenerator.getInstance().addDoc(obj);
solrDocCount++;
} catch (DocGeneratorException e) {
LabelParserException lp = new LabelParserException(obj.getLabel().getLabelURI(), null,
Expand Down
54 changes: 30 additions & 24 deletions catalog-legacy/src/main/java/gov/nasa/pds/citool/search/DocGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.util.Map;
import java.util.TreeMap;
import java.util.logging.Logger;
import gov.nasa.pds.citool.ingestor.CatalogObject;
import gov.nasa.pds.citool.registry.model.RegistryObject;
import gov.nasa.pds.citool.registry.model.Slots;
import gov.nasa.pds.citool.util.RegistryObjectCache;
Expand All @@ -19,6 +20,8 @@
import gov.nasa.pds.search.core.schema.OutputStringFormat;
import gov.nasa.pds.search.core.schema.Product;
import gov.nasa.pds.search.core.util.PDSDateConvert;
import gov.nasa.pds.tools.LabelParserException;
import gov.nasa.pds.tools.constants.Constants.ProblemType;


public class DocGenerator
Expand All @@ -34,7 +37,7 @@ private DocGenerator(String outDir) throws Exception
log = Logger.getLogger(this.getClass().getName());
this.outDir = outDir;
}

public static void init(String outDir) throws Exception
{
instance = new DocGenerator(outDir);
Expand Down Expand Up @@ -71,29 +74,29 @@ public void addVolume(String volumeId) throws Exception
}


public void addDoc(RegistryObject ro) throws DocGeneratorException, IOException
public void addDoc(CatalogObject obj) throws DocGeneratorException, IOException
{
String objType = ro.getObjectType();
Product conf = DocConfigManager.getInstance().getConfigByObjectType(objType);
if (conf == null) {
throw new DocGeneratorException(
"Solr Doc Generator not configured to support " + objType);
}
Map<String, List<String>> docFields = getDocFields(conf, ro.getSlots());
writer.write(docFields);
RegistryObject ro = obj.getExtrinsicObject();
String objType = ro.getObjectType();

Product conf = DocConfigManager.getInstance().getConfigByObjectType(objType);
if (conf == null) {
throw new DocGeneratorException("Solr Doc Generator not configured to support " + objType);
}

Map<String, List<String>> docFields = getDocFields(conf, ro.getSlots(), obj);
writer.write(docFields);
}


private Map<String, List<String>> getDocFields(Product conf, Slots slots)
private Map<String, List<String>> getDocFields(Product conf, Slots slots, CatalogObject obj)
throws DocGeneratorException
{
Map<String, List<String>> docFields = new TreeMap<String, List<String>>();

for(Field field: conf.getIndexFields().getField())
{
List<String> values = getFieldValues(field, slots);
List<String> values = getFieldValues(field, slots, obj);
if(!values.isEmpty())
{
docFields.put(field.getName(), values);
Expand Down Expand Up @@ -140,7 +143,8 @@ private String convertValue(Field field, String value)
}


private List<String> handleComplexPath(String regPath, Slots slots)
private List<String> handleComplexPath(String regPath, Slots slots, CatalogObject obj)
throws DocGeneratorException
{
String pathArray[] = regPath.split("\\.");
if(pathArray == null || pathArray.length < 2)
Expand All @@ -150,10 +154,11 @@ private List<String> handleComplexPath(String regPath, Slots slots)
}

List<String> lids = slots.get(pathArray[0]);
if(lids == null || lids.isEmpty())
{
log.warning("Could not find slot " + pathArray[0]);
return null;
if (lids == null || lids.isEmpty()) {
LabelParserException lp = new LabelParserException(obj.getLabel().getLabelURI(), null,
null, "ingest.warning.missingRefValue", ProblemType.UNKNOWN_VALUE, pathArray[0]);
obj.getLabel().addProblem(lp);
return null;
}

List<String> values = new ArrayList<String>();
Expand All @@ -179,7 +184,8 @@ private List<String> handleComplexPath(String regPath, Slots slots)
}


private List<String> getFieldValues(Field field, Slots slots) throws DocGeneratorException
private List<String> getFieldValues(Field field, Slots slots, CatalogObject obj)
throws DocGeneratorException
{
List<String> docValues = new ArrayList<String>();

Expand All @@ -190,7 +196,7 @@ private List<String> getFieldValues(Field field, Slots slots) throws DocGenerato

if(regPath.contains("."))
{
values = handleComplexPath(regPath, slots);
values = handleComplexPath(regPath, slots, obj);
}
else
{
Expand All @@ -209,7 +215,7 @@ private List<String> getFieldValues(Field field, Slots slots) throws DocGenerato
// Output string
if(docValues.isEmpty() && field.getOutputString() != null)
{
String value = handleTemplate(field.getOutputString(), slots);
String value = handleTemplate(field.getOutputString(), slots, obj);
if(value != null)
{
docValues.add(value);
Expand All @@ -226,7 +232,7 @@ private List<String> getFieldValues(Field field, Slots slots) throws DocGenerato
}


private String handleTemplate(OutputString outString, Slots slots)
private String handleTemplate(OutputString outString, Slots slots, CatalogObject catalogObject)
throws DocGeneratorException
{
String str = outString.getValue();
Expand All @@ -242,7 +248,7 @@ private String handleTemplate(OutputString outString, Slots slots)
try {
if(key.contains("."))
{
List<String> vals = handleComplexPath(key, slots);
List<String> vals = handleComplexPath(key, slots, catalogObject);
value = (vals != null && vals.size() > 0) ? vals.get(0) : null;
}
else
Expand Down
2 changes: 2 additions & 0 deletions catalog-legacy/src/main/java/resources.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ parser.warning.unknownUnits = The unit, "{1}", used for the value of "{0}", was
parser.warning.invalidUnits = The unit, "{1}", appears to be invalid for the id, "{0}".
parser.error.badValue = The value, "{1}", was unable to be cast to a valid type.
parser.info.placeholderValue = The value, "{1}", for "{0}" is intended as a placeholder and should be replaced before delivery.
parser.warning.missingPointer = Missing pointer file: "{0}"

## Object / Group Problems
parser.error.invalidElement = "{0}" contains the element "{1}" which is neither required nor optional.
Expand Down Expand Up @@ -151,6 +152,7 @@ ingest.error.conn = Failed to connect to the {0} - {1}.
ingest.error.dbconn = Failed to connect to the database.
ingest.error.missingKeywords = {0} - one or more of the required keywords was not found: {1}
ingest.error.missingKeyword = Failed to ingest into "{0}" table. - the keyword {1} was not found.
ingest.warning.missingRefValue = Missing value for reference "{0}". Setting to null.
ingest.error.failExecution = Failed to execute {0} method.
ingest.error.failIngestion = Failed to ingest one or more catalog object(s)..{0}
ingest.error.invalidType = Invalid type: {0} should be {1}.
Expand Down

0 comments on commit 0541c1d

Please sign in to comment.