Bugfix/kbdev 1243 add support for factor and fusion 3

bin/load.js

@@ -109,6 +109,16 @@ civicParser.add_argument('--noUpdate', {
     default: false,
     help: 'Will not check for updating content of existing GraphKB Statements',
 });
+civicParser.add_argument('--noDeleteOnUnmatched', {
+    action: 'store_true',
+    default: false,
+    help: 'Will not delete GraphKB Statements from valid sourceID but not matching a combination',
+});
+civicParser.add_argument('--deleteDeprecated', {
+    action: 'store_true',
+    default: false,
+    help: 'Will delete GraphKB Statements from deprecated sourceID',
+});
 
 const clinicaltrialsgovParser = subparsers.add_parser('clinicaltrialsgov');
 clinicaltrialsgovParser.add_argument('--days', {

src/civic/evidenceItem.js

@@ -181,8 +181,7 @@ const processCombination = async (conn, {
             logger.verbose(`converted variant name (${variant.name}) to variants (${processedVariants.map(v => v.displayName).join(', and ')})`);
             variants.push(...processedVariants);
         } catch (err) {
-            logger.error(`unable to process the variant (id=${rawRecord.variant.id}, name=${rawRecord.variant.name})`);
-            throw err;
+            throw new Error(`unable to process the variant (id=${variant.id}, name=${variant.name})`);
         }
     }
 

src/civic/index.js

@@ -19,6 +19,7 @@ const {
     contentMatching,
     createStatement,
     deleteStatements,
+    getStatements,
     needsUpdate,
     updateStatement,
 } = require('./statement');
@@ -55,16 +56,20 @@ const incrementCounts = (initial, updates) => {
  *
  * @param {object} param0
  * @param {ApiConnection} param0.conn the api connection object for GraphKB
+ * @param {?boolean} param0.deleteDeprecated deletion of existing GraphKB Statements related to deprecated evidence(s)
  * @param {string} param0.errorLogPrefix prefix to the generated error json file
  * @param {number} param0.maxRecords limit of EvidenceItem records to be processed and upload
- * @param {?boolean} param0.noUpdate for avoiding deletion/update of existing GraphKB Statements
+ * @param {?boolean} param0.noDeleteOnUnmatched no deletion of existing GraphKB Statements related to unmatched combination(s)
+ * @param {?boolean} param0.noUpdate no update of existing GraphKB Statements
  * @param {string[]} param0.trustedCurators a list of curator IDs for submitted-only EvidenceItems
  * @param {?string} param0.url url to use as the base for accessing the civic ApiConnection
  */
 const upload = async ({
     conn,
+    deleteDeprecated = false, // Won't delete deprecated sourceIds by default
     errorLogPrefix,
     maxRecords,
+    noDeleteOnUnmatched = false,
     noUpdate = false,
     trustedCurators,
     url = BASE_URL,
@@ -181,7 +186,8 @@ const upload = async ({
         processingIntoCombinations: new Map(),
         relevance: new Map(),
     };
-    const casesToReview = new Map();
+    const statementsToReviewUnmatchedProcessingError = new Map();
+    const statementsToReviewUnmatched = new Map();
 
     logger.info(`\n\n${'#'.repeat(80)}\n## PROCESSING RECORDS\n${'#'.repeat(80)}\n`);
     let recordNumber = 0;
@@ -366,13 +372,17 @@ const upload = async ({
         }
 
         // DELETE
-        if (!noUpdate && toDelete.length > 0) {
+        if (toDelete.length > 0) {
             const rids = toDelete.map(el => el['@rid']);
 
             if (processCombinationErrors > 0) {
                 // Do not delete any statements if some combinations have processing errors
-                logger.info(`${toDelete.length} unmatched statement(s) to be reviewed for deletion`);
-                casesToReview.set(id, rids);
+                logger.warn(`${toDelete.length} unmatched statement(s). To be reviewed since some processing errors occured`);
+                statementsToReviewUnmatchedProcessingError.set(id, rids);
+            } else if (noDeleteOnUnmatched) {
+                // Do not delete any statements if noDeleteOnUnmatched flag
+                logger.warn(`${toDelete.length} unmatched statement(s). To be reviewed since the noDeleteOnUnmatched flag is being used`);
+                statementsToReviewUnmatched.set(id, rids);
             } else {
                 loaclCountsST.delete = await deleteStatements(conn, { rids });
             }
@@ -414,40 +424,59 @@ const upload = async ({
     }
 
     // DELETING UNWANTED GRAPHKB STATEMENTS
-    // sourceIds no longer in CIViC (not accepted/submitted by trustedCurators) but still in GraphKB
+    // sourceIds no longer in CIViC (not accepted/submitted-by-trustedCurators) but still in GraphKB
     const allIdsFromCivic = new Set(evidenceItems.map(r => r.id.toString()));
-    const toDelete = new Set([...sourceIdsFromGKB].filter(x => !allIdsFromCivic.has(x)));
+    const sourceIdstoDeleteStatementsFrom = Array.from(
+        new Set([...sourceIdsFromGKB].filter(x => !allIdsFromCivic.has(x))),
+    );
     logger.info();
     logger.info('***** Deprecated items *****');
-    logger.warn(`${toDelete.size} deprecated ${SOURCE_DEFN.name} Evidence Items still in GraphKB Statement`);
+    logger.warn(`${sourceIdstoDeleteStatementsFrom.length} deprecated ${SOURCE_DEFN.name} Evidence Items still in GraphKB Statement`);
 
-    if (toDelete.size > 0) {
-        logger.info(`sourceIds: ${Array.from(toDelete)}`);
+    if (sourceIdstoDeleteStatementsFrom.length > 0) {
+        logger.info(`sourceIds: ${sourceIdstoDeleteStatementsFrom}`);
     }
 
     // GraphKB Statements Soft-deletion
-    if (!noUpdate && toDelete.size > 0) {
-        const deletedCount = await deleteStatements(conn, {
-            source: sourceRid,
-            sourceIds: Array.from(toDelete),
-        });
-        const attempts = deletedCount.success + deletedCount.err;
-        logger.info(`${deletedCount.success}/${attempts} soft-deleted statements`);
-
-        if (countsST) {
-            countsST.delete.err += deletedCount.err;
-            countsST.delete.success += deletedCount.success;
+    if (sourceIdstoDeleteStatementsFrom.length > 0) {
+        if (!deleteDeprecated) {
+            // Do not delete any statements if no deleteDeprecated flag
+            const deprecatedStatementRids = await getStatements(
+                conn,
+                { source: sourceRid, sourceIds: sourceIdstoDeleteStatementsFrom },
+            );
+            logger.warn(`${deprecatedStatementRids.length} corresponding deprecated statement(s). To be reviewed since no deleteDeprecated flag`);
+            const deprecatedStatementsFilepath = `${errorLogPrefix}-civic-deprecatedStatements.json`;
+            logger.info(`writing ${deprecatedStatementsFilepath}`);
+            fs.writeFileSync(
+                deprecatedStatementsFilepath,
+                JSON.stringify(deprecatedStatementRids, null, 2),
+            );
         } else {
-            countsST = { delete: { err: deletedCount.err, success: deletedCount.success } };
+            const deletedCount = await deleteStatements(conn, {
+                source: sourceRid,
+                sourceIds: sourceIdstoDeleteStatementsFrom,
+            });
+            const attempts = deletedCount.success + deletedCount.err;
+            logger.info(`${deletedCount.success}/${attempts} soft-deleted statements`);
+
+            if (countsST) {
+                countsST.delete.err += deletedCount.err;
+                countsST.delete.success += deletedCount.success;
+            } else {
+                countsST = { delete: { err: deletedCount.err, success: deletedCount.success } };
+            }
         }
     }
 
     // Logging processing error cases to be reviewed,
     // so a reviewer can decide if corresponding statements need to be deleted or not
     logger.info();
-    logger.info('***** Cases to be reviewed for deletion *****');
-    logger.warn(`${casesToReview.size} Evidence Item(s) with processing errors leading to unmatched Statement(s)`);
-    casesToReview.forEach((v, k) => logger.info(`${k} -> ${JSON.stringify(v)}`));
+    logger.info('***** Unmatched cases to be reviewed for deletion *****');
+    logger.warn(`${statementsToReviewUnmatchedProcessingError.size} Evidence Item(s) with processing errors leading to unmatched Statement(s)`);
+    statementsToReviewUnmatchedProcessingError.forEach((v, k) => logger.info(`${k} -> ${JSON.stringify(v)}`));
+    logger.warn(`${statementsToReviewUnmatched.size} Evidence Item(s) with unmatched Statement(s) with no processing error involved`);
+    statementsToReviewUnmatched.forEach((v, k) => logger.info(`${k} -> ${JSON.stringify(v)}`));
 
     // Logging Statement CRUD operations counts
     if (countsST) {

src/civic/statement.js

@@ -177,6 +177,31 @@ const contentMatching = ({
     return records;
 };
 
+/**
+ * Given source & list of sourceIds, returns corresponding statements
+ *
+ * @param {ApiConnection} conn the api connection object for GraphKB
+ * @param {object} param1
+ * @param {string} param1.source the source RID
+ * @param {string[]} param1.sourceIds an array of sourceIds
+ * @returns {string[]} a list of statement RIDs
+ */
+const getStatements = async (conn, { source, sourceIds }) => {
+    const records = await conn.getRecords({
+        filters: {
+            AND: [
+                { sourceId: sourceIds },
+                { source },
+            ],
+        },
+        target: 'Statement',
+    });
+    const rids = records.map(
+        (el) => el['@rid'],
+    );
+    return rids;
+};
+
 /**
  * Given content from CIViC, try to create the GraphKB record
  *
@@ -250,18 +275,7 @@ const deleteStatements = async (conn, { rids = [], source, sourceIds }) => {
     // Get rids to delete if none provided
     if (rids.length === 0) {
         logger.info('Loading corresponding GraphKB statement RIDs to delete');
-        const records = await conn.getRecords({
-            filters: {
-                AND: [
-                    { sourceId: sourceIds },
-                    { source },
-                ],
-            },
-            target: 'Statement',
-        });
-        rids.push(...records.map(
-            (el) => el['@rid'],
-        ));
+        rids.push(...await getStatements(conn, { source, sourceIds }));
         logger.info(`${rids.length} RIDs found`);
     }
 
@@ -286,6 +300,7 @@ module.exports = {
     contentMatching,
     createStatement,
     deleteStatements,
+    getStatements,
     isMatching,
     needsUpdate,
     updateStatement,

src/civic/variant.js

@@ -273,7 +273,11 @@ const uploadNormalizedVariant = async (conn, normalizedVariant, feature) => {
                 SOURCE_DEFN.name,
             );
         } catch (err) {
-            variantType = await conn.getVocabularyTerm(normalizedVariant.type || content.type);
+            try {
+                variantType = await conn.getVocabularyTerm(normalizedVariant.type || content.type);
+            } catch (e) {
+                throw new Error(`Unable to upload CIVIC variant (${normalizedVariant}})`);
+            }
         }
         content.type = rid(variantType);
 
@@ -392,6 +396,7 @@ const processVariantRecord = async (conn, civicVariantRecord, feature) => {
         }
     } catch (err) {
         VARIANT_CACHE.set(JSON.stringify(rawVariant), { err });
+        throw err;
     }
 
     VARIANT_CACHE.set(JSON.stringify(rawVariant), { result });