feat: Include created (ingested) timestamp in Member

rdf-connect · Oct 17, 2024 · ef801c0 · ef801c0
1 parent 1b91f4e
commit ef801c0
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 6 deletions.
diff --git a/lib/memberManager.ts b/lib/memberManager.ts
@@ -2,7 +2,7 @@ import { Quad, Term } from "@rdfjs/types";
 import { Fragment, Member } from "./page";
 import { FetchedPage } from "./pageFetcher";
 import { CBDShapeExtractor } from "extract-cbd-shape";
-import { DC, TREE } from "@treecg/types";
+import { DC, LDES, TREE } from "@treecg/types";
 import { LDESInfo } from "./client";
 import { getObjects, memberFromQuads, Notifier } from "./utils";
 import { RdfStore } from "rdf-stores";
@@ -161,7 +161,7 @@ export class Manager {
         member: Term,
         data: RdfStore,
     ): Promise<Quad[]> {
-        return await this.extractor.extract(data, member, this.shapeId);
+        return await this.extractor.extract(data, member, this.shapeId, [namedNode(LDES.custom("IngestionMetadata"))]);
     }
 
     private async extractMember(
@@ -171,6 +171,7 @@ export class Manager {
         if (this.state.has(member.value)) return;
 
         const quads: Quad[] = await this.extractMemberQuads(member, data);
+        const created = getObjects(data, member, DC.terms.custom("created"), namedNode(LDES.custom("IngestionMetadata")))[0]?.value;
 
         if (quads.length > 0) {
             this.state.add(member.value);
@@ -179,6 +180,7 @@ export class Manager {
                 quads,
                 this.timestampPath,
                 this.isVersionOfPath,
+                created ? new Date(created) : undefined,
             );
         }
     }

diff --git a/lib/page.ts b/lib/page.ts
@@ -1,19 +1,23 @@
 import { Quad, Term } from "@rdfjs/types";
-import { RDF, TREE } from "@treecg/types";
+import { DC, LDES, RDF, TREE } from "@treecg/types";
 import { CBDShapeExtractor } from "extract-cbd-shape";
 import { State } from "./state";
 import { RdfStore } from "rdf-stores";
 import { getObjects, memberFromQuads } from "./utils";
 import { Condition } from "./condition";
 import { RelationCondition } from "./condition/range";
 import { getLoggerFor } from "./utils/logUtil";
+import { DataFactory } from "n3";
+
+const { namedNode } = DataFactory;
 
 export interface Member {
     id: Term;
     quads: Quad[];
     timestamp?: string | Date;
     isVersionOf?: string;
     type?: Term;
+    created?: Date;
 }
 
 export interface Relations {
@@ -53,8 +57,9 @@ export function extractMembers(
     const members = getObjects(store, stream, TREE.terms.member, null);
 
     async function extractMember(member: Term) {
-        const quads = await extractor.extract(store, member, shapeId);
-        cb(memberFromQuads(member, quads, timestampPath, isVersionOfPath));
+        const quads = await extractor.extract(store, member, shapeId, [namedNode(LDES.custom("IngestionMetadata"))]);
+        const created = getObjects(store, member, DC.terms.custom("created"), namedNode(LDES.custom("IngestionMetadata")))[0]?.value;
+        cb(memberFromQuads(member, quads, timestampPath, isVersionOfPath, created ? new Date(created) : undefined));
     }
 
     const out: Promise<void>[] = [];

diff --git a/lib/utils.ts b/lib/utils.ts
@@ -523,6 +523,7 @@ export function memberFromQuads(
     quads: Quad[],
     timestampPath: Term | undefined,
     isVersionOfPath: Term | undefined,
+    created?: Date,
 ): Member {
     // Get timestamp
     let timestamp: string | Date | undefined;
@@ -553,7 +554,7 @@ export function memberFromQuads(
     const type: Term | undefined = quads.find(
         (x) => x.subject.equals(member) && x.predicate.value === RDF.type,
     )?.object;
-    return { quads, id: member, isVersionOf, timestamp, type };
+    return { quads, id: member, isVersionOf, timestamp, type, created };
 }
 
 /**