feat!: publish content claims by multihash (#61)

* Claims can now be published by multihash, the `content` property should be a `Link` OR `{ digest: Uint8Array }` when sending an invocation * Read API `GET /claims/:cid` is deprecated * Added `GET /claims/cid/:cid` for reading claims by CID * Added `GET /claims/multihash/:multihash` for reading claims by (base58btc encoded) multihash * ~~Removed "Relation claim" - this is not published by any w3up infra and has significant overlap with upcomsing [w3-index](https://github.com/w3s-project/specs/blob/main/w3-index.md) claim~~ resolves #60 BREAKING CHANGE: Client read interface and client claim types now use multihashes. Relation claim has been removed in favour of upcoming dag-index claim.
storacha · May 29, 2024 · 151f4a1 · 151f4a1
1 parent 0a09254
commit 151f4a1
Show file tree

Hide file tree

Showing 22 changed files with 4,967 additions and 1,127 deletions.
diff --git a/README.md b/README.md
@@ -115,7 +115,6 @@ Input:
 }
 ```
 
-
 ## Usage
 
 ### Client libraries
@@ -128,14 +127,18 @@ Client libraries make reading and writing claims easier.
 
 The production deployment is at https://claims.web3.storage.
 
-#### `GET /claims/:cid`
+#### `GET /claims/multihash/:multihash`
 
-Fetch a CAR full of content claims for the content CID in the URL path.
+Fetch a CAR full of content claims for the base58 encoded (Multibase `base58btc`) content hash in the URL path.
 
 Query parameters:
 
 * `?walk=` - a CSV list of properties in claims to walk in order to return additional claims about the related CIDs. Any property that is a CID can be walked. e.g. `?walk=parts,includes`.
 
+#### `GET /claims/cid/:cid`
+
+As above, except passing a CID instead of multihash.
+
 ### CLI
 
 There is a command line interface for invoking the HTTP API in [./packages/cli](./packages/cli).

diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/cli/bin.js b/packages/cli/bin.js
@@ -189,7 +189,7 @@ prog
   .action(async (contentArg, opts) => {
     const content = Link.parse(contentArg)
     const walk = Array.isArray(opts.walk) ? opts.walk : opts.walk?.split(',')
-    const res = await Client.fetch(content, { walk, serviceURL })
+    const res = await Client.fetch(content.multihash, { walk, serviceURL })
     if (!res.ok) throw new Error(`unexpected service status: ${res.status}`, { cause: await res.text() })
     if (!res.body) throw new Error('missing response body')
 

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -18,9 +18,9 @@
     "@ucanto/principal": "^9.0.1",
     "@ucanto/transport": "^9.1.1",
     "@web3-storage/content-claims": "file:../core",
-    "carstream": "^1.0.2",
+    "carstream": "^2.0.0",
     "dotenv": "^16.3.1",
-    "multiformats": "^12.0.1",
+    "multiformats": "^13.1.0",
     "parse-duration": "^1.1.0",
     "sade": "^1.8.1",
     "typescript": "^5.1.6"

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -27,7 +27,6 @@
     "location",
     "inclusion",
     "partition",
-    "relation",
     "UCAN",
     "IPLD",
     "IPFS"
@@ -112,11 +111,11 @@
   },
   "dependencies": {
     "@ucanto/client": "^9.0.1",
+    "@ucanto/interface": "^10.0.0",
     "@ucanto/server": "^10.0.0",
     "@ucanto/transport": "^9.1.1",
-    "@ucanto/interface": "^10.0.0",
-    "carstream": "^1.0.2",
-    "multiformats": "^12.0.1"
+    "carstream": "^2.0.0",
+    "multiformats": "^13.1.0"
   },
   "devDependencies": {
     "@ipld/dag-cbor": "^9.0.3",

diff --git a/packages/core/src/capability/assert.js b/packages/core/src/capability/assert.js
@@ -1,4 +1,6 @@
-import { capability, URI, Link, Schema } from '@ucanto/server'
+import { capability, URI, Schema } from '@ucanto/server'
+
+const linkOrDigest = () => Schema.link().or(Schema.struct({ digest: Schema.bytes() }))
 
 export const assert = capability({
   can: 'assert/*',
@@ -13,7 +15,7 @@ export const location = capability({
   with: URI.match({ protocol: 'did:' }),
   nb: Schema.struct({
     /** CAR CID */
-    content: Link,
+    content: linkOrDigest(),
     location: Schema.array(URI),
     range: Schema.struct({
       offset: Schema.integer(),
@@ -30,10 +32,10 @@ export const inclusion = capability({
   with: URI.match({ protocol: 'did:' }),
   nb: Schema.struct({
     /** CAR CID */
-    content: Link,
+    content: linkOrDigest(),
     /** CARv2 index CID */
-    includes: Link.match({ version: 1 }),
-    proof: Link.match({ version: 1 }).optional()
+    includes: Schema.link({ version: 1 }),
+    proof: Schema.link({ version: 1 }).optional()
   })
 })
 
@@ -45,10 +47,10 @@ export const partition = capability({
   with: URI.match({ protocol: 'did:' }),
   nb: Schema.struct({
     /** Content root CID */
-    content: Link,
+    content: linkOrDigest(),
     /** CIDs CID */
-    blocks: Link.match({ version: 1 }).optional(),
-    parts: Schema.array(Link.match({ version: 1 }))
+    blocks: Schema.link({ version: 1 }).optional(),
+    parts: Schema.array(Schema.link({ version: 1 }))
   })
 })
 
@@ -59,17 +61,17 @@ export const relation = capability({
   can: 'assert/relation',
   with: URI.match({ protocol: 'did:' }),
   nb: Schema.struct({
-    content: Link,
+    content: linkOrDigest(),
     /** CIDs this content links to directly. */
-    children: Schema.array(Link),
+    children: Schema.array(Schema.link()),
     /** Parts this content and it's children can be read from. */
     parts: Schema.array(Schema.struct({
-      content: Link.match({ version: 1 }),
+      content: Schema.link({ version: 1 }),
       /** CID of contents (CARv2 index) included in this part. */
       includes: Schema.struct({
-        content: Link.match({ version: 1 }),
+        content: Schema.link({ version: 1 }),
         /** CIDs of parts this index may be found in. */
-        parts: Schema.array(Link.match({ version: 1 })).optional()
+        parts: Schema.array(Schema.link({ version: 1 })).optional()
       }).optional()
     }))
   })
@@ -82,7 +84,7 @@ export const equals = capability({
   can: 'assert/equals',
   with: URI.match({ protocol: 'did:' }),
   nb: Schema.struct({
-    content: Link,
-    equals: Link
+    content: linkOrDigest(),
+    equals: Schema.link()
   })
 })
diff --git a/packages/core/src/client/api.ts b/packages/core/src/client/api.ts
@@ -1,10 +1,10 @@
-import { Link, URI, UnknownLink, Block } from '@ucanto/client'
+import { Link, URI, UnknownLink, Block, MultihashDigest } from '@ucanto/client'
 import * as Assert from '../capability/assert.js'
 
 /** A verifiable claim about data. */
 export interface ContentClaim<T extends string> {
-  /** Subject of the claim e.g. CAR CID, DAG root CID etc. */
-  readonly content: UnknownLink
+  /** Subject of the claim e.g. CAR, DAG root etc. */
+  readonly content: MultihashDigest
   /** Discriminator for different types of claims. */
   readonly type: T
   /**
@@ -68,8 +68,6 @@ export interface RelationPartInclusion {
 
 /** A claim that the same data is referred to by another CID and/or multihash */
 export interface EqualsClaim extends ContentClaim<typeof Assert.equals.can> {
-  /** Any CID e.g a CAR CID */
-  readonly content: UnknownLink
   /** A CID that is equivalent to the content CID e.g the Piece CID for that CAR CID */
   readonly equals: UnknownLink
 }

diff --git a/packages/core/src/client/index.js b/packages/core/src/client/index.js
@@ -3,7 +3,9 @@ import { extract as extractDelegation } from '@ucanto/core/delegation'
 import { connect, invoke, delegate } from '@ucanto/client'
 import { CAR, HTTP } from '@ucanto/transport'
 import { sha256 } from 'multiformats/hashes/sha2'
+import { decode as decodeDigest } from 'multiformats/hashes/digest'
 import { equals } from 'multiformats/bytes'
+import { base58btc } from 'multiformats/bases/base58'
 import { CARReaderStream } from 'carstream/reader'
 import * as Assert from '../capability/assert.js'
 
@@ -21,6 +23,25 @@ export const connection = connect({
 
 export { connect, invoke, delegate, CAR, HTTP }
 
+const assertCapNames = [
+  Assert.location.can,
+  Assert.partition.can,
+  Assert.inclusion.can,
+  Assert.relation.can,
+  Assert.equals.can
+]
+
+/**
+ * @param {import('@ucanto/interface').Capability} cap
+ * @returns {cap is import('../server/api.js').AnyAssertCap}
+ */
+const isAssertCap = cap =>
+  // @ts-expect-error
+  assertCapNames.includes(cap.can) &&
+  'nb' in cap &&
+  typeof cap.nb === 'object' &&
+  'content' in cap.nb
+
 /**
  * @param {Uint8Array} bytes
  * @returns {Promise<import('./api.js').Claim>}
@@ -31,53 +52,41 @@ export const decode = async bytes => {
     throw new Error('failed to decode claim', { cause: delegation.error })
   }
   const cap = delegation.ok.capabilities[0]
-  if (!cap.nb || typeof cap.nb !== 'object' || !('content' in cap.nb)) {
+  if (!isAssertCap(cap)) {
     throw new Error('invalid claim')
   }
   // @ts-expect-error
   return {
     ...cap.nb,
-    type: claimType(cap.can),
+    content: 'digest' in cap.nb.content ? decodeDigest(cap.nb.content.digest) : cap.nb.content.multihash,
+    type: cap.can,
     export: () => delegation.ok.export(),
     archive: async () => bytes
   }
 }
 
-/** @param {string} can */
-const claimType = can =>
-  can === Assert.location.can
-    ? Assert.location.can
-    : can === Assert.partition.can
-      ? Assert.partition.can
-      : can === Assert.inclusion.can
-        ? Assert.inclusion.can
-        : can === Assert.relation.can
-          ? Assert.relation.can
-          : can === Assert.equals.can
-            ? Assert.equals.can
-            : 'unknown'
-
 /**
  * Fetch a CAR archive of claims from the service. Note: no verification is
  * performed on the response data.
  *
  * @typedef {{
-*   walk?: Array<'parts'|'includes'|'children'>
-*   serviceURL?: URL
-* }} FetchOptions
-* @param {import('@ucanto/client').UnknownLink} content
-* @param {FetchOptions} [options]
-*/
+ *   walk?: Array<'parts'|'includes'|'children'>
+ *   serviceURL?: URL
+ * }} FetchOptions
+ * @param {import('multiformats').MultihashDigest} content
+ * @param {FetchOptions} [options]
+ */
 export const fetch = async (content, options) => {
-  const url = new URL(`/claims/${content}`, options?.serviceURL ?? serviceURL)
+  const path = `/claims/multihash/${base58btc.encode(content.bytes)}`
+  const url = new URL(path, options?.serviceURL ?? serviceURL)
   if (options?.walk) url.searchParams.set('walk', options.walk.join(','))
   return globalThis.fetch(url)
 }
 
 /**
  * Read content claims from the service for the given content CID.
  *
- * @param {import('@ucanto/client').UnknownLink} content
+ * @param {import('multiformats').MultihashDigest} content
  * @param {FetchOptions} [options]
  * @returns {Promise<import('./api.js').Claim[]>}
  */

diff --git a/packages/core/src/server/api.ts b/packages/core/src/server/api.ts
@@ -1,5 +1,5 @@
-import { MultihashDigest } from 'multiformats/hashes/digest'
-import { UnknownLink, Link } from 'multiformats/link'
+import { MultihashDigest } from 'multiformats'
+import { Link } from 'multiformats/link'
 import { AnyAssertCap } from './service/api.js'
 
 export { AnyAssertCap }
@@ -13,7 +13,7 @@ export interface Claim {
 }
 
 export interface ClaimFetcher {
-  get (content: UnknownLink): Promise<Claim[]>
+  get (content: MultihashDigest): Promise<Claim[]>
 }
 
 export interface ClaimStore extends ClaimFetcher {

diff --git a/packages/core/src/server/index.js b/packages/core/src/server/index.js
@@ -30,7 +30,7 @@ export const createServer = ({ id, codec, errorReporter: errorHandler, validateA
  * returning a stream of content claims.
  *
  * @param {{ claimFetcher: import('./api.js').ClaimFetcher }} context
- * @param {import('multiformats').UnknownLink} content
+ * @param {import('multiformats').MultihashDigest} content
  * @param {Set<string>} walk
  */
 export const walkClaims = (context, content, walk) => {
@@ -69,13 +69,13 @@ export const walkClaims = (context, content, walk) => {
                 if (Array.isArray(content)) {
                   for (const c of content) {
                     if (Link.isLink(c)) {
-                      queue.push(c)
+                      queue.push(c.multihash)
                     } else if (content && typeof content === 'object') {
                       walkKeys(content)
                     }
                   }
                 } else if (Link.isLink(content)) {
-                  queue.push(content)
+                  queue.push(content.multihash)
                 } else if (content && typeof content === 'object') {
                   walkKeys(content)
                 }

diff --git a/packages/core/src/server/service/assert.js b/packages/core/src/server/service/assert.js
@@ -1,4 +1,5 @@
 import * as Server from '@ucanto/server'
+import * as Digest from 'multiformats/hashes/digest'
 import * as Assert from '../../capability/assert.js'
 
 /**
@@ -29,7 +30,9 @@ export const handler = async ({ capability, invocation }, { claimStore }) => {
   const claim = {
     claim: invocation.cid,
     bytes: archive.ok,
-    content: content.multihash,
+    content: 'digest' in content
+      ? Digest.decode(content.digest)
+      : content.multihash,
     expiration: invocation.expiration,
     value: capability
   }