From 6cbe379d0ca450c33340376bc480e9c539a0eac2 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Wed, 4 Sep 2024 08:12:16 +0100 Subject: [PATCH 01/17] Add example validation Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 23 ++++++ bin/check-examples.sh | 94 +++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 .github/workflows/validate_examples.yml create mode 100755 bin/check-examples.sh diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml new file mode 100644 index 0000000..eee9760 --- /dev/null +++ b/.github/workflows/validate_examples.yml @@ -0,0 +1,23 @@ +on: + pull_request: + paths: + - 'docs/*.md' + - 'docs/examples/**/*.json' + push: + paths: + - 'docs/*.md' + - 'docs/examples/**/*.json' +jobs: + validate-examples: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7 + - name: Install Python dependencies + run: | + python3 -m pip install pyshacl==0.26.0 check-jsonschema==0.29.2 + - name: Install dependencies + run: | + sudo apt install -y gawk + - name: Check examples + run: | + ./bin/check-examples.sh diff --git a/bin/check-examples.sh b/bin/check-examples.sh new file mode 100755 index 0000000..be62f7f --- /dev/null +++ b/bin/check-examples.sh @@ -0,0 +1,94 @@ +#! /bin/bash +# +# Validates SPDX example, both in separate files and inline in the +# documentation +# +# SPDX-License-Identifier: MIT + +set -e + +THIS_DIR=$(dirname $0) +SCHEMA_FILE="https://spdx.org/schema/3.0.0/spdx-json-schema.json" +RDF_FILE="https://spdx.org/rdf/3.0.0/spdx-model.ttl" +CONTEXT_FILE="https://spdx.org/rdf/3.0.0/spdx-context.jsonld" +SPDX_VERSION="3.0.0" + +for f in examples/jsonld/*.json; do + echo "Checking $f" + + check-jsonschema \ + -v \ + --schemafile $SCHEMA_FILE \ + $f + + pyshacl \ + -s $RDF_FILE \ + -e $RDF_FILE \ + $f +done + +T=$(mktemp -d) + +check_schema() { + check-jsonschema \ + -v \ + --schemafile $SCHEMA_FILE \ + "$1" +} + +check_model() { + pyshacl \ + -s $RDF_FILE \ + -e $RDF_FILE \ + "$1" +} + + +for f in $THIS_DIR/../docs/annexes/*.md; do + if ! grep -q '^```json' $f; then + continue + fi + echo "Checking $f" + DEST=$T/$(basename $f) + mkdir -p $DEST + + cat $f | awk -v DEST="$DEST" 'BEGIN{flag=0} /^```json/, $0=="```" { if (/^---$/){flag++} else if ($0 !~ /^```.*/ ) print $0 > DEST "/doc-" flag ".spdx.json"}' + + echo "[" > $DEST/combined.json + + for doc in $DEST/*.spdx.json; do + if ! grep -q '@context' $doc; then + mv $doc $doc.fragment + cat >> $doc <> $doc + cat >> $doc <> $DEST/combined.json + echo "," >> $DEST/combined.json + done + + echo "{}]" >> $DEST/combined.json + + check_model $DEST/combined.json +done From 2bcad06b48d8079c32c57d8ebf8b7404f2a2d24d Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Wed, 4 Sep 2024 08:31:23 +0100 Subject: [PATCH 02/17] Check if there's jsonld files Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index be62f7f..e739afe 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -13,22 +13,6 @@ RDF_FILE="https://spdx.org/rdf/3.0.0/spdx-model.ttl" CONTEXT_FILE="https://spdx.org/rdf/3.0.0/spdx-context.jsonld" SPDX_VERSION="3.0.0" -for f in examples/jsonld/*.json; do - echo "Checking $f" - - check-jsonschema \ - -v \ - --schemafile $SCHEMA_FILE \ - $f - - pyshacl \ - -s $RDF_FILE \ - -e $RDF_FILE \ - $f -done - -T=$(mktemp -d) - check_schema() { check-jsonschema \ -v \ @@ -43,8 +27,15 @@ check_model() { "$1" } +if [ "$(ls $THIS_DIR/../docs/examples/jsonld/*.json 2>/dev/null)" ]; then + for f in $THIS_DIR/../docs/examples/jsonld/*.json; do + echo "Checking $f" + check_schema $f + check_model $f + done +fi -for f in $THIS_DIR/../docs/annexes/*.md; do +for f in $THIS_DIR/../docs/*.md; do if ! grep -q '^```json' $f; then continue fi From 30415ba5aa37ddc01fd136600d80d58c96623153 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Wed, 4 Sep 2024 08:33:45 +0100 Subject: [PATCH 03/17] add mktemp back Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index e739afe..11b652d 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -35,6 +35,8 @@ if [ "$(ls $THIS_DIR/../docs/examples/jsonld/*.json 2>/dev/null)" ]; then done fi +T=$(mktemp -d) + for f in $THIS_DIR/../docs/*.md; do if ! grep -q '^```json' $f; then continue From ce473d331e43d57b366cb1bd37df237e49e5a00a Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Wed, 4 Sep 2024 08:37:19 +0100 Subject: [PATCH 04/17] Add json file to test checking Signed-off-by: Arthit Suriyawongkul --- docs/examples/jsonld/package_sbom.json | 98 ++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 docs/examples/jsonld/package_sbom.json diff --git a/docs/examples/jsonld/package_sbom.json b/docs/examples/jsonld/package_sbom.json new file mode 100644 index 0000000..a382351 --- /dev/null +++ b/docs/examples/jsonld/package_sbom.json @@ -0,0 +1,98 @@ +{ + "@context": "https://spdx.org/rdf/3.0.0/spdx-context.jsonld", + "@graph": [ + { + "type": "CreationInfo", + "@id": "_:creationinfo", + "createdBy": [ + "http://spdx.example.com/Agent/JoshuaWatt" + ], + "specVersion": "3.0.0", + "created": "2024-03-06T00:00:00Z" + }, + { + "type": "Person", + "spdxId": "http://spdx.example.com/Agent/JoshuaWatt", + "name": "Joshua Watt", + "creationInfo": "_:creationinfo", + "externalIdentifier": [ + { + "type": "ExternalIdentifier", + "externalIdentifierType": "email", + "identifier": "JPEWhacker@gmail.com" + } + ] + }, + { + "type": "SpdxDocument", + "spdxId": "http://spdx.example.com/Document1", + "creationInfo": "_:creationinfo", + "rootElement": [ + "http://spdx.example.com/BOM1" + ], + "element": [ + "http://spdx.example.com/BOM1", + "http://spdx.example.com/Agent/JoshuaWatt", + "http://spdx.example.com/Package1/myprogram", + "http://spdx.example.com/Relationship/1" + ], + "profileConformance": [ + "core", + "software" + ] + }, + { + "type": "software_Sbom", + "spdxId": "http://spdx.example.com/BOM1", + "creationInfo": "_:creationinfo", + "rootElement": [ + "http://spdx.example.com/Package1" + ], + "element": [ + "http://spdx.example.com/Package1/myprogram", + "http://spdx.example.com/Package1" + ], + "software_sbomType": [ + "build" + ] + }, + { + "type": "software_Package", + "spdxId": "http://spdx.example.com/Package1", + "creationInfo": "_:creationinfo", + "name": "my-package", + "software_packageVersion": "1.0", + "software_downloadLocation": "http://dl.example.com/my-package_1.0.0.tar", + "builtTime": "2024-03-06T00:00:00Z", + "originatedBy": [ + "http://spdx.example.com/Agent/JoshuaWatt" + ] + }, + { + "type": "software_File", + "spdxId": "http://spdx.example.com/Package1/myprogram", + "creationInfo": "_:creationinfo", + "name": "myprogram", + "software_primaryPurpose": "executable", + "software_additionalPurpose": [ + "application" + ], + "software_copyrightText": "Copyright 2024, Joshua Watt", + "builtTime": "2024-03-06T00:00:00Z", + "originatedBy": [ + "http://spdx.example.com/Agent/JoshuaWatt" + ] + }, + { + "type": "Relationship", + "spdxId": "http://spdx.example.com/Relationship/1", + "creationInfo": "_:creationinfo", + "from": "http://spdx.example.com/Package1", + "relationshipType": "contains", + "to": [ + "http://spdx.example.com/Package1/myprogram" + ], + "completeness": "complete" + } + ] +} From 27f0b733050a9c2cee43a4674e146cc89cb84d3d Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Wed, 4 Sep 2024 08:41:46 +0100 Subject: [PATCH 05/17] Move examples to root, outside of docs Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 4 ++-- bin/check-examples.sh | 4 ++-- {docs/examples => examples}/jsonld/package_sbom.json | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename {docs/examples => examples}/jsonld/package_sbom.json (100%) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index eee9760..e99b99e 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -2,11 +2,11 @@ on: pull_request: paths: - 'docs/*.md' - - 'docs/examples/**/*.json' + - 'examples/**/*.json' push: paths: - 'docs/*.md' - - 'docs/examples/**/*.json' + - 'examples/**/*.json' jobs: validate-examples: runs-on: ubuntu-latest diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 11b652d..b178b32 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -27,8 +27,8 @@ check_model() { "$1" } -if [ "$(ls $THIS_DIR/../docs/examples/jsonld/*.json 2>/dev/null)" ]; then - for f in $THIS_DIR/../docs/examples/jsonld/*.json; do +if [ "$(ls $THIS_DIR/../examples/jsonld/*.json 2>/dev/null)" ]; then + for f in $THIS_DIR/../examples/jsonld/*.json; do echo "Checking $f" check_schema $f check_model $f diff --git a/docs/examples/jsonld/package_sbom.json b/examples/jsonld/package_sbom.json similarity index 100% rename from docs/examples/jsonld/package_sbom.json rename to examples/jsonld/package_sbom.json From cc07e39e97e785bed4c27c3ecad677fd1146ebce Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Wed, 4 Sep 2024 08:45:59 +0100 Subject: [PATCH 06/17] Remove test example Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 6 +- examples/jsonld/package_sbom.json | 98 ------------------------- 2 files changed, 3 insertions(+), 101 deletions(-) delete mode 100644 examples/jsonld/package_sbom.json diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index e99b99e..da85dd7 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -1,12 +1,12 @@ on: pull_request: paths: - - 'docs/*.md' - - 'examples/**/*.json' + - 'docs/*.md' # For code snippets inside a document + - 'examples/jsonld/*.json' push: paths: - 'docs/*.md' - - 'examples/**/*.json' + - 'examples/jsonld/*.json' jobs: validate-examples: runs-on: ubuntu-latest diff --git a/examples/jsonld/package_sbom.json b/examples/jsonld/package_sbom.json deleted file mode 100644 index a382351..0000000 --- a/examples/jsonld/package_sbom.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "@context": "https://spdx.org/rdf/3.0.0/spdx-context.jsonld", - "@graph": [ - { - "type": "CreationInfo", - "@id": "_:creationinfo", - "createdBy": [ - "http://spdx.example.com/Agent/JoshuaWatt" - ], - "specVersion": "3.0.0", - "created": "2024-03-06T00:00:00Z" - }, - { - "type": "Person", - "spdxId": "http://spdx.example.com/Agent/JoshuaWatt", - "name": "Joshua Watt", - "creationInfo": "_:creationinfo", - "externalIdentifier": [ - { - "type": "ExternalIdentifier", - "externalIdentifierType": "email", - "identifier": "JPEWhacker@gmail.com" - } - ] - }, - { - "type": "SpdxDocument", - "spdxId": "http://spdx.example.com/Document1", - "creationInfo": "_:creationinfo", - "rootElement": [ - "http://spdx.example.com/BOM1" - ], - "element": [ - "http://spdx.example.com/BOM1", - "http://spdx.example.com/Agent/JoshuaWatt", - "http://spdx.example.com/Package1/myprogram", - "http://spdx.example.com/Relationship/1" - ], - "profileConformance": [ - "core", - "software" - ] - }, - { - "type": "software_Sbom", - "spdxId": "http://spdx.example.com/BOM1", - "creationInfo": "_:creationinfo", - "rootElement": [ - "http://spdx.example.com/Package1" - ], - "element": [ - "http://spdx.example.com/Package1/myprogram", - "http://spdx.example.com/Package1" - ], - "software_sbomType": [ - "build" - ] - }, - { - "type": "software_Package", - "spdxId": "http://spdx.example.com/Package1", - "creationInfo": "_:creationinfo", - "name": "my-package", - "software_packageVersion": "1.0", - "software_downloadLocation": "http://dl.example.com/my-package_1.0.0.tar", - "builtTime": "2024-03-06T00:00:00Z", - "originatedBy": [ - "http://spdx.example.com/Agent/JoshuaWatt" - ] - }, - { - "type": "software_File", - "spdxId": "http://spdx.example.com/Package1/myprogram", - "creationInfo": "_:creationinfo", - "name": "myprogram", - "software_primaryPurpose": "executable", - "software_additionalPurpose": [ - "application" - ], - "software_copyrightText": "Copyright 2024, Joshua Watt", - "builtTime": "2024-03-06T00:00:00Z", - "originatedBy": [ - "http://spdx.example.com/Agent/JoshuaWatt" - ] - }, - { - "type": "Relationship", - "spdxId": "http://spdx.example.com/Relationship/1", - "creationInfo": "_:creationinfo", - "from": "http://spdx.example.com/Package1", - "relationshipType": "contains", - "to": [ - "http://spdx.example.com/Package1/myprogram" - ], - "completeness": "complete" - } - ] -} From 6af91c246e31f10fb5c9ef4c805bed7377d31e46 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 5 Sep 2024 07:53:02 +0100 Subject: [PATCH 07/17] Add workflow_dispatch Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index da85dd7..c639c51 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -7,6 +7,7 @@ on: paths: - 'docs/*.md' - 'examples/jsonld/*.json' + workflow_dispatch: {} jobs: validate-examples: runs-on: ubuntu-latest From 65a31a85047baaefcaf3d6d841023b761e5e3c77 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sat, 7 Sep 2024 13:46:02 +0100 Subject: [PATCH 08/17] Update variable names Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index b178b32..a39be56 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -8,25 +8,26 @@ set -e THIS_DIR=$(dirname $0) -SCHEMA_FILE="https://spdx.org/schema/3.0.0/spdx-json-schema.json" -RDF_FILE="https://spdx.org/rdf/3.0.0/spdx-model.ttl" -CONTEXT_FILE="https://spdx.org/rdf/3.0.0/spdx-context.jsonld" +SCHEMA_URL="https://spdx.org/schema/3.0.0/spdx-json-schema.json" +RDF_URL="https://spdx.org/rdf/3.0.0/spdx-model.ttl" +CONTEXT_URL="https://spdx.org/rdf/3.0.0/spdx-context.jsonld" SPDX_VERSION="3.0.0" check_schema() { check-jsonschema \ -v \ - --schemafile $SCHEMA_FILE \ + --schemafile $SCHEMA_URL \ "$1" } check_model() { pyshacl \ - -s $RDF_FILE \ - -e $RDF_FILE \ + -s $RDF_URL \ + -e $RDF_URL \ "$1" } +# Check examples in JSON files in examples/jsonld/ if [ "$(ls $THIS_DIR/../examples/jsonld/*.json 2>/dev/null)" ]; then for f in $THIS_DIR/../examples/jsonld/*.json; do echo "Checking $f" @@ -35,18 +36,20 @@ if [ "$(ls $THIS_DIR/../examples/jsonld/*.json 2>/dev/null)" ]; then done fi -T=$(mktemp -d) - +# Check examples in inline code snippets in Markdown files in docs/annexes/ +TEMP=$(mktemp -d) for f in $THIS_DIR/../docs/*.md; do if ! grep -q '^```json' $f; then continue fi echo "Checking $f" - DEST=$T/$(basename $f) + DEST=$TEMP/$(basename $f) mkdir -p $DEST + # Read inline code snippets and save them in separate, numbered files. cat $f | awk -v DEST="$DEST" 'BEGIN{flag=0} /^```json/, $0=="```" { if (/^---$/){flag++} else if ($0 !~ /^```.*/ ) print $0 > DEST "/doc-" flag ".spdx.json"}' + # Combine all JSON code snippets into a single file, with SPDX context and creation info. echo "[" > $DEST/combined.json for doc in $DEST/*.spdx.json; do @@ -54,7 +57,7 @@ for f in $THIS_DIR/../docs/*.md; do mv $doc $doc.fragment cat >> $doc <> $doc From 0310eb60d995806aac1f67a621fd715dd0930b36 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Tue, 10 Sep 2024 17:44:53 +0100 Subject: [PATCH 09/17] Update check-examples.sh Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index a39be56..7cc0e21 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -7,11 +7,11 @@ set -e -THIS_DIR=$(dirname $0) -SCHEMA_URL="https://spdx.org/schema/3.0.0/spdx-json-schema.json" -RDF_URL="https://spdx.org/rdf/3.0.0/spdx-model.ttl" -CONTEXT_URL="https://spdx.org/rdf/3.0.0/spdx-context.jsonld" -SPDX_VERSION="3.0.0" +THIS_DIR=$(dirname "$0") +"3.0.0" +SCHEMA_URL="https://spdx.org/schema/${SPDX_VERSION}/spdx-json-schema.json" +RDF_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-model.ttl" +CONTEXT_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-context.jsonld" check_schema() { check-jsonschema \ From 6d14ed5fa9f318fcf9443192741ffc4e02870812 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 7 Oct 2024 17:14:45 +0700 Subject: [PATCH 10/17] Update dependency versions Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 5 +++-- bin/check-examples.sh | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index c639c51..4ff1b38 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MIT on: pull_request: paths: @@ -12,10 +13,10 @@ jobs: validate-examples: runs-on: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7 + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0 - name: Install Python dependencies run: | - python3 -m pip install pyshacl==0.26.0 check-jsonschema==0.29.2 + python3 -m pip install pyshacl==0.26.0 check-jsonschema==0.29.3 spdx3-validate==0.0.4 - name: Install dependencies run: | sudo apt install -y gawk diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 7cc0e21..8a195d4 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -1,14 +1,14 @@ #! /bin/bash # +# SPDX-License-Identifier: MIT +# # Validates SPDX example, both in separate files and inline in the # documentation -# -# SPDX-License-Identifier: MIT set -e THIS_DIR=$(dirname "$0") -"3.0.0" +SPDX_VERSION="3.0.0" SCHEMA_URL="https://spdx.org/schema/${SPDX_VERSION}/spdx-json-schema.json" RDF_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-model.ttl" CONTEXT_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-context.jsonld" From a23ad201986718dcd92888bec93ed75b69a5f53a Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 7 Oct 2024 17:28:15 +0700 Subject: [PATCH 11/17] Add spdx3-validate Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 65 ++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 8a195d4..375e298 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -8,41 +8,70 @@ set -e THIS_DIR=$(dirname "$0") -SPDX_VERSION="3.0.0" +MD_DIR=docs +JSON_DIR=examples/jsonld + +SPDX_VERSION="3.0.1" SCHEMA_URL="https://spdx.org/schema/${SPDX_VERSION}/spdx-json-schema.json" RDF_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-model.ttl" CONTEXT_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-context.jsonld" + +# print validation setup +echo "Checking examples" +echo "SPDX version : $SPDX_VERSION" +echo "Schema : $SCHEMA_URL" +echo "Schema resolved : $(curl -I "$SCHEMA_URL" 2>/dev/null | grep -i "location:" | awk '{print $2}')" +echo "RDF : $RDF_URL" +echo "RDF resolved : $(curl -I "$RDF_URL" 2>/dev/null | grep -i "location:" | awk '{print $2}')" +echo "Context : $CONTEXT_URL" +echo "Context resolved : $(curl -I "$CONTEXT_URL" 2>/dev/null | grep -i "location:" | awk '{print $2}')" +echo "$(check-jsonschema --version)" +echo -n "$(pyshacl --version)" +echo "spdx3-validate version: $(spdx3-validate --version)" +echo "" + check_schema() { + echo "Checking schema (check-jsonschema): $1" check-jsonschema \ - -v \ + --verbose \ --schemafile $SCHEMA_URL \ "$1" } check_model() { + echo "Checking model (pyschacl): $1" pyshacl \ - -s $RDF_URL \ - -e $RDF_URL \ + --shacl $RDF_URL \ + --ont-graph $RDF_URL \ "$1" } -# Check examples in JSON files in examples/jsonld/ -if [ "$(ls $THIS_DIR/../examples/jsonld/*.json 2>/dev/null)" ]; then - for f in $THIS_DIR/../examples/jsonld/*.json; do - echo "Checking $f" +check_spdx() { + echo "SPDX 3 validating (spdx3-validate): $1" + spdx3-validate --json $1 +} + + +# Check examples in JSON files +if [ "$(ls $THIS_DIR/../$JSON_DIR/*.json 2>/dev/null)" ]; then + for f in $THIS_DIR/../$JSON_DIR/*.json; do check_schema $f + echo "" check_model $f + echo "" + check_spdx $f + echo "" done fi -# Check examples in inline code snippets in Markdown files in docs/annexes/ +# Check examples in inline code snippets in Markdown files TEMP=$(mktemp -d) -for f in $THIS_DIR/../docs/*.md; do +for f in $THIS_DIR/../$MD_DIR/*.md; do if ! grep -q '^```json' $f; then continue fi - echo "Checking $f" + echo "Extract snippets from $f" DEST=$TEMP/$(basename $f) mkdir -p $DEST @@ -50,7 +79,8 @@ for f in $THIS_DIR/../docs/*.md; do cat $f | awk -v DEST="$DEST" 'BEGIN{flag=0} /^```json/, $0=="```" { if (/^---$/){flag++} else if ($0 !~ /^```.*/ ) print $0 > DEST "/doc-" flag ".spdx.json"}' # Combine all JSON code snippets into a single file, with SPDX context and creation info. - echo "[" > $DEST/combined.json + COMBINED_JSON = $DEST/__combined.json + echo "[" > $COMBINED_JSON for doc in $DEST/*.spdx.json; do if ! grep -q '@context' $doc; then @@ -80,11 +110,14 @@ HEREDOC HEREDOC fi check_schema $doc - cat $doc >> $DEST/combined.json - echo "," >> $DEST/combined.json + cat $doc >> $COMBINED_JSON + echo "," >> $COMBINED_JSON done - echo "{}]" >> $DEST/combined.json + echo "{}]" >> $COMBINED_JSON - check_model $DEST/combined.json + check_model $COMBINED_JSON + echo "" + check_spdx $COMBINED_JSON + echo "" done From 12d4aaec6f316f3e478d31beded976e4ae0c1a6a Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 7 Oct 2024 17:35:31 +0700 Subject: [PATCH 12/17] Sort install Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 4ff1b38..e82ac05 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0 - name: Install Python dependencies run: | - python3 -m pip install pyshacl==0.26.0 check-jsonschema==0.29.3 spdx3-validate==0.0.4 + python3 -m pip install check-jsonschema==0.29.3 pyshacl==0.26.0 spdx3-validate==0.0.4 - name: Install dependencies run: | sudo apt install -y gawk From 422aa1ead9e6a6d46ee96e3d97838f0f5e08f158 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 7 Oct 2024 17:40:54 +0700 Subject: [PATCH 13/17] Print checking locations Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 375e298..2d0535f 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -1,9 +1,9 @@ #! /bin/bash # -# SPDX-License-Identifier: MIT -# # Validates SPDX example, both in separate files and inline in the # documentation +# +# SPDX-License-Identifier: MIT set -e @@ -16,9 +16,10 @@ SCHEMA_URL="https://spdx.org/schema/${SPDX_VERSION}/spdx-json-schema.json" RDF_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-model.ttl" CONTEXT_URL="https://spdx.org/rdf/${SPDX_VERSION}/spdx-context.jsonld" - # print validation setup -echo "Checking examples" +echo "Checking examples in" +echo "Snippets : $MD_DIR" +echo "Files : $JSON_DIR" echo "SPDX version : $SPDX_VERSION" echo "Schema : $SCHEMA_URL" echo "Schema resolved : $(curl -I "$SCHEMA_URL" 2>/dev/null | grep -i "location:" | awk '{print $2}')" @@ -52,7 +53,6 @@ check_spdx() { spdx3-validate --json $1 } - # Check examples in JSON files if [ "$(ls $THIS_DIR/../$JSON_DIR/*.json 2>/dev/null)" ]; then for f in $THIS_DIR/../$JSON_DIR/*.json; do From 3783d34e0e25c62a2d805faa77f4aede4d6286e7 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 7 Oct 2024 18:02:14 +0700 Subject: [PATCH 14/17] Add SPDX-FileCopyrightText Signed-off-by: Arthit Suriyawongkul --- bin/check-examples.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/check-examples.sh b/bin/check-examples.sh index 2d0535f..a39d784 100755 --- a/bin/check-examples.sh +++ b/bin/check-examples.sh @@ -4,6 +4,7 @@ # documentation # # SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: Copyright 2024 The SPDX Contributors set -e From daf0e99be560c862150c56bef896645547ed3fb3 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Tue, 8 Oct 2024 03:17:20 +0700 Subject: [PATCH 15/17] Update checkout action to v4.2.1 Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index e82ac05..91742eb 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: Copyright 2024 The SPDX Contributors on: pull_request: paths: @@ -13,7 +13,7 @@ jobs: validate-examples: runs-on: ubuntu-latest steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1 - name: Install Python dependencies run: | python3 -m pip install check-jsonschema==0.29.3 pyshacl==0.26.0 spdx3-validate==0.0.4 From 56a4c18a88a7901fa896b03275b5fa63d99bbc9d Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sat, 12 Oct 2024 07:25:39 +0700 Subject: [PATCH 16/17] check-jsonschema==0.29.4 pyshacl==0.27.0 Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 91742eb..0b1c948 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1 - name: Install Python dependencies run: | - python3 -m pip install check-jsonschema==0.29.3 pyshacl==0.26.0 spdx3-validate==0.0.4 + python3 -m pip install check-jsonschema==0.29.4 pyshacl==0.27.0 spdx3-validate==0.0.4 - name: Install dependencies run: | sudo apt install -y gawk From e250eacef3e452032fb0a26d6a334edd8525cf87 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sun, 5 Jan 2025 10:00:38 +0000 Subject: [PATCH 17/17] Update dependency versions Signed-off-by: Arthit Suriyawongkul --- .github/workflows/validate_examples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_examples.yml b/.github/workflows/validate_examples.yml index 0b1c948..8838cf0 100644 --- a/.github/workflows/validate_examples.yml +++ b/.github/workflows/validate_examples.yml @@ -13,10 +13,10 @@ jobs: validate-examples: runs-on: ubuntu-latest steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 - name: Install Python dependencies run: | - python3 -m pip install check-jsonschema==0.29.4 pyshacl==0.27.0 spdx3-validate==0.0.4 + python3 -m pip install check-jsonschema==0.30.0 pyshacl==0.29.1 spdx3-validate==0.0.5 - name: Install dependencies run: | sudo apt install -y gawk