e2e ingestion working

lunasec-io · Mar 22, 2023 · fb0618c · fb0618c
1 parent f23e965
commit fb0618c
Show file tree

Hide file tree

Showing 18 changed files with 333 additions and 110 deletions.
diff --git a/go.mod b/go.mod
@@ -86,6 +86,8 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect
+	github.com/CloudyKit/jet/v6 v6.2.0 // indirect
 	github.com/CycloneDX/cyclonedx-go v0.7.1-0.20221222100750-41a1ac565cce // indirect
 	github.com/DataDog/zstd v1.4.5 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect

diff --git a/go.sum b/go.sum
@@ -190,8 +190,11 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 h1:sR+/8Yb4slttB4vD+b9btVEnWgL3Q00OBTzVT8B9C0c=
 github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
 github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=
+github.com/CloudyKit/jet/v6 v6.2.0 h1:EpcZ6SR9n28BUGtNJSvlBqf90IpjeFr36Tizxhn/oME=
+github.com/CloudyKit/jet/v6 v6.2.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4=
 github.com/CycloneDX/cyclonedx-go v0.7.1-0.20221222100750-41a1ac565cce h1:o5r3msApzvtE5LhcMkxWaKernD/PK0HpMccu7ywBj5Q=
 github.com/CycloneDX/cyclonedx-go v0.7.1-0.20221222100750-41a1ac565cce/go.mod h1:XURd0m8zvnLE5aIRqg6JOVRl7qZ/pWBtuFa9EHjQwFc=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=

diff --git a/lunatrace/bsl/hasura/metadata/databases/lunatrace/tables/tables.yaml b/lunatrace/bsl/hasura/metadata/databases/lunatrace/tables/tables.yaml
@@ -48,6 +48,7 @@
 - "!include vulnerability_affected_range_event.yaml"
 - "!include vulnerability_affected_version.yaml"
 - "!include vulnerability_cisa_known_exploited.yaml"
+- "!include vulnerability_code_snippet.yaml"
 - "!include vulnerability_credit.yaml"
 - "!include vulnerability_cwe.yaml"
 - "!include vulnerability_equivalent.yaml"

diff --git a/lunatrace/bsl/hasura/metadata/databases/lunatrace/tables/vulnerability_code_snippet.yaml b/lunatrace/bsl/hasura/metadata/databases/lunatrace/tables/vulnerability_code_snippet.yaml
@@ -0,0 +1,10 @@
+table:
+  name: code_snippet
+  schema: vulnerability
+object_relationships:
+  - name: reference
+    using:
+      foreign_key_constraint_on: reference_id
+  - name: vulnerabilityByVulnerability
+    using:
+      foreign_key_constraint_on: vulnerability
diff --git a/lunatrace/bsl/hasura/metadata/remote_schemas.yaml b/lunatrace/bsl/hasura/metadata/remote_schemas.yaml
@@ -13,81 +13,81 @@
     - role: user
       definition:
         schema: |
-          scalar JSON
-          scalar UUID
           type AuthenticatedRepoCloneUrlOutput {
-            url: String
+          	url: String
           }
+          scalar JSON
           type Mutation {
-            presignManifestUpload(project_id: UUID!): PresignedUrlResponse
+          	presignManifestUpload(project_id: UUID!): PresignedUrlResponse
           }
           type PresignedUrlResponse {
-            bucket: String!
-            headers: JSON!
-            key: String!
-            url: String!
+          	bucket: String!
+          	headers: JSON!
+          	key: String!
+          	url: String!
           }
           type Query {
-            authenticatedRepoCloneUrl(repoGithubId: Int!): AuthenticatedRepoCloneUrlOutput
-            fakeQueryToHackHasuraBeingABuggyMess: String
-            sbomUrl(buildId: UUID!): String
+          	authenticatedRepoCloneUrl(repoGithubId: Int!): AuthenticatedRepoCloneUrlOutput
+          	fakeQueryToHackHasuraBeingABuggyMess: String
+          	sbomUrl(buildId: UUID!): String
           }
           type SbomUploadUrlOutput {
-            error: Boolean!
-            uploadUrl: UploadUrl
+          	error: Boolean!
+          	uploadUrl: UploadUrl
           }
+          scalar UUID
           type UploadUrl {
-            headers: JSON!
-            url: String!
+          	headers: JSON!
+          	url: String!
           }
     - role: service
       definition:
         schema: |
-          scalar JSON
-          scalar UUID
           type AuthenticatedRepoCloneUrlOutput {
-            url: String
+          	url: String
           }
+          scalar JSON
           type Mutation {
-            presignManifestUpload(project_id: UUID!): PresignedUrlResponse
+          	presignManifestUpload(project_id: UUID!): PresignedUrlResponse
           }
           type PresignedUrlResponse {
-            bucket: String!
-            headers: JSON!
-            key: String!
-            url: String!
+          	bucket: String!
+          	headers: JSON!
+          	key: String!
+          	url: String!
           }
           type Query {
-            authenticatedRepoCloneUrl(repoGithubId: Int!): AuthenticatedRepoCloneUrlOutput
-            fakeQueryToHackHasuraBeingABuggyMess: String
-            presignSbomUpload(orgId: UUID!, buildId: UUID!): SbomUploadUrlOutput
-            sbomUrl(buildId: UUID!): String
+          	authenticatedRepoCloneUrl(repoGithubId: Int!): AuthenticatedRepoCloneUrlOutput
+          	fakeQueryToHackHasuraBeingABuggyMess: String
+          	presignSbomUpload(orgId: UUID!, buildId: UUID!): SbomUploadUrlOutput
+          	sbomUrl(buildId: UUID!): String
+          }
+          input SbomUploadUrlInput {
+          	orgId: UUID!
+          	projectId: UUID!
           }
           type SbomUploadUrlOutput {
-            error: Boolean!
-            uploadUrl: UploadUrl
+          	error: Boolean!
+          	uploadUrl: UploadUrl
           }
+          scalar UUID
           type UploadUrl {
-            headers: JSON!
-            url: String!
-          }
-          input SbomUploadUrlInput {
-            orgId: UUID!
-            projectId: UUID!
+          	headers: JSON!
+          	url: String!
           }
     - role: cli
       definition:
         schema: |
           scalar JSON
-          scalar UUID
           type Query {
-            presignSbomUpload(orgId: UUID!, buildId: UUID!): SbomUploadUrlOutput
+          	presignSbomUpload(orgId: UUID!, buildId: UUID!): SbomUploadUrlOutput
           }
           type SbomUploadUrlOutput {
-            error: Boolean!
-            uploadUrl: UploadUrl
+          	error: Boolean!
+          	uploadUrl: UploadUrl
           }
+          scalar UUID
           type UploadUrl {
-            headers: JSON!
-            url: String!
+          	headers: JSON!
+          	url: String!
           }
diff --git a/lunatrace/bsl/hasura/migrations/lunatrace/1679066158985_add_ml_tables/up.sql b/lunatrace/bsl/hasura/migrations/lunatrace/1679066158985_add_ml_tables/up.sql
@@ -7,6 +7,7 @@ CREATE TABLE vulnerability.code_snippet
     reference_id uuid NULL references vulnerability.reference,
     -- Include url since reference might be null but its still nice to be able to point a source like a vuln-db link for non-scraped content
     source_url text NOT NULL,
+    -- Messed up and forgot the ID part of this name..its not the end of the world though so leaving it
     vulnerability uuid NOT NULL references vulnerability.vulnerability,
     code text NOT NULL,
     score integer NOT NULL,

diff --git a/...race/bsl/hasura/migrations/lunatrace/1679503926339_make-code-snippet-text-unique/down.sql b/...race/bsl/hasura/migrations/lunatrace/1679503926339_make-code-snippet-text-unique/down.sql
@@ -0,0 +1 @@
+DROP INDEX vulnerability.code_snippet_unq_idx;
diff --git a/lunatrace/bsl/hasura/migrations/lunatrace/1679503926339_make-code-snippet-text-unique/up.sql b/lunatrace/bsl/hasura/migrations/lunatrace/1679503926339_make-code-snippet-text-unique/up.sql
@@ -0,0 +1,2 @@
+
+CREATE UNIQUE INDEX code_snippet_unq_idx ON vulnerability.code_snippet (vulnerability, code);
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@

		CREATE UNIQUE INDEX code_snippet_unq_idx ON vulnerability.code_snippet (vulnerability, code);