diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24993639945..dd8ed4766a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,6 +107,13 @@ jobs: command: test args: --package graph-tests --test runner_tests + - name: Run file link resolver test + id: file-link-resolver-test + uses: actions-rs/cargo@v1 + with: + command: test + args: --package graph-tests --test file_link_resolver + integration-tests: name: Run integration tests runs-on: ubuntu-latest diff --git a/graph/src/components/link_resolver/file.rs b/graph/src/components/link_resolver/file.rs new file mode 100644 index 00000000000..827a3f267e5 --- /dev/null +++ b/graph/src/components/link_resolver/file.rs @@ -0,0 +1,188 @@ +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::anyhow; +use async_trait::async_trait; +use slog::Logger; + +use crate::data::subgraph::Link; +use crate::prelude::{Error, JsonValueStream, LinkResolver as LinkResolverTrait}; + +#[derive(Clone, Debug)] +pub struct FileLinkResolver { + base_dir: Option, + timeout: Duration, +} + +impl FileLinkResolver { + /// Create a new FileLinkResolver + /// + /// All paths are treated as absolute paths. + pub fn new() -> Self { + Self { + base_dir: None, + timeout: Duration::from_secs(30), + } + } + + /// Create a new FileLinkResolver with a base directory + /// + /// All paths that are not absolute will be considered + /// relative to this base directory. + pub fn with_base_dir>(base_dir: P) -> Self { + Self { + base_dir: Some(base_dir.as_ref().to_owned()), + timeout: Duration::from_secs(30), + } + } + + fn resolve_path(&self, link: &str) -> PathBuf { + let path = Path::new(link); + + // Return the path as is if base_dir is None, or join with base_dir if present. + // if "link" is an absolute path, join will simply return that path. + self.base_dir + .as_ref() + .map_or_else(|| path.to_owned(), |base_dir| base_dir.join(link)) + } +} + +pub fn remove_prefix(link: &str) -> &str { + const IPFS: &str = "/ipfs/"; + if link.starts_with(IPFS) { + &link[IPFS.len()..] + } else { + link + } +} + +#[async_trait] +impl LinkResolverTrait for FileLinkResolver { + fn with_timeout(&self, timeout: Duration) -> Box { + let mut resolver = self.clone(); + resolver.timeout = timeout; + Box::new(resolver) + } + + fn with_retries(&self) -> Box { + Box::new(self.clone()) + } + + async fn cat(&self, logger: &Logger, link: &Link) -> Result, Error> { + let link = remove_prefix(&link.link); + let path = self.resolve_path(&link); + + slog::debug!(logger, "File resolver: reading file"; + "path" => path.to_string_lossy().to_string()); + + match tokio::fs::read(&path).await { + Ok(data) => Ok(data), + Err(e) => { + slog::error!(logger, "Failed to read file"; + "path" => path.to_string_lossy().to_string(), + "error" => e.to_string()); + Err(anyhow!("Failed to read file {}: {}", path.display(), e).into()) + } + } + } + + async fn get_block(&self, _logger: &Logger, _link: &Link) -> Result, Error> { + Err(anyhow!("get_block is not implemented for FileLinkResolver").into()) + } + + async fn json_stream(&self, _logger: &Logger, _link: &Link) -> Result { + Err(anyhow!("json_stream is not implemented for FileLinkResolver").into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + use std::fs; + use std::io::Write; + + #[tokio::test] + async fn test_file_resolver_absolute() { + // Test the resolver without a base directory (absolute paths only) + + // Create a temporary directory for test files + let temp_dir = env::temp_dir().join("file_resolver_test"); + let _ = fs::create_dir_all(&temp_dir); + + // Create a test file in the temp directory + let test_file_path = temp_dir.join("test.txt"); + let test_content = b"Hello, world!"; + let mut file = fs::File::create(&test_file_path).unwrap(); + file.write_all(test_content).unwrap(); + + // Create a resolver without a base directory + let resolver = FileLinkResolver::new(); + let logger = slog::Logger::root(slog::Discard, slog::o!()); + + // Test valid path resolution + let link = Link { + link: test_file_path.to_string_lossy().to_string(), + }; + let result = resolver.cat(&logger, &link).await.unwrap(); + assert_eq!(result, test_content); + + // Test path with leading slash that likely doesn't exist + let link = Link { + link: "/test.txt".to_string(), + }; + let result = resolver.cat(&logger, &link).await; + assert!( + result.is_err(), + "Reading /test.txt should fail as it doesn't exist" + ); + + // Clean up + let _ = fs::remove_file(test_file_path); + let _ = fs::remove_dir(temp_dir); + } + + #[tokio::test] + async fn test_file_resolver_with_base_dir() { + // Test the resolver with a base directory + + // Create a temporary directory for test files + let temp_dir = env::temp_dir().join("file_resolver_test_base_dir"); + let _ = fs::create_dir_all(&temp_dir); + + // Create a test file in the temp directory + let test_file_path = temp_dir.join("test.txt"); + let test_content = b"Hello from base dir!"; + let mut file = fs::File::create(&test_file_path).unwrap(); + file.write_all(test_content).unwrap(); + + // Create a resolver with a base directory + let resolver = FileLinkResolver::with_base_dir(&temp_dir); + let logger = slog::Logger::root(slog::Discard, slog::o!()); + + // Test relative path (no leading slash) + let link = Link { + link: "test.txt".to_string(), + }; + let result = resolver.cat(&logger, &link).await.unwrap(); + assert_eq!(result, test_content); + + // Test absolute path + let link = Link { + link: test_file_path.to_string_lossy().to_string(), + }; + let result = resolver.cat(&logger, &link).await.unwrap(); + assert_eq!(result, test_content); + + // Test missing file + let link = Link { + link: "missing.txt".to_string(), + }; + let result = resolver.cat(&logger, &link).await; + assert!(result.is_err()); + + // Clean up + let _ = fs::remove_file(test_file_path); + let _ = fs::remove_dir(temp_dir); + } +} diff --git a/graph/src/components/link_resolver/mod.rs b/graph/src/components/link_resolver/mod.rs index 1115b59cdc3..851b4296b47 100644 --- a/graph/src/components/link_resolver/mod.rs +++ b/graph/src/components/link_resolver/mod.rs @@ -7,10 +7,12 @@ use crate::prelude::Error; use std::fmt::Debug; mod arweave; +mod file; mod ipfs; pub use arweave::*; use async_trait::async_trait; +pub use file::*; pub use ipfs::*; /// Resolves links to subgraph manifests and resources referenced by them. diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index 77c8ba67d36..10c4e471e38 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -116,20 +116,23 @@ impl DeploymentHash { pub fn new(s: impl Into) -> Result { let s = s.into(); - // Enforce length limit - if s.len() > 46 { - return Err(s); - } + // When the disable_deployment_hash_validation flag is set, we skip the validation + if !ENV_VARS.disable_deployment_hash_validation { + // Enforce length limit + if s.len() > 46 { + return Err(s); + } - // Check that the ID contains only allowed characters. - if !s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { - return Err(s); - } + // Check that the ID contains only allowed characters. + if !s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(s); + } - // Allow only deployment id's for 'real' subgraphs, not the old - // metadata subgraph. - if s == "subgraphs" { - return Err(s); + // Allow only deployment id's for 'real' subgraphs, not the old + // metadata subgraph. + if s == "subgraphs" { + return Err(s); + } } Ok(DeploymentHash(s)) @@ -397,12 +400,65 @@ impl From> for DataSourceContext { } /// IPLD link. -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq)] pub struct Link { - #[serde(rename = "/")] pub link: String, } +/// Custom deserializer for Link +/// This handles both formats: +/// 1. Simple string: "schema.graphql" or "subgraph.yaml" which is used in [`FileLinkResolver`] +/// FileLinkResolver is used in local development environments +/// 2. IPLD format: { "/": "Qm..." } which is used in [`IpfsLinkResolver`] +impl<'de> de::Deserialize<'de> for Link { + fn deserialize(deserializer: D) -> Result + where + D: de::Deserializer<'de>, + { + struct LinkVisitor; + + impl<'de> de::Visitor<'de> for LinkVisitor { + type Value = Link; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("string or map with '/' key") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(Link { + link: value.to_string(), + }) + } + + fn visit_map(self, mut map: A) -> Result + where + A: de::MapAccess<'de>, + { + let mut link = None; + + while let Some(key) = map.next_key::()? { + if key == "/" { + if link.is_some() { + return Err(de::Error::duplicate_field("/")); + } + link = Some(map.next_value()?); + } else { + return Err(de::Error::unknown_field(&key, &["/"])); + } + } + + link.map(|l: String| Link { link: l }) + .ok_or_else(|| de::Error::missing_field("/")) + } + } + + deserializer.deserialize_any(LinkVisitor) + } +} + impl From for Link { fn from(s: S) -> Self { Self { diff --git a/graph/src/env/mod.rs b/graph/src/env/mod.rs index eff0ebea16e..9611bd2726a 100644 --- a/graph/src/env/mod.rs +++ b/graph/src/env/mod.rs @@ -225,6 +225,12 @@ pub struct EnvVars { /// if no genesis hash can be retrieved from an adapter. If enabled, the adapter is /// ignored if unable to produce a genesis hash or produces a different an unexpected hash. pub genesis_validation_enabled: bool, + /// Whether to enforce deployment hash validation rules. + /// When disabled, any string can be used as a deployment hash. + /// When enabled, deployment hashes must meet length and character constraints. + /// + /// Set by the flag `GRAPH_NODE_DISABLE_DEPLOYMENT_HASH_VALIDATION`. Enabled by default. + pub disable_deployment_hash_validation: bool, /// How long do we wait for a response from the provider before considering that it is unavailable. /// Default is 30s. pub genesis_validation_timeout: Duration, @@ -332,6 +338,7 @@ impl EnvVars { section_map: inner.section_map, firehose_grpc_max_decode_size_mb: inner.firehose_grpc_max_decode_size_mb, genesis_validation_enabled: inner.genesis_validation_enabled.0, + disable_deployment_hash_validation: inner.disable_deployment_hash_validation.0, genesis_validation_timeout: Duration::from_secs(inner.genesis_validation_timeout), graphman_server_auth_token: inner.graphman_server_auth_token, firehose_disable_extended_blocks_for_chains: @@ -528,6 +535,11 @@ struct Inner { firehose_block_fetch_timeout: u64, #[envconfig(from = "GRAPH_FIREHOSE_FETCH_BLOCK_BATCH_SIZE", default = "10")] firehose_block_fetch_batch_size: usize, + #[envconfig( + from = "GRAPH_NODE_DISABLE_DEPLOYMENT_HASH_VALIDATION", + default = "false" + )] + disable_deployment_hash_validation: EnvVarBoolean, } #[derive(Clone, Debug)] diff --git a/tests/runner-tests/file-link-resolver/abis/Contract.abi b/tests/runner-tests/file-link-resolver/abis/Contract.abi new file mode 100644 index 00000000000..9d9f56b9263 --- /dev/null +++ b/tests/runner-tests/file-link-resolver/abis/Contract.abi @@ -0,0 +1,15 @@ +[ + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "string", + "name": "testCommand", + "type": "string" + } + ], + "name": "TestEvent", + "type": "event" + } +] diff --git a/tests/runner-tests/file-link-resolver/package.json b/tests/runner-tests/file-link-resolver/package.json new file mode 100644 index 00000000000..11e796fc72c --- /dev/null +++ b/tests/runner-tests/file-link-resolver/package.json @@ -0,0 +1,13 @@ +{ + "name": "file-link-resolver", + "version": "0.1.0", + "scripts": { + "codegen": "graph codegen --skip-migrations", + "create:test": "graph create test/file-link-resolver --node $GRAPH_NODE_ADMIN_URI", + "deploy:test": "graph deploy test/file-link-resolver --version-label v0.0.1 --ipfs $IPFS_URI --node $GRAPH_NODE_ADMIN_URI" + }, + "devDependencies": { + "@graphprotocol/graph-cli": "0.60.0", + "@graphprotocol/graph-ts": "0.31.0" + } +} diff --git a/tests/runner-tests/file-link-resolver/schema.graphql b/tests/runner-tests/file-link-resolver/schema.graphql new file mode 100644 index 00000000000..2eec3606b65 --- /dev/null +++ b/tests/runner-tests/file-link-resolver/schema.graphql @@ -0,0 +1,5 @@ +type Block @entity { + id: ID! + number: BigInt! + hash: Bytes! +} \ No newline at end of file diff --git a/tests/runner-tests/file-link-resolver/src/mapping.ts b/tests/runner-tests/file-link-resolver/src/mapping.ts new file mode 100644 index 00000000000..ecce2ff9de5 --- /dev/null +++ b/tests/runner-tests/file-link-resolver/src/mapping.ts @@ -0,0 +1,11 @@ +import { ethereum, log } from "@graphprotocol/graph-ts"; +import { Block } from "../generated/schema"; + +export function handleBlock(block: ethereum.Block): void { + log.info("Processing block: {}", [block.number.toString()]); + + let blockEntity = new Block(block.number.toString()); + blockEntity.number = block.number; + blockEntity.hash = block.hash; + blockEntity.save(); +} diff --git a/tests/runner-tests/file-link-resolver/subgraph.yaml b/tests/runner-tests/file-link-resolver/subgraph.yaml new file mode 100644 index 00000000000..4a50915beb4 --- /dev/null +++ b/tests/runner-tests/file-link-resolver/subgraph.yaml @@ -0,0 +1,22 @@ +specVersion: 0.0.8 +schema: + file: ./schema.graphql +dataSources: + - kind: ethereum/contract + name: Contract + network: test + source: + address: "0x0000000000000000000000000000000000000000" + abi: Contract + mapping: + kind: ethereum/events + apiVersion: 0.0.7 + language: wasm/assemblyscript + entities: + - Block + abis: + - name: Contract + file: ./abis/Contract.abi + blockHandlers: + - handler: handleBlock + file: ./src/mapping.ts diff --git a/tests/runner-tests/yarn.lock b/tests/runner-tests/yarn.lock index 2f9f1287bec..ee3d9e5202a 100644 --- a/tests/runner-tests/yarn.lock +++ b/tests/runner-tests/yarn.lock @@ -349,40 +349,6 @@ which "2.0.2" yaml "1.10.2" -"@graphprotocol/graph-cli@0.79.0-alpha-20240711124603-49edf22": - version "0.79.0-alpha-20240711124603-49edf22" - resolved "https://registry.yarnpkg.com/@graphprotocol/graph-cli/-/graph-cli-0.79.0-alpha-20240711124603-49edf22.tgz#4e3f6201932a0b68ce64d6badd8432cf2bead3c2" - integrity sha512-fZrdPiFbbbBVMnvsjfKA+j48WzzquaHQIpozBqnUKRPCV1n1NenIaq2nH16mlMwovRIS7AAIVCpa0QYQuPzw7Q== - dependencies: - "@float-capital/float-subgraph-uncrashable" "^0.0.0-alpha.4" - "@oclif/core" "2.8.6" - "@oclif/plugin-autocomplete" "^2.3.6" - "@oclif/plugin-not-found" "^2.4.0" - "@whatwg-node/fetch" "^0.8.4" - assemblyscript "0.19.23" - binary-install-raw "0.0.13" - chalk "3.0.0" - chokidar "3.5.3" - debug "4.3.4" - docker-compose "0.23.19" - dockerode "2.5.8" - fs-extra "9.1.0" - glob "9.3.5" - gluegun "5.1.6" - graphql "15.5.0" - immutable "4.2.1" - ipfs-http-client "55.0.0" - jayson "4.0.0" - js-yaml "3.14.1" - open "8.4.2" - prettier "3.0.3" - semver "7.4.0" - sync-request "6.1.0" - tmp-promise "3.0.3" - web3-eth-abi "1.7.0" - which "2.0.2" - yaml "1.10.2" - "@graphprotocol/graph-ts@0.30.0": version "0.30.0" resolved "https://registry.npmjs.org/@graphprotocol/graph-ts/-/graph-ts-0.30.0.tgz" @@ -1507,11 +1473,6 @@ defaults@^1.0.3: dependencies: clone "^1.0.2" -define-lazy-prop@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz#3f7ae421129bcaaac9bc74905c98a0009ec9ee7f" - integrity sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og== - delay@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz" @@ -1584,13 +1545,6 @@ ejs@3.1.6: dependencies: jake "^10.6.1" -ejs@3.1.8: - version "3.1.8" - resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.8.tgz#758d32910c78047585c7ef1f92f9ee041c1c190b" - integrity sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ== - dependencies: - jake "^10.8.5" - ejs@^3.1.8: version "3.1.9" resolved "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz" @@ -2055,42 +2009,6 @@ gluegun@5.1.2: which "2.0.2" yargs-parser "^21.0.0" -gluegun@5.1.6: - version "5.1.6" - resolved "https://registry.yarnpkg.com/gluegun/-/gluegun-5.1.6.tgz#74ec13193913dc610f5c1a4039972c70c96a7bad" - integrity sha512-9zbi4EQWIVvSOftJWquWzr9gLX2kaDgPkNR5dYWbM53eVvCI3iKuxLlnKoHC0v4uPoq+Kr/+F569tjoFbA4DSA== - dependencies: - apisauce "^2.1.5" - app-module-path "^2.2.0" - cli-table3 "0.6.0" - colors "1.4.0" - cosmiconfig "7.0.1" - cross-spawn "7.0.3" - ejs "3.1.8" - enquirer "2.3.6" - execa "5.1.1" - fs-jetpack "4.3.1" - lodash.camelcase "^4.3.0" - lodash.kebabcase "^4.1.1" - lodash.lowercase "^4.3.0" - lodash.lowerfirst "^4.3.1" - lodash.pad "^4.5.1" - lodash.padend "^4.6.1" - lodash.padstart "^4.6.1" - lodash.repeat "^4.1.0" - lodash.snakecase "^4.1.1" - lodash.startcase "^4.4.0" - lodash.trim "^4.5.1" - lodash.trimend "^4.5.1" - lodash.trimstart "^4.5.1" - lodash.uppercase "^4.3.0" - lodash.upperfirst "^4.3.1" - ora "4.0.2" - pluralize "^8.0.0" - semver "7.3.5" - which "2.0.2" - yargs-parser "^21.0.0" - graceful-fs@^4.1.6, graceful-fs@^4.2.0: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" @@ -2377,7 +2295,7 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" -is-docker@^2.0.0, is-docker@^2.1.1: +is-docker@^2.0.0: version "2.2.1" resolved "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz" integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ== @@ -3022,15 +2940,6 @@ onetime@^5.1.0, onetime@^5.1.2: dependencies: mimic-fn "^2.1.0" -open@8.4.2: - version "8.4.2" - resolved "https://registry.yarnpkg.com/open/-/open-8.4.2.tgz#5b5ffe2a8f793dcd2aad73e550cb87b59cb084f9" - integrity sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ== - dependencies: - define-lazy-prop "^2.0.0" - is-docker "^2.1.1" - is-wsl "^2.2.0" - ora@4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/ora/-/ora-4.0.2.tgz" @@ -3151,11 +3060,6 @@ prettier@1.19.1: resolved "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz" integrity sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew== -prettier@3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.3.tgz#432a51f7ba422d1469096c0fdc28e235db8f9643" - integrity sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg== - process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz" diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index 217c7f705b6..b8151857db3 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -3,6 +3,7 @@ pub mod substreams; use std::collections::{BTreeSet, HashMap}; use std::marker::PhantomData; +use std::path::PathBuf; use std::sync::Mutex; use std::time::{Duration, Instant}; @@ -17,7 +18,9 @@ use graph::blockchain::{ TriggerFilterWrapper, TriggersAdapter, TriggersAdapterSelector, }; use graph::cheap_clone::CheapClone; -use graph::components::link_resolver::{ArweaveClient, ArweaveResolver, FileSizeLimit}; +use graph::components::link_resolver::{ + ArweaveClient, ArweaveResolver, FileLinkResolver, FileSizeLimit, +}; use graph::components::metrics::MetricsRegistry; use graph::components::network_provider::ChainName; use graph::components::store::{BlockStore, DeploymentLocator, EthereumCallCache, SourceableStore}; @@ -38,7 +41,7 @@ use graph::prelude::ethabi::ethereum_types::H256; use graph::prelude::serde_json::{self, json}; use graph::prelude::{ async_trait, lazy_static, q, r, ApiVersion, BigInt, BlockNumber, DeploymentHash, - GraphQlRunner as _, IpfsResolver, LoggerFactory, NodeId, QueryError, + GraphQlRunner as _, IpfsResolver, LinkResolver, LoggerFactory, NodeId, QueryError, SubgraphAssignmentProvider, SubgraphCountMetric, SubgraphName, SubgraphRegistrar, SubgraphStore as _, SubgraphVersionSwitchingMode, TriggerProcessor, }; @@ -455,6 +458,38 @@ pub async fn setup( chain: &impl TestChainTrait, graft_block: Option, env_vars: Option, +) -> TestContext { + setup_inner(test_info, stores, chain, graft_block, env_vars, None).await +} + +pub async fn setup_with_file_link_resolver( + test_info: &TestInfo, + stores: &Stores, + chain: &impl TestChainTrait, + graft_block: Option, + env_vars: Option, +) -> TestContext { + let mut base_dir = PathBuf::from(test_info.test_dir.clone()); + base_dir.push("build"); + let link_resolver = Arc::new(FileLinkResolver::with_base_dir(base_dir)); + setup_inner( + test_info, + stores, + chain, + graft_block, + env_vars, + Some(link_resolver), + ) + .await +} + +pub async fn setup_inner( + test_info: &TestInfo, + stores: &Stores, + chain: &impl TestChainTrait, + graft_block: Option, + env_vars: Option, + link_resolver: Option>, ) -> TestContext { let env_vars = Arc::new(match env_vars { Some(ev) => ev, @@ -483,10 +518,13 @@ pub async fn setup( .unwrap(), ); - let link_resolver = Arc::new(IpfsResolver::new( - ipfs_client.cheap_clone(), - Default::default(), - )); + let link_resolver = match link_resolver { + Some(link_resolver) => link_resolver, + None => Arc::new(IpfsResolver::new( + ipfs_client.cheap_clone(), + Default::default(), + )), + }; let ipfs_service = ipfs_service( ipfs_client.cheap_clone(), diff --git a/tests/src/lib.rs b/tests/src/lib.rs index c89168d7003..2b67fc4dc44 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -4,6 +4,7 @@ pub mod fixture; pub mod helpers; #[macro_use] pub mod macros; +pub mod recipe; pub mod subgraph; pub use config::{Config, DbConfig, EthConfig, CONFIG}; diff --git a/tests/src/recipe.rs b/tests/src/recipe.rs new file mode 100644 index 00000000000..de7f0ebe0c1 --- /dev/null +++ b/tests/src/recipe.rs @@ -0,0 +1,133 @@ +use crate::{ + fixture::{stores, Stores, TestInfo}, + helpers::run_cmd, +}; +use graph::ipfs; +use graph::prelude::{DeploymentHash, SubgraphName}; +use std::process::Command; +pub struct RunnerTestRecipe { + pub stores: Stores, + pub test_info: TestInfo, +} + +impl RunnerTestRecipe { + pub async fn new(test_name: &str, subgraph_name: &str) -> Self { + let subgraph_name = SubgraphName::new(subgraph_name).unwrap(); + let test_dir = format!("./runner-tests/{}", subgraph_name); + + let (stores, hash) = tokio::join!( + stores(test_name, "./runner-tests/config.simple.toml"), + build_subgraph(&test_dir, None) + ); + + Self { + stores, + test_info: TestInfo { + test_dir, + test_name: test_name.to_string(), + subgraph_name, + hash, + }, + } + } + + /// Builds a new test subgraph with a custom deploy command. + pub async fn new_with_custom_cmd(name: &str, subgraph_name: &str, deploy_cmd: &str) -> Self { + let subgraph_name = SubgraphName::new(subgraph_name).unwrap(); + let test_dir = format!("./runner-tests/{}", subgraph_name); + + let (stores, hash) = tokio::join!( + stores(name, "./runner-tests/config.simple.toml"), + build_subgraph(&test_dir, Some(deploy_cmd)) + ); + + Self { + stores, + test_info: TestInfo { + test_dir, + test_name: name.to_string(), + subgraph_name, + hash, + }, + } + } + + pub async fn new_with_file_link_resolver( + name: &str, + subgraph_name: &str, + manifest: &str, + ) -> Self { + let subgraph_name = SubgraphName::new(subgraph_name).unwrap(); + let test_dir = format!("./runner-tests/{}", subgraph_name); + + let stores = stores(name, "./runner-tests/config.simple.toml").await; + build_subgraph(&test_dir, None).await; + let hash = DeploymentHash::new(manifest).unwrap(); + Self { + stores, + test_info: TestInfo { + test_dir, + test_name: name.to_string(), + subgraph_name, + hash, + }, + } + } +} + +/// deploy_cmd is the command to run to deploy the subgraph. If it is None, the +/// default `yarn deploy:test` is used. +async fn build_subgraph(dir: &str, deploy_cmd: Option<&str>) -> DeploymentHash { + build_subgraph_with_yarn_cmd(dir, deploy_cmd.unwrap_or("deploy:test")).await +} + +async fn build_subgraph_with_yarn_cmd(dir: &str, yarn_cmd: &str) -> DeploymentHash { + build_subgraph_with_yarn_cmd_and_arg(dir, yarn_cmd, None).await +} + +pub async fn build_subgraph_with_yarn_cmd_and_arg( + dir: &str, + yarn_cmd: &str, + arg: Option<&str>, +) -> DeploymentHash { + // Test that IPFS is up. + ipfs::IpfsRpcClient::new(ipfs::ServerAddress::local_rpc_api(), &graph::log::discard()) + .await + .expect("Could not connect to IPFS, make sure it's running at port 5001"); + + // Make sure dependencies are present. + + run_cmd( + Command::new("yarn") + .arg("install") + .arg("--mutex") + .arg("file:.yarn-mutex") + .current_dir("./runner-tests/"), + ); + + // Run codegen. + run_cmd(Command::new("yarn").arg("codegen").current_dir(dir)); + + let mut args = vec![yarn_cmd]; + args.extend(arg); + + // Run `deploy` for the side effect of uploading to IPFS, the graph node url + // is fake and the actual deploy call is meant to fail. + let deploy_output = run_cmd( + Command::new("yarn") + .args(&args) + .env("IPFS_URI", "http://127.0.0.1:5001") + .env("GRAPH_NODE_ADMIN_URI", "http://localhost:0") + .current_dir(dir), + ); + + // Hack to extract deployment id from `graph deploy` output. + const ID_PREFIX: &str = "Build completed: "; + let Some(mut line) = deploy_output.lines().find(|line| line.contains(ID_PREFIX)) else { + panic!("No deployment id found, graph deploy probably had an error") + }; + if !line.starts_with(ID_PREFIX) { + line = &line[5..line.len() - 5]; // workaround for colored output + } + DeploymentHash::new(line.trim_start_matches(ID_PREFIX)).unwrap() +} diff --git a/tests/tests/file_link_resolver.rs b/tests/tests/file_link_resolver.rs new file mode 100644 index 00000000000..1b12aef64c4 --- /dev/null +++ b/tests/tests/file_link_resolver.rs @@ -0,0 +1,62 @@ +use graph::object; +use graph_tests::{ + fixture::{ + self, + ethereum::{chain, empty_block, genesis}, + test_ptr, + }, + recipe::RunnerTestRecipe, +}; + +#[tokio::test] +async fn file_link_resolver() -> anyhow::Result<()> { + std::env::set_var("GRAPH_NODE_DISABLE_DEPLOYMENT_HASH_VALIDATION", "true"); + let RunnerTestRecipe { stores, test_info } = RunnerTestRecipe::new_with_file_link_resolver( + "file_link_resolver", + "file-link-resolver", + "subgraph.yaml", + ) + .await; + + let blocks = { + let block_0 = genesis(); + let block_1 = empty_block(block_0.ptr(), test_ptr(1)); + let block_2 = empty_block(block_1.ptr(), test_ptr(2)); + let block_3 = empty_block(block_2.ptr(), test_ptr(3)); + + vec![block_0, block_1, block_2, block_3] + }; + + let chain = chain(&test_info.test_name, blocks, &stores, None).await; + + let ctx = fixture::setup_with_file_link_resolver(&test_info, &stores, &chain, None, None).await; + ctx.start_and_sync_to(test_ptr(3)).await; + let query = r#"{ blocks(first: 4, orderBy: number) { id, hash } }"#; + let query_res = ctx.query(query).await.unwrap(); + + assert_eq!( + query_res, + Some(object! { + blocks: vec![ + object! { + id: test_ptr(0).number.to_string(), + hash: format!("0x{}", test_ptr(0).hash_hex()), + }, + object! { + id: test_ptr(1).number.to_string(), + hash: format!("0x{}", test_ptr(1).hash_hex()), + }, + object! { + id: test_ptr(2).number.to_string(), + hash: format!("0x{}", test_ptr(2).hash_hex()), + }, + object! { + id: test_ptr(3).number.to_string(), + hash: format!("0x{}", test_ptr(3).hash_hex()), + }, + ] + }) + ); + + Ok(()) +} diff --git a/tests/tests/runner_tests.rs b/tests/tests/runner_tests.rs index 261c886dfea..48a1bb67ffe 100644 --- a/tests/tests/runner_tests.rs +++ b/tests/tests/runner_tests.rs @@ -1,5 +1,4 @@ use std::marker::PhantomData; -use std::process::Command; use std::str::FromStr; use std::sync::atomic::{self, AtomicBool}; use std::sync::Arc; @@ -13,14 +12,11 @@ use graph::data::subgraph::schema::{SubgraphError, SubgraphHealth}; use graph::data::value::Word; use graph::data_source::CausalityRegion; use graph::env::{EnvVars, TEST_WITH_NO_REORG}; -use graph::ipfs; use graph::ipfs::test_utils::add_files_to_local_ipfs_node_for_testing; use graph::object; use graph::prelude::ethabi::ethereum_types::H256; use graph::prelude::web3::types::Address; -use graph::prelude::{ - hex, CheapClone, DeploymentHash, SubgraphAssignmentProvider, SubgraphName, SubgraphStore, -}; +use graph::prelude::{hex, CheapClone, SubgraphAssignmentProvider, SubgraphName, SubgraphStore}; use graph_tests::fixture::ethereum::{ chain, empty_block, generate_empty_blocks_for_range, genesis, push_test_command, push_test_log, push_test_polling_trigger, @@ -28,60 +24,12 @@ use graph_tests::fixture::ethereum::{ use graph_tests::fixture::substreams::chain as substreams_chain; use graph_tests::fixture::{ - self, stores, test_ptr, test_ptr_reorged, MockAdapterSelector, NoopAdapterSelector, Stores, - TestChainTrait, TestContext, TestInfo, + self, test_ptr, test_ptr_reorged, MockAdapterSelector, NoopAdapterSelector, TestChainTrait, + TestContext, TestInfo, }; -use graph_tests::helpers::run_cmd; +use graph_tests::recipe::{build_subgraph_with_yarn_cmd_and_arg, RunnerTestRecipe}; use slog::{o, Discard, Logger}; -struct RunnerTestRecipe { - pub stores: Stores, - pub test_info: TestInfo, -} - -impl RunnerTestRecipe { - async fn new(test_name: &str, subgraph_name: &str) -> Self { - let subgraph_name = SubgraphName::new(subgraph_name).unwrap(); - let test_dir = format!("./runner-tests/{}", subgraph_name); - - let (stores, hash) = tokio::join!( - stores(test_name, "./runner-tests/config.simple.toml"), - build_subgraph(&test_dir, None) - ); - - Self { - stores, - test_info: TestInfo { - test_dir, - test_name: test_name.to_string(), - subgraph_name, - hash, - }, - } - } - - /// Builds a new test subgraph with a custom deploy command. - async fn new_with_custom_cmd(name: &str, subgraph_name: &str, deploy_cmd: &str) -> Self { - let subgraph_name = SubgraphName::new(subgraph_name).unwrap(); - let test_dir = format!("./runner-tests/{}", subgraph_name); - - let (stores, hash) = tokio::join!( - stores(name, "./runner-tests/config.simple.toml"), - build_subgraph(&test_dir, Some(deploy_cmd)) - ); - - Self { - stores, - test_info: TestInfo { - test_dir, - test_name: name.to_string(), - subgraph_name, - hash, - }, - } - } -} - fn assert_eq_ignore_backtrace(err: &SubgraphError, expected: &SubgraphError) { let equal = { if err.subgraph_id != expected.subgraph_id @@ -1278,60 +1226,3 @@ async fn arweave_file_data_sources() { Some(object! { file: object!{ id: id, content: content.clone() } }) ); } - -/// deploy_cmd is the command to run to deploy the subgraph. If it is None, the -/// default `yarn deploy:test` is used. -async fn build_subgraph(dir: &str, deploy_cmd: Option<&str>) -> DeploymentHash { - build_subgraph_with_yarn_cmd(dir, deploy_cmd.unwrap_or("deploy:test")).await -} - -async fn build_subgraph_with_yarn_cmd(dir: &str, yarn_cmd: &str) -> DeploymentHash { - build_subgraph_with_yarn_cmd_and_arg(dir, yarn_cmd, None).await -} - -async fn build_subgraph_with_yarn_cmd_and_arg( - dir: &str, - yarn_cmd: &str, - arg: Option<&str>, -) -> DeploymentHash { - // Test that IPFS is up. - ipfs::IpfsRpcClient::new(ipfs::ServerAddress::local_rpc_api(), &graph::log::discard()) - .await - .expect("Could not connect to IPFS, make sure it's running at port 5001"); - - // Make sure dependencies are present. - - run_cmd( - Command::new("yarn") - .arg("install") - .arg("--mutex") - .arg("file:.yarn-mutex") - .current_dir("./runner-tests/"), - ); - - // Run codegen. - run_cmd(Command::new("yarn").arg("codegen").current_dir(dir)); - - let mut args = vec![yarn_cmd]; - args.extend(arg); - - // Run `deploy` for the side effect of uploading to IPFS, the graph node url - // is fake and the actual deploy call is meant to fail. - let deploy_output = run_cmd( - Command::new("yarn") - .args(&args) - .env("IPFS_URI", "http://127.0.0.1:5001") - .env("GRAPH_NODE_ADMIN_URI", "http://localhost:0") - .current_dir(dir), - ); - - // Hack to extract deployment id from `graph deploy` output. - const ID_PREFIX: &str = "Build completed: "; - let Some(mut line) = deploy_output.lines().find(|line| line.contains(ID_PREFIX)) else { - panic!("No deployment id found, graph deploy probably had an error") - }; - if !line.starts_with(ID_PREFIX) { - line = &line[5..line.len() - 5]; // workaround for colored output - } - DeploymentHash::new(line.trim_start_matches(ID_PREFIX)).unwrap() -}