From 5d2e3c81dc328bc48fc0673a58a4c59583587917 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Mon, 3 Feb 2025 13:17:05 +0400 Subject: [PATCH] Subgraph Compositions: Validations --- graph/src/data/subgraph/mod.rs | 2 +- graph/src/data_source/mod.rs | 2 +- graph/src/data_source/subgraph.rs | 95 ++++++++++++++++++- .../tests/chain/ethereum/manifest.rs | 88 +++++++++++++++-- .../source-subgraph/subgraph.yaml | 2 +- .../subgraph-data-sources/subgraph.yaml | 2 +- .../subgraph-data-sources/abis/Contract.abi | 15 --- .../subgraph-data-sources/package.json | 13 --- .../subgraph-data-sources/schema.graphql | 6 -- .../subgraph-data-sources/src/mapping.ts | 35 ------- .../subgraph-data-sources/subgraph.yaml | 19 ---- tests/tests/runner_tests.rs | 55 +---------- 12 files changed, 180 insertions(+), 154 deletions(-) delete mode 100644 tests/runner-tests/subgraph-data-sources/abis/Contract.abi delete mode 100644 tests/runner-tests/subgraph-data-sources/package.json delete mode 100644 tests/runner-tests/subgraph-data-sources/schema.graphql delete mode 100644 tests/runner-tests/subgraph-data-sources/src/mapping.ts delete mode 100644 tests/runner-tests/subgraph-data-sources/subgraph.yaml diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index a1d1156dccf..0d1b0b5d05d 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -577,7 +577,7 @@ pub struct BaseSubgraphManifest { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct IndexerHints { - prune: Option, + pub prune: Option, } impl IndexerHints { diff --git a/graph/src/data_source/mod.rs b/graph/src/data_source/mod.rs index 751b71837e7..4c56e99ea9b 100644 --- a/graph/src/data_source/mod.rs +++ b/graph/src/data_source/mod.rs @@ -339,7 +339,7 @@ impl UnresolvedDataSource { .await .map(DataSource::Onchain), Self::Subgraph(unresolved) => unresolved - .resolve(resolver, logger, manifest_idx) + .resolve::(resolver, logger, manifest_idx) .await .map(DataSource::Subgraph), Self::Offchain(_unresolved) => { diff --git a/graph/src/data_source/subgraph.rs b/graph/src/data_source/subgraph.rs index 93f5d920825..54fd62d33bb 100644 --- a/graph/src/data_source/subgraph.rs +++ b/graph/src/data_source/subgraph.rs @@ -2,12 +2,16 @@ use crate::{ blockchain::{block_stream::EntitySourceOperation, Block, Blockchain}, components::{link_resolver::LinkResolver, store::BlockNumber}, data::{ - subgraph::{calls_host_fn, SPEC_VERSION_1_3_0}, + subgraph::{ + calls_host_fn, SubgraphManifest, UnresolvedSubgraphManifest, LATEST_VERSION, + SPEC_VERSION_1_3_0, + }, value::Word, }, data_source::{self, common::DeclaredCall}, ensure, prelude::{CheapClone, DataSourceContext, DeploymentHash, Link}, + schema::TypeKind, }; use anyhow::{anyhow, Context, Error, Result}; use futures03::{stream::FuturesOrdered, TryStreamExt}; @@ -211,8 +215,62 @@ pub struct UnresolvedMapping { } impl UnresolvedDataSource { + fn validate_mapping_entities( + mapping_entities: &[String], + source_manifest: &SubgraphManifest, + ) -> Result<(), Error> { + for entity in mapping_entities { + let type_kind = source_manifest.schema.kind_of_declared_type(&entity); + + match type_kind { + Some(TypeKind::Interface) => { + return Err(anyhow!( + "Entity {} is an interface and cannot be used as a mapping entity", + entity + )); + } + Some(TypeKind::Aggregation) => { + return Err(anyhow!( + "Entity {} is an aggregation and cannot be used as a mapping entity", + entity + )); + } + None => { + return Err(anyhow!("Entity {} not found in source manifest", entity)); + } + Some(TypeKind::Object) => {} + } + } + Ok(()) + } + + async fn resolve_source_manifest( + &self, + resolver: &Arc, + logger: &Logger, + ) -> Result>, Error> { + let source_raw = resolver + .cat(logger, &self.source.address.to_ipfs_link()) + .await + .context("Failed to resolve source subgraph manifest")?; + + let source_raw: serde_yaml::Mapping = serde_yaml::from_slice(&source_raw) + .context("Failed to parse source subgraph manifest as YAML")?; + + let deployment_hash = self.source.address.clone(); + + let source_manifest = UnresolvedSubgraphManifest::::parse(deployment_hash, source_raw) + .context("Failed to parse source subgraph manifest")?; + + source_manifest + .resolve(resolver, logger, LATEST_VERSION.clone()) + .await + .context("Failed to resolve source subgraph manifest") + .map(Arc::new) + } + #[allow(dead_code)] - pub(super) async fn resolve( + pub(super) async fn resolve( self, resolver: &Arc, logger: &Logger, @@ -224,7 +282,38 @@ impl UnresolvedDataSource { "source" => format_args!("{:?}", &self.source), ); - let kind = self.kind; + let kind = self.kind.clone(); + let source_manifest = self.resolve_source_manifest::(resolver, logger).await?; + let source_spec_version = &source_manifest.spec_version; + + if source_spec_version < &SPEC_VERSION_1_3_0 { + return Err(anyhow!( + "Source subgraph manifest spec version {} is not supported, minimum supported version is {}", + source_spec_version, + SPEC_VERSION_1_3_0 + )); + } + + let pruning_enabled = match source_manifest.indexer_hints.as_ref() { + None => false, + Some(hints) => hints.prune.is_some(), + }; + + if pruning_enabled { + return Err(anyhow!( + "Pruning is enabled for source subgraph, which is not supported" + )); + } + + let mapping_entities: Vec = self + .mapping + .handlers + .iter() + .map(|handler| handler.entity.clone()) + .collect(); + + Self::validate_mapping_entities(&mapping_entities, &source_manifest)?; + let source = Source { address: self.source.address, start_block: self.source.start_block, diff --git a/store/test-store/tests/chain/ethereum/manifest.rs b/store/test-store/tests/chain/ethereum/manifest.rs index c750adb7b72..d28a1207161 100644 --- a/store/test-store/tests/chain/ethereum/manifest.rs +++ b/store/test-store/tests/chain/ethereum/manifest.rs @@ -37,6 +37,32 @@ const GQL_SCHEMA: &str = r#" type TestEntity @entity { id: ID! } "#; const GQL_SCHEMA_FULLTEXT: &str = include_str!("full-text.graphql"); +const SOURCE_SUBGRAPH_MANIFEST: &str = " +dataSources: [] +schema: + file: + /: /ipfs/QmSourceSchema +specVersion: 1.3.0 +"; + +const SOURCE_SUBGRAPH_SCHEMA: &str = " +type TestEntity @entity { id: ID! } +type User @entity { id: ID! } +type Profile @entity { id: ID! } + +type TokenData @entity(timeseries: true) { + id: Int8! + timestamp: Timestamp! + amount: BigDecimal! +} + +type TokenStats @aggregation(intervals: [\"hour\", \"day\"], source: \"TokenData\") { + id: Int8! + timestamp: Timestamp! + totalAmount: BigDecimal! @aggregate(fn: \"sum\", arg: \"amount\") +} +"; + const MAPPING_WITH_IPFS_FUNC_WASM: &[u8] = include_bytes!("ipfs-on-ethereum-contracts.wasm"); const ABI: &str = "[{\"type\":\"function\", \"inputs\": [{\"name\": \"i\",\"type\": \"uint256\"}],\"name\":\"get\",\"outputs\": [{\"type\": \"address\",\"name\": \"o\"}]}]"; const FILE: &str = "{}"; @@ -83,10 +109,10 @@ impl LinkResolverTrait for TextResolver { } } -async fn resolve_manifest( +async fn try_resolve_manifest( text: &str, max_spec_version: Version, -) -> SubgraphManifest { +) -> Result, anyhow::Error> { let mut resolver = TextResolver::default(); let id = DeploymentHash::new("Qmmanifest").unwrap(); @@ -94,12 +120,22 @@ async fn resolve_manifest( resolver.add("/ipfs/Qmschema", &GQL_SCHEMA); resolver.add("/ipfs/Qmabi", &ABI); resolver.add("/ipfs/Qmmapping", &MAPPING_WITH_IPFS_FUNC_WASM); + resolver.add("/ipfs/QmSource", &SOURCE_SUBGRAPH_MANIFEST); + resolver.add("/ipfs/QmSource2", &SOURCE_SUBGRAPH_MANIFEST); + resolver.add("/ipfs/QmSourceSchema", &SOURCE_SUBGRAPH_SCHEMA); resolver.add(FILE_CID, &FILE); let resolver: Arc = Arc::new(resolver); - let raw = serde_yaml::from_str(text).unwrap(); - SubgraphManifest::resolve_from_raw(id, raw, &resolver, &LOGGER, max_spec_version) + let raw = serde_yaml::from_str(text)?; + Ok(SubgraphManifest::resolve_from_raw(id, raw, &resolver, &LOGGER, max_spec_version).await?) +} + +async fn resolve_manifest( + text: &str, + max_spec_version: Version, +) -> SubgraphManifest { + try_resolve_manifest(text, max_spec_version) .await .expect("Parsing simple manifest works") } @@ -184,7 +220,7 @@ dataSources: - Gravatar network: mainnet source: - address: 'QmSWWT2yrTFDZSL8tRyoHEVrcEKAUsY2hj2TMQDfdDZU8h' + address: 'QmSource' startBlock: 9562480 mapping: apiVersion: 0.0.6 @@ -195,7 +231,7 @@ dataSources: /: /ipfs/Qmmapping handlers: - handler: handleEntity - entity: User + entity: TestEntity specVersion: 1.3.0 "; @@ -214,6 +250,42 @@ specVersion: 1.3.0 } } +#[tokio::test] +async fn subgraph_ds_manifest_aggregations_should_fail() { + let yaml = " +schema: + file: + /: /ipfs/Qmschema +dataSources: + - name: SubgraphSource + kind: subgraph + entities: + - Gravatar + network: mainnet + source: + address: 'QmSource' + startBlock: 9562480 + mapping: + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - TestEntity + file: + /: /ipfs/Qmmapping + handlers: + - handler: handleEntity + entity: TokenStats # This is an aggregation and should fail +specVersion: 1.3.0 +"; + + let result = try_resolve_manifest(yaml, SPEC_VERSION_1_3_0).await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err + .to_string() + .contains("Entity TokenStats is an aggregation and cannot be used as a mapping entity")); +} + #[tokio::test] async fn graft_manifest() { const YAML: &str = " @@ -1506,7 +1578,7 @@ dataSources: - Gravatar network: mainnet source: - address: 'QmSWWT2yrTFDZSL8tRyoHEVrcEKAUsY2hj2TMQDfdDZU8h' + address: 'QmSource' startBlock: 9562480 mapping: apiVersion: 0.0.6 @@ -1537,6 +1609,8 @@ dataSources: resolver.add("/ipfs/Qmabi", &ABI); resolver.add("/ipfs/Qmschema", &GQL_SCHEMA); resolver.add("/ipfs/Qmmapping", &MAPPING_WITH_IPFS_FUNC_WASM); + resolver.add("/ipfs/QmSource", &SOURCE_SUBGRAPH_MANIFEST); + resolver.add("/ipfs/QmSourceSchema", &SOURCE_SUBGRAPH_SCHEMA); let resolver: Arc = Arc::new(resolver); diff --git a/tests/integration-tests/source-subgraph/subgraph.yaml b/tests/integration-tests/source-subgraph/subgraph.yaml index c531c44cb6c..22006e72dda 100644 --- a/tests/integration-tests/source-subgraph/subgraph.yaml +++ b/tests/integration-tests/source-subgraph/subgraph.yaml @@ -1,4 +1,4 @@ -specVersion: 0.0.8 +specVersion: 1.3.0 schema: file: ./schema.graphql dataSources: diff --git a/tests/integration-tests/subgraph-data-sources/subgraph.yaml b/tests/integration-tests/subgraph-data-sources/subgraph.yaml index cdcbcbabec7..70ba2972abd 100644 --- a/tests/integration-tests/subgraph-data-sources/subgraph.yaml +++ b/tests/integration-tests/subgraph-data-sources/subgraph.yaml @@ -6,7 +6,7 @@ dataSources: name: Contract network: test source: - address: 'Qmaqf8cRxfxbduZppSHKG9DMuX5JZPMoGuwGb2DQuo48sq' + address: 'QmaKaj4gCYo4TmGq27tgqwrsBLwNncHGvR6Q9e6wDBYo8M' startBlock: 0 mapping: apiVersion: 0.0.7 diff --git a/tests/runner-tests/subgraph-data-sources/abis/Contract.abi b/tests/runner-tests/subgraph-data-sources/abis/Contract.abi deleted file mode 100644 index 9d9f56b9263..00000000000 --- a/tests/runner-tests/subgraph-data-sources/abis/Contract.abi +++ /dev/null @@ -1,15 +0,0 @@ -[ - { - "anonymous": false, - "inputs": [ - { - "indexed": false, - "internalType": "string", - "name": "testCommand", - "type": "string" - } - ], - "name": "TestEvent", - "type": "event" - } -] diff --git a/tests/runner-tests/subgraph-data-sources/package.json b/tests/runner-tests/subgraph-data-sources/package.json deleted file mode 100644 index 87537290ad2..00000000000 --- a/tests/runner-tests/subgraph-data-sources/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "subgraph-data-sources", - "version": "0.1.0", - "scripts": { - "codegen": "graph codegen --skip-migrations", - "create:test": "graph create test/subgraph-data-sources --node $GRAPH_NODE_ADMIN_URI", - "deploy:test": "graph deploy test/subgraph-data-sources --version-label v0.0.1 --ipfs $IPFS_URI --node $GRAPH_NODE_ADMIN_URI" - }, - "devDependencies": { - "@graphprotocol/graph-cli": "0.79.0-alpha-20240711124603-49edf22", - "@graphprotocol/graph-ts": "0.31.0" - } -} diff --git a/tests/runner-tests/subgraph-data-sources/schema.graphql b/tests/runner-tests/subgraph-data-sources/schema.graphql deleted file mode 100644 index 6f97fa65c43..00000000000 --- a/tests/runner-tests/subgraph-data-sources/schema.graphql +++ /dev/null @@ -1,6 +0,0 @@ -type Data @entity { - id: ID! - foo: String - bar: Int - isTest: Boolean -} diff --git a/tests/runner-tests/subgraph-data-sources/src/mapping.ts b/tests/runner-tests/subgraph-data-sources/src/mapping.ts deleted file mode 100644 index cd5c1d4dcd1..00000000000 --- a/tests/runner-tests/subgraph-data-sources/src/mapping.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { Entity, log } from '@graphprotocol/graph-ts'; - -export const SubgraphEntityOpCreate: u32 = 0; -export const SubgraphEntityOpModify: u32 = 1; -export const SubgraphEntityOpDelete: u32 = 2; - -export class EntityTrigger { - constructor( - public entityOp: u32, - public entityType: string, - public entity: Entity, - public vid: i64, - ) {} -} - -export function handleBlock(content: EntityTrigger): void { - let stringContent = content.entity.getString('val'); - log.info('Content: {}', [stringContent]); - log.info('EntityOp: {}', [content.entityOp.toString()]); - - switch (content.entityOp) { - case SubgraphEntityOpCreate: { - log.info('Entity created: {}', [content.entityType]); - break - } - case SubgraphEntityOpModify: { - log.info('Entity modified: {}', [content.entityType]); - break; - } - case SubgraphEntityOpDelete: { - log.info('Entity deleted: {}', [content.entityType]); - break; - } - } -} diff --git a/tests/runner-tests/subgraph-data-sources/subgraph.yaml b/tests/runner-tests/subgraph-data-sources/subgraph.yaml deleted file mode 100644 index 01f719d069f..00000000000 --- a/tests/runner-tests/subgraph-data-sources/subgraph.yaml +++ /dev/null @@ -1,19 +0,0 @@ -specVersion: 1.3.0 -schema: - file: ./schema.graphql -dataSources: - - kind: subgraph - name: Contract - network: test - source: - address: 'QmRFXhvyvbm4z5Lo7z2mN9Ckmo623uuB2jJYbRmAXgYKXJ' - startBlock: 0 - mapping: - apiVersion: 0.0.7 - language: wasm/assemblyscript - entities: - - Gravatar - handlers: - - handler: handleBlock - entity: User - file: ./src/mapping.ts diff --git a/tests/tests/runner_tests.rs b/tests/tests/runner_tests.rs index 25c5cfe532b..ac645884b5d 100644 --- a/tests/tests/runner_tests.rs +++ b/tests/tests/runner_tests.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use std::time::Duration; use assert_json_diff::assert_json_eq; -use graph::blockchain::block_stream::{BlockWithTriggers, EntityOperationKind}; +use graph::blockchain::block_stream::BlockWithTriggers; use graph::blockchain::{Block, BlockPtr, Blockchain}; use graph::data::store::scalar::Bytes; use graph::data::subgraph::schema::{SubgraphError, SubgraphHealth}; @@ -19,12 +19,11 @@ use graph::object; use graph::prelude::ethabi::ethereum_types::H256; use graph::prelude::web3::types::Address; use graph::prelude::{ - hex, CheapClone, DeploymentHash, SubgraphAssignmentProvider, SubgraphName, SubgraphStore, Value, + hex, CheapClone, DeploymentHash, SubgraphAssignmentProvider, SubgraphName, SubgraphStore, }; -use graph::schema::InputSchema; use graph_tests::fixture::ethereum::{ chain, empty_block, generate_empty_blocks_for_range, genesis, push_test_command, push_test_log, - push_test_polling_trigger, push_test_subgraph_trigger, + push_test_polling_trigger, }; use graph_tests::fixture::substreams::chain as substreams_chain; @@ -1092,54 +1091,6 @@ async fn parse_data_source_context() { ); } -#[tokio::test] -async fn subgraph_data_sources() { - let RunnerTestRecipe { stores, test_info } = - RunnerTestRecipe::new("subgraph-data-sources", "subgraph-data-sources").await; - - let schema = InputSchema::parse_latest( - "type User @entity { id: String!, val: String! }", - DeploymentHash::new("test").unwrap(), - ) - .unwrap(); - - let entity = schema - .make_entity(vec![ - ("id".into(), Value::String("id".to_owned())), - ("val".into(), Value::String("DATA".to_owned())), - ]) - .unwrap(); - - let entity_type = schema.entity_type("User").unwrap(); - - let blocks = { - let block_0 = genesis(); - let mut block_1 = empty_block(block_0.ptr(), test_ptr(1)); - - push_test_subgraph_trigger( - &mut block_1, - DeploymentHash::new("QmRFXhvyvbm4z5Lo7z2mN9Ckmo623uuB2jJYbRmAXgYKXJ").unwrap(), - entity, - entity_type, - EntityOperationKind::Create, - 1, - ); - - let block_2 = empty_block(block_1.ptr(), test_ptr(2)); - vec![block_0, block_1, block_2] - }; - let stop_block = blocks.last().unwrap().block.ptr(); - let chain = chain(&test_info.test_name, blocks, &stores, None).await; - - let ctx = fixture::setup(&test_info, &stores, &chain, None, None).await; - let _ = ctx - .runner(stop_block) - .await - .run_for_test(true) - .await - .unwrap(); -} - #[tokio::test] async fn retry_create_ds() { let RunnerTestRecipe { stores, test_info } =