graph init: add subgraph composition (#1920)

* Fix subgraphs without abi field failing to build

* Fix graph init for composed subgraphs

* Add changeset

* Fix validation not working

* Support declared calls in manifest

* Lint fix

* Address review comments

* Dont allow adding new contracts when subgraph is a composed subgraph

* Allow init of subgraph datasource subgraphs without the interactive mode

* Reduce code duplication between subgraph datasource and normal data source

* prevent using --from-contract and --from-source-subgraph flags together

* cli: validate protocol and source subgraph relationship

* chore(dependencies): updated changesets for modified dependencies

* change flag name for source subgraph

* Refactor manifest validation util functions

* get start block from source manifest

* set fromSubgraph to be default value for graph init in interactive mode

* fix protocol flag validation

* Add init test for subgraphs

* Fix error message

* chore(dependencies): updated changesets for modified dependencies

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: YaroShkvorets <[email protected]>

Author: 3 people

.changeset/curly-buses-hang.md

@@ -0,0 +1,5 @@
+---
+'@graphprotocol/graph-cli': minor
+---
+
+Add support for subgraph datasource in `graph init`

packages/cli/src/commands/codegen.ts

@@ -43,7 +43,6 @@ export default class CodegenCommand extends Command {
       summary: 'IPFS node to use for fetching subgraph data.',
       char: 'i',
       default: DEFAULT_IPFS_URL,
-      hidden: true,
     }),
     'uncrashable-config': Flags.file({
       summary: 'Directory for uncrashable config.',

packages/cli/src/commands/init.ts

@@ -23,7 +23,13 @@ import EthereumABI from '../protocols/ethereum/abi.js';
 import Protocol, { ProtocolName } from '../protocols/index.js';
 import { abiEvents } from '../scaffold/schema.js';
 import Schema from '../schema.js';
-import { createIpfsClient, loadSubgraphSchemaFromIPFS } from '../utils.js';
+import {
+  createIpfsClient,
+  getMinStartBlock,
+  loadManifestYaml,
+  loadSubgraphSchemaFromIPFS,
+  validateSubgraphNetworkMatch,
+} from '../utils.js';
 import { validateContract } from '../validation/index.js';
 import AddCommand from './add.js';
 
@@ -54,6 +60,10 @@ export default class InitCommand extends Command {
       summary: 'Graph node for which to initialize.',
       char: 'g',
     }),
+    'from-subgraph': Flags.string({
+      description: 'Creates a scaffold based on an existing subgraph.',
+      exclusive: ['from-example', 'from-contract'],
+    }),
     'from-contract': Flags.string({
       description: 'Creates a scaffold based on an existing contract.',
       exclusive: ['from-example'],
@@ -88,7 +98,6 @@ export default class InitCommand extends Command {
       description: 'Block number to start indexing from.',
       // TODO: using a default sets the value and therefore requires --from-contract
       // default: '0',
-      dependsOn: ['from-contract'],
     }),
 
     abi: Flags.string({
@@ -110,7 +119,6 @@ export default class InitCommand extends Command {
       summary: 'IPFS node to use for fetching subgraph data.',
       char: 'i',
       default: DEFAULT_IPFS_URL,
-      hidden: true,
     }),
   };
 
@@ -127,6 +135,7 @@ export default class InitCommand extends Command {
       protocol,
       node: nodeFlag,
       'from-contract': fromContract,
+      'from-subgraph': fromSubgraph,
       'contract-name': contractName,
       'from-example': fromExample,
       'index-events': indexEvents,
@@ -141,11 +150,20 @@ export default class InitCommand extends Command {
 
     initDebugger('Flags: %O', flags);
 
+    if (startBlock && !(fromContract || fromSubgraph)) {
+      this.error('--start-block can only be used with --from-contract or --from-subgraph');
+    }
+
+    if (fromContract && fromSubgraph) {
+      this.error('Cannot use both --from-contract and --from-subgraph at the same time');
+    }
+
     if (skipGit) {
       this.warn(
         'The --skip-git flag will be removed in the next major version. By default we will stop initializing a Git repository.',
       );
     }
+
     if ((fromContract || spkgPath) && !network && !fromExample) {
       this.error('--network is required when using --from-contract or --spkg');
     }
@@ -199,16 +217,15 @@ export default class InitCommand extends Command {
     let abi!: EthereumABI;
 
     // If all parameters are provided from the command-line,
-    // go straight to creating the subgraph from an existing contract
-    if ((fromContract || spkgPath) && protocol && subgraphName && directory && network && node) {
-      const registry = await loadRegistry();
-      const contractService = new ContractService(registry);
-      const sourcifyContractInfo = await contractService.getFromSourcify(
-        EthereumABI,
-        network,
-        fromContract!,
-      );
-
+    // go straight to creating the subgraph from an existing contract or source subgraph
+    if (
+      (fromContract || spkgPath || fromSubgraph) &&
+      protocol &&
+      subgraphName &&
+      directory &&
+      network &&
+      node
+    ) {
       if (!protocolChoices.includes(protocol as ProtocolName)) {
         this.error(
           `Protocol '${protocol}' is not supported, choose from these options: ${protocolChoices.join(
@@ -220,7 +237,31 @@ export default class InitCommand extends Command {
 
       const protocolInstance = new Protocol(protocol as ProtocolName);
 
-      if (protocolInstance.hasABIs()) {
+      if (fromSubgraph && !protocolInstance.isComposedSubgraph()) {
+        this.error('--protocol can only be subgraph when using --from-subgraph');
+      }
+
+      if (
+        fromContract &&
+        (protocolInstance.isComposedSubgraph() || protocolInstance.isSubstreams())
+      ) {
+        this.error('--protocol cannot be subgraph or substreams when using --from-contract');
+      }
+
+      if (spkgPath && !protocolInstance.isSubstreams()) {
+        this.error('--protocol can only be substreams when using --spkg');
+      }
+
+      // Only fetch contract info and ABI for non-source-subgraph cases
+      if (!fromSubgraph && protocolInstance.hasABIs()) {
+        const registry = await loadRegistry();
+        const contractService = new ContractService(registry);
+        const sourcifyContractInfo = await contractService.getFromSourcify(
+          EthereumABI,
+          network,
+          fromContract!,
+        );
+
         const ABI = protocolInstance.getABI();
         if (abiPath) {
           try {
@@ -244,7 +285,7 @@ export default class InitCommand extends Command {
           protocolInstance,
           abi,
           directory,
-          source: fromContract!,
+          source: fromSubgraph || fromContract!,
           indexEvents,
           network,
           subgraphName,
@@ -288,7 +329,7 @@ export default class InitCommand extends Command {
         abi,
         abiPath,
         directory,
-        source: fromContract,
+        source: fromContract || fromSubgraph,
         indexEvents,
         fromExample,
         subgraphName,
@@ -534,7 +575,7 @@ async function processInitForm(
               value: 'contract',
             },
             { message: 'Substreams', name: 'substreams', value: 'substreams' },
-            // { message: 'Subgraph', name: 'subgraph', value: 'subgraph' },
+            { message: 'Subgraph', name: 'subgraph', value: 'subgraph' },
           ].filter(({ name }) => name),
         });
 
@@ -604,6 +645,30 @@ async function processInitForm(
       },
     });
 
+    promptManager.addStep({
+      type: 'input',
+      name: 'ipfs',
+      message: `IPFS node to use for fetching subgraph manifest`,
+      initial: ipfsUrl,
+      skip: () => !isComposedSubgraph,
+      validate: value => {
+        if (!value) {
+          return 'IPFS node URL cannot be empty';
+        }
+        try {
+          new URL(value);
+          return true;
+        } catch {
+          return 'Please enter a valid URL';
+        }
+      },
+      result: value => {
+        ipfsNode = value;
+        initDebugger.extend('processInitForm')('ipfs: %O', value);
+        return value;
+      },
+    });
+
     promptManager.addStep({
       type: 'input',
       name: 'source',
@@ -616,9 +681,16 @@ async function processInitForm(
         isSubstreams ||
         (!protocolInstance.hasContract() && !isComposedSubgraph),
       initial: initContract,
-      validate: async (value: string) => {
+      validate: async (value: string): Promise<string | boolean> => {
         if (isComposedSubgraph) {
-          return value.startsWith('Qm') ? true : 'Subgraph deployment ID must start with Qm';
+          const ipfs = createIpfsClient(ipfsNode);
+          const manifestYaml = await loadManifestYaml(ipfs, value);
+          const { valid, error } = validateSubgraphNetworkMatch(manifestYaml, network.id);
+          if (!valid) {
+            return error || 'Invalid subgraph network match';
+          }
+          startBlock ||= getMinStartBlock(manifestYaml)?.toString();
+          return true;
         }
         if (initFromExample !== undefined || !protocolInstance.hasContract()) {
           return true;
@@ -668,6 +740,7 @@ async function processInitForm(
         } else {
           abiFromApi = initAbi;
         }
+
         // If startBlock is not provided, try to fetch it from Etherscan API
         if (!initStartBlock) {
           startBlock = await retryWithPrompt(() =>
@@ -699,19 +772,6 @@ async function processInitForm(
       },
     });
 
-    promptManager.addStep({
-      type: 'input',
-      name: 'ipfs',
-      message: `IPFS node to use for fetching subgraph manifest`,
-      initial: ipfsUrl,
-      skip: () => !isComposedSubgraph,
-      result: value => {
-        ipfsNode = value;
-        initDebugger.extend('processInitForm')('ipfs: %O', value);
-        return value;
-      },
-    });
-
     promptManager.addStep({
       type: 'input',
       name: 'spkg',
@@ -751,7 +811,7 @@ async function processInitForm(
         isSubstreams ||
         !!initAbiPath ||
         isComposedSubgraph,
-      validate: async (value: string) => {
+      validate: async (value: string): Promise<string | boolean> => {
         if (
           initFromExample ||
           abiFromApi ||
@@ -1199,6 +1259,14 @@ async function initSubgraphFromContract(
         },
       });
 
+      // Validate network match first
+      const manifestYaml = await loadManifestYaml(ipfsClient, source);
+      const { valid, error } = validateSubgraphNetworkMatch(manifestYaml, network);
+      if (!valid) {
+        throw new Error(error || 'Invalid subgraph network match');
+      }
+
+      startBlock ||= getMinStartBlock(manifestYaml)?.toString();
       const schemaString = await loadSubgraphSchemaFromIPFS(ipfsClient, source);
       const schema = await Schema.loadFromString(schemaString);
       entities = schema.getEntityNames();
@@ -1208,8 +1276,9 @@ async function initSubgraphFromContract(
   }
 
   if (
-    !protocolInstance.isComposedSubgraph() &&
+    !isComposedSubgraph &&
     protocolInstance.hasABIs() &&
+    abi && // Add check for abi existence
     (abiEvents(abi).size === 0 ||
       // @ts-expect-error TODO: the abiEvents result is expected to be a List, how's it an array?
       abiEvents(abi).length === 0)
@@ -1224,6 +1293,12 @@ async function initSubgraphFromContract(
     `Failed to create subgraph scaffold`,
     `Warnings while creating subgraph scaffold`,
     async spinner => {
+      initDebugger('Generating scaffold with ABI:', abi);
+      initDebugger('ABI data:', abi?.data);
+      if (abi) {
+        initDebugger('ABI events:', abiEvents(abi));
+      }
+
       const scaffold = await generateScaffold(
         {
           protocolInstance,
@@ -1280,7 +1355,7 @@ async function initSubgraphFromContract(
       this.exit(1);
     }
 
-    while (addContract) {
+    while (addContract && !isComposedSubgraph) {
       addContract = await addAnotherContract
         .bind(this)({
           protocolInstance,

packages/cli/src/compiler/index.ts

@@ -508,6 +508,9 @@ export default class Compiler {
       `Failed to write compiled subgraph to ${displayDir}`,
       `Warnings while writing compiled subgraph to ${displayDir}`,
       async spinner => {
+        // Add debug log for initial subgraph state
+        compilerDebug('Initial subgraph state:', subgraph.toJS());
+
         // Copy schema and update its path
         subgraph = subgraph.updateIn(['schema', 'file'], schemaFile => {
           const schemaFilePath = path.resolve(this.sourceDir, schemaFile as string);
@@ -518,32 +521,49 @@ export default class Compiler {
           return path.relative(this.options.outputDir, targetFile);
         });
 
+        // Add debug log before processing data sources
+        compilerDebug('Processing dataSources:', subgraph.get('dataSources').toJS());
+
         // Copy data source files and update their paths
         subgraph = subgraph.update('dataSources', (dataSources: any[]) =>
           dataSources.map(dataSource => {
+            // Add debug log for each data source
+            compilerDebug('Processing dataSource:', dataSource.toJS());
+
             let updatedDataSource = dataSource;
 
             if (this.protocol.hasABIs()) {
-              updatedDataSource = updatedDataSource
-                // Write data source ABIs to the output directory
-                .updateIn(['mapping', 'abis'], (abis: any[]) =>
-                  abis.map((abi: any) =>
-                    abi.update('file', (abiFile: string) => {
-                      abiFile = path.resolve(this.sourceDir, abiFile);
-                      const abiData = this.ABI.load(abi.get('name'), abiFile);
-                      return path.relative(
-                        this.options.outputDir,
-                        this._writeSubgraphFile(
-                          abiFile,
-                          JSON.stringify(abiData.data.toJS(), null, 2),
-                          this.sourceDir,
-                          this.subgraphDir(this.options.outputDir, dataSource),
-                          spinner,
-                        ),
-                      );
-                    }),
-                  ),
+              // Add debug log for ABIs
+              compilerDebug(
+                'Processing ABIs for dataSource:',
+                dataSource.getIn(['mapping', 'abis'])?.toJS() || 'undefined',
+              );
+
+              updatedDataSource = updatedDataSource.updateIn(['mapping', 'abis'], (abis: any[]) => {
+                compilerDebug('ABIs value:', Array.isArray(abis) ? abis : 'undefined');
+
+                if (!abis) {
+                  compilerDebug('No ABIs found for dataSource');
+                  return immutable.List();
+                }
+
+                return abis.map((abi: any) =>
+                  abi.update('file', (abiFile: string) => {
+                    abiFile = path.resolve(this.sourceDir, abiFile);
+                    const abiData = this.ABI.load(abi.get('name'), abiFile);
+                    return path.relative(
+                      this.options.outputDir,
+                      this._writeSubgraphFile(
+                        abiFile,
+                        JSON.stringify(abiData.data.toJS(), null, 2),
+                        this.sourceDir,
+                        this.subgraphDir(this.options.outputDir, dataSource),
+                        spinner,
+                      ),
+                    );
+                  }),
                 );
+              });
             }
 
             if (protocol.name == 'substreams' || protocol.name == 'substreams/triggers') {

packages/cli/src/protocols/subgraph/manifest.graphql

@@ -63,6 +63,7 @@ type ContractABI {
 type EntityHandler {
   handler: String!
   entity: String!
+  calls: JSON
 }
 
 type Graft {