Devops: Update docker compose + add CI (#2)

* devops: Fix docker-compose

* devops: Add basic Rust CI github actions

* docs: Add CLI section to README

* style: Cargo fmt

* fix: Fix test

* docs: Fix docstrings

* docs: Fix docstrings #2

* test: Disable some tests requiring running neo4j instance

.dockerignore

@@ -1,4 +1,4 @@
-target/
-data/
-ipfs-cache/
-docker/
+**/target/
+**/data/
+**/ipfs-cache/
+**/docker/

.github/workflows/ci.yaml

@@ -0,0 +1,90 @@
+# Inspired by https://github.com/joshka/github-workflows/blob/main/.github/workflows/rust-check.yml
+name: Lint & Test
+
+on:
+  pull_request:
+    branches:
+      - main
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+
+# ensure that the workflow is only triggered once per PR, subsequent pushes to the PR will cancel
+# and restart the workflow. See https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  fmt:
+    name: stable / fmt
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Rust stable
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt
+
+      - name: Run cargo fmt
+        run: cargo fmt -- --check
+
+  clippy:
+    name: stable / clippy
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Rust stable
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: clippy
+
+      - name: Run clippy action
+        uses: clechasseur/rs-clippy-check@v3
+        with:
+          args: --all-features
+
+  test:
+    name: stable / test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Rust stable
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Install nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: nextest
+
+      - name: Test with latest nextest release
+        uses: actions-rs/cargo@v1
+        with:
+          command: nextest
+          args: run --all-features
+
+  doc:
+    name: stable / doc
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Rust stable
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rust-docs
+
+      - name: Run cargo doc
+        run: cargo doc --no-deps --all-features
+        env:
+          RUSTDOCFLAGS: -D warnings

.gitignore

@@ -1,4 +1,5 @@
 **/target
 **/ipfs-cache
 **/data
-*.json
+*.json
+.env

README.md

@@ -1,53 +1,35 @@
-## Setup
+# GRC20 Neo4j Indexer
+This repo contains a GRC20 indexer that uses Neo4j to store triple data. 
+
+## Running the indexer
+❗Important: Both methods require the `SUBSTREAMS_ENDPOINT_URL` (and optionally `SUBSTREAMS_API_TOKEN` if using a substreams provider with authentication) environment variables to be set.
+
+### With `docker-compose`
+```bash
+cd docker/
+docker compose up
+```
+
+### Without docker
 ### 1. Start Neo4j
 ```bash
 docker run \
     --publish=7474:7474 --publish=7687:7687 \
-    --volume=data:/data \
+    --volume=./data:/data \
     --env=NEO4J_AUTH=none \
     neo4j
 ```
 
-### 2. Download root dump
+### 2. Compile and run the indexer
+In a separate terminal, run the following command:
 ```bash
-wget https://gateway.lighthouse.storage/ipfs/bafkreif4acly7y46hx7optzfxtehxotizgqjz5h5vszo7vtmzsnm4ktxjy
-```
-
-## Run
-Import
-```bash
-cargo run --bin import -- --reset-db --rollup \
-    --neo4j-uri neo4j://localhost:7687 \
-    --neo4j-user neo4j \
-    --neo4j-pass neo4j 
-```
-
-Indexer node
-```bash
-cargo run --bin kg-node -- --rollup --reset-db \
+cargo run --bin kg-node -- \
+    --rollup \
+    --reset-db \
     --neo4j-uri neo4j://localhost:7687 \
     --neo4j-user neo4j \
     --neo4j-pass neo4j
 ```
 
-Codegen
-```bash
-cargo run --package kg-cli -- \
-    --neo4j-uri neo4j://localhost:7687 \
-    --neo4j-user neo4j \
-    --neo4j-pass neo4j \
-    codegen
-```
-
-## Docker compose
-### 1. Start neo4j
-```bash
-cd docker/
-docker compose up neo4j
-```
-
-### 2. Start kg-node
-```bash
-cd docker/
-docker compose up kg-node
-```
+## GRC20 CLI
+Coming soon™️

core/src/system_ids.rs

@@ -11,23 +11,21 @@ pub const SPACE: &str = "362c1dbddc6444bba3c4652f38a642d7";
 pub const INDEXED_SPACE: &str = "306598522df542f69ad72921c33ad84b";
 pub const ATTRIBUTE: &str = "808a04ceb21c4d888ad12e240613e5ca";
 
-/**
- * Relations are a data model that enable us to create references between some
- * arbitrary id and a set of entity ids.
- *
- * They act similarly to Join Tables in a relational database, but are designed
- * around the graph-based nature of the Geo data model.
- *
- * Relations are themselves entities, so can store any metadata about the relation
- * as triples. Currently Relation entities cannot have their own relations. This is a
- * technical limitation to avoid infinitely creating recursive relations.
- *
- * ┌─────────────────────┐       ┌────────────────────┐      ┌──────────────────────┐
- * │                     │       │                    │      │                      │
- * │      Entity         │◄──────┤     Relation       │─────►│        Entity        │
- * │                     │       │                    │      │                      │
- * └─────────────────────┘       └────────────────────┘      └──────────────────────┘
- */
+// Relations are a data model that enable us to create references between some
+// arbitrary id and a set of entity ids.
+//
+// They act similarly to Join Tables in a relational database, but are designed
+// around the graph-based nature of the Geo data model.
+//
+// Relations are themselves entities, so can store any metadata about the relation
+// as triples. Currently Relation entities cannot have their own relations. This is a
+// technical limitation to avoid infinitely creating recursive relations.
+//
+// ┌─────────────────────┐       ┌────────────────────┐      ┌──────────────────────┐
+// │                     │       │                    │      │                      │
+// │      Entity         │◄──────┤     Relation       │─────►│        Entity        │
+// │                     │       │                    │      │                      │
+// └─────────────────────┘       └────────────────────┘      └──────────────────────┘
 
 /**
  * Relation type. This is the entity representing the Join between the
@@ -239,12 +237,10 @@ pub const PEOPLE_SPACE: &str = "0xb4476A42A66eC1356A58D300555169E17db6756c";
 
 pub const BROADER_SPACES: &str = "03aa11edd69a4d5ea0aea0f197614cfd";
 
-/**
- * Addresses for important contracts on our L3.
- *
- * Note: If you want to test deployments on a different network (e.g. local or Mumbai),
- * you can update these addresses to point to the correct contracts on that network.
- */
+// Addresses for important contracts on our L3.
+//
+// Note: If you want to test deployments on a different network (e.g. local or Mumbai),
+// you can update these addresses to point to the correct contracts on that network.
 
 pub const PROFILE_REGISTRY_ADDRESS: &str = "0xc066E89bF7669b905f869Cb936818b0fd0bc456d";
 pub const MEMBERSHIP_CONTRACT_ADDRESS: &str = "0x34a94160f4B0f86d932927DFfb326354dB279181";

node/docker/Dockerfile → docker/Dockerfile

@@ -4,20 +4,24 @@ WORKDIR /kg-node
 COPY . .
 RUN apt-get update && apt-get upgrade -y
 RUN apt-get install libssl-dev protobuf-compiler -y
-RUN cargo build --release
+RUN cargo build --release --bin kg-node
 
 FROM debian:bookworm-slim
 
 ENV neo4j_uri ""
 ENV neo4j_user ""
 ENV neo4j_pass ""
+ENV SUBSTREAMS_API_TOKEN ""
+ENV SUBSTREAMS_ENDPOINT_URL ""
 
 RUN apt-get update && apt-get upgrade -y
 RUN apt-get install -y libssl-dev
-COPY --from=builder /kg-node/target/release/main .
+COPY --from=builder /kg-node/target/release/kg-node .
+COPY --from=builder /kg-node/geo-substream.spkg .
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 
-CMD ./main \
+CMD ./kg-node \
+    --reset-db \
     --rollup \
     --neo4j-uri $neo4j_uri \
     --neo4j-user $neo4j_user \

node/docker/docker-compose.yaml → docker/docker-compose.yaml

@@ -9,22 +9,22 @@ services:
       - ./data/neo4j:/data
     environment:
       NEO4J_AUTH: none
-  neo4j_2:
-    image: neo4j:latest
-    ports:
-      - "7475:7474"
-      - "7688:7687"
-    volumes:
-      - ./data/neo4j_2:/data
-    environment:
-      NEO4J_AUTH: none
+    healthcheck:
+      test: wget http://localhost:7474 || exit 1
+      interval: 1s
+      timeout: 10s
+      retries: 20
+      start_period: 3s
   kg-node:
     build:
       context: ..
       dockerfile: docker/Dockerfile
     depends_on:
-      - neo4j
+      neo4j:
+        condition: service_healthy
     environment:
       neo4j_uri: neo4j://neo4j:7687
       neo4j_user: neo4j
       neo4j_pass: neo4j
+      SUBSTREAMS_API_TOKEN: ${SUBSTREAMS_API_TOKEN}
+      SUBSTREAMS_ENDPOINT_URL: ${SUBSTREAMS_ENDPOINT_URL}

ipfs/src/lib.rs

@@ -151,6 +151,6 @@ mod tests {
 
         let client = super::IpfsClient::from_url("https://gateway.lighthouse.storage/ipfs/");
 
-        let edit = client.get::<Edit>(ipfs_hash, false).await.unwrap();
+        let _ = client.get::<Edit>(ipfs_hash, false).await.unwrap();
     }
 }

node/src/main.rs

@@ -1,11 +1,12 @@
+use std::env;
+
 use anyhow::Error;
 use clap::{Args, Parser};
 use kg_node::{events::EventHandler, kg};
 use substreams_sink_rust::Sink;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
 
-const ENDPOINT_URL: &str = "https://geotest.substreams.pinax.network:443";
 const PKG_FILE: &str = "geo-substream.spkg";
 const MODULE_NAME: &str = "geo_out";
 
@@ -16,6 +17,9 @@ const STOP_BLOCK: u64 = 0;
 async fn main() -> Result<(), Error> {
     set_log_level();
     init_tracing();
+    let endpoint_url =
+        env::var("SUBSTREAMS_ENDPOINT_URL").expect("SUBSTREAMS_ENDPOINT_URL not set");
+
     let args = AppArgs::parse();
 
     let kg_client = kg::Client::new(
@@ -31,8 +35,14 @@ async fn main() -> Result<(), Error> {
 
     let sink = EventHandler::new(kg_client);
 
-    sink.run(ENDPOINT_URL, PKG_FILE, MODULE_NAME, START_BLOCK, STOP_BLOCK)
-        .await?;
+    sink.run(
+        &endpoint_url,
+        PKG_FILE,
+        MODULE_NAME,
+        START_BLOCK,
+        STOP_BLOCK,
+    )
+    .await?;
 
     Ok(())
 }