From ebc270b336fc462fa8e7b1b24033e34512fccfad Mon Sep 17 00:00:00 2001 From: JGiter Date: Tue, 10 Jan 2023 15:13:53 +0200 Subject: [PATCH] feat: add ipfs module --- .env.dev | 16 ++-- .gitignore | 3 +- README.md | 10 +- devops/dev/values.yaml | 2 +- e2e/app.e2e.spec.ts | 18 ++-- e2e/ipfs/ipfs.testSuite.ts | 112 +++++++++++++++++++++++ e2e/setup-ipfs.ts | 23 ----- e2e/setupIpfsCluster/docker-compose.yml | 116 ++++++++++++++++++++++++ e2e/setupIpfsCluster/index.ts | 53 +++++++++++ e2e/setupIpfsCluster/nginx.conf | 17 ++++ src/common/cid.pipe.ts | 9 ++ src/env-vars-validation-schema.ts | 15 +-- src/modules/auth/login.strategy.ts | 12 +++ src/modules/did/did.module.ts | 13 +-- src/modules/did/did.processor.ts | 1 - src/modules/did/did.service.spec.ts | 4 +- src/modules/did/did.service.ts | 12 +-- src/modules/ipfs/ipfs.controller.ts | 46 ++++++++++ src/modules/ipfs/ipfs.module.ts | 114 ++++++++++++++++++----- src/modules/ipfs/ipfs.service.ts | 83 ++++++++++++++++- src/modules/ipfs/ipfs.types.ts | 26 ++++-- src/modules/ipfs/pins.processor.ts | 36 ++++++++ 22 files changed, 635 insertions(+), 106 deletions(-) create mode 100644 e2e/ipfs/ipfs.testSuite.ts delete mode 100644 e2e/setup-ipfs.ts create mode 100644 e2e/setupIpfsCluster/docker-compose.yml create mode 100644 e2e/setupIpfsCluster/index.ts create mode 100644 e2e/setupIpfsCluster/nginx.conf create mode 100644 src/common/cid.pipe.ts create mode 100644 src/modules/ipfs/ipfs.controller.ts create mode 100644 src/modules/ipfs/pins.processor.ts diff --git a/.env.dev b/.env.dev index d8148885a..afc77a0a9 100644 --- a/.env.dev +++ b/.env.dev @@ -41,13 +41,15 @@ ASSETS_SYNC_HISTORY_INTERVAL_IN_HOURS=21 ASSETS_SYNC_ENABLED=true # IPFS -IPFS_CLUSTER_ROOT=$IPFS_CLUSTER_ROOT -IPFS_CLUSTER_USER=$IPFS_CLUSTER_USER -IPFS_CLUSTER_PASSWORD=$IPFS_CLUSTER_PASSWORD -IPFS_CLIENT_HOST= -IPFS_CLIENT_PORT= -IPFS_CLIENT_PROJECT_ID= -IPFS_CLIENT_PROJECT_SECRET= +IPFS_CLUSTER_ROOT_URL # Cluster can be deployed locally with e2e/setupIpfsCluster/index.ts +IPFS_CLUSTER_USER # Not needed locally +IPFS_CLUSTER_PASSWORD # Not needed locally +IPFS_CLIENT_URL # Can be public gateway https://ipfs.github.io/public-gateway-checker/, Inura gateway or IPFS cluster. When cluster used as IPFS gateway it supports only GET +IPFS_CLIENT_PROTO # These can be +IPFS_CLIENT_HOST # used instead of +IPFS_CLIENT_PORT # IPFS_CLIENT_URL +IPFS_CLIENT_USER # User of cluster or Id of Infura project +IPFS_CLIENT_PASSWORD # Password to cluster or secret of Infura project # INTERVALS DIDDOC_SYNC_INTERVAL_IN_HOURS=1 diff --git a/.gitignore b/.gitignore index 337bee7a9..d80db6eec 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ /dist /node_modules /src/ethers -.env # Logs logs @@ -20,6 +19,7 @@ ormlogs.log # Tests /coverage /.nyc_output +/e2e/setupIpfsCluster/ipfs0/ # IDEs and editors /.idea @@ -41,6 +41,7 @@ ormlogs.log # Using either dev or prod docker-compose file docker-compose.yml +!/e2e/setupIpfsCluster/docker-compose.yml db_dumps private.pem public.pem diff --git a/README.md b/README.md index a9f620515..cf03e6b0a 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,7 @@ $ cp .env.dev .env $ cp docker-compose.dev.yml docker-compose.yml ``` -Set the following values in your `.env`: - -```text -ENS_URL= -IPFS_CLIENT_HOST= -IPFS_CLIENT_PORT= -IPFS_CLIENT_PROJECT_ID= -IPFS_CLIENT_PROJECT_SECRET= -``` +Fill in configuration values in your `.env`. For reference look at `.env.dev` ### Production ```bash diff --git a/devops/dev/values.yaml b/devops/dev/values.yaml index f5dfb231d..a844ccb04 100644 --- a/devops/dev/values.yaml +++ b/devops/dev/values.yaml @@ -144,6 +144,6 @@ iam-cache-server-helm: STRATEGY_PRIVATE_KEY: AgBM4BIbZIvZ9ErUDLWmmgw0UMEGk4k9hO54/tos4Rf9bUeewlDyAOrnz+gcl8uXuwFiGazQYNia2JFOjCapKnVt0h3LUjvubN4G2cVoQQl9E2+BDzSb7Ucji2UsdleN1ScOwd6XOADdw7Pa1LUXdpf32rf5YADhdYLGDiYy6hqpJ0OPd+61s92HgtyKHioTSP3fJ4YyFqgXQ4wORRR15rdwJdvzSxUrzGnft/ceuB+ilGJc0qSu/jT+SifQ1411zZ6/vsvhfvSq7q4wXEibn2OWSy7MxxNdYafVC7PbyFgc9i6t8V2YKJzUigIGV53ds9GuMK8i8CeGVg2R3e4iHyuOuWv6LeNjbdUd6QeAynOFVos9rBWwS/L7g+Rv0aUUnD016GYSoPwLGPRwvpyhk3jRtJ94uj4FAkUwWGWKagTA+F0uZyIjq2emO4opE8J5FwbPJOA3SKxfo0yPxgQF7S7uTAGlslQdxSrBk9ICUb5FuMVZ32Bxv3fbJ4R3xIS6D8g9hfPG3/P+zpvTxMzPlOjn7bgXHpbV6qOtt47OjTb9A+ELEVnwF/hOY6efSNa/MYlDU0wjF8CGveKFmYkSo9QHnTx7ruGu1s18yXMQZGK47WEERFR3WS0JqB7iQKv2KhPlMJLQapbROLP3MEYGWpEExDdfQRksmvCpQvQhM/7rtBpjobktxKfP6FVsiCTiL0A8Lkbo+i48oVti3SK6vNF2kDBhy4b6uLWwztYNvz3XL1ZalmcWSwe6iMtscD8Vcjms4FVe3hBRPZbXK5/wOpMq ENS_URL: AgCaICh+WJ7pQ2MsRfznNhqFAqauOls7BMQeKb79yyrrFE9PbGRfVaFjL/R1HPzFbdfPzD2OT1kvcJDN/R6Jc4CH9+Vg1z6QTaknXhHsZtBiN+BRCmRwfyno0QFjsWTLVZ83NTSKP2EvwqU5h4/peKavCx7pSkDb31xqf0Mq9V9Kapqw7YvKjEjfvRzdbtAxBtLt3v2UoyOyCldnfXcnenDyHmSgBn6uKS6U8pK/5RlGlsegvCokzR20vw4l8bf60uR/36fX+5ttoYzF+BWI3FWgrry2S/EChv4umk0M8Nr8vYeNc40cLuVJPlxFTr/eHAb+uqKe/bhavJqAvzJFW714/QTNDDKyWqmZMVj6H6BSLSVHHmh6M0Cp2YOH07xecHCgPWqTWzJq1HCfMAuB9hH5wQYVxLhYoIze6XzD8AAzJEFUJ+n+TXmaxH7kuqhVJGbgTtXxbk+QvKeZmX36IcBOl6BxkUlm9VGvj91zCip9zC/HiBbl8zL6nTd6JpFYgJylaLBeOLvMjoyk25vfs+viehC9HbEtsgcLtNevSAzA1sCBFF1c2x4vkyi4yFI2OIp1+FY6xPBkaR5rqMlwiPXS8Al9lgTATHoZgr+tkdcjFwg2wEokmyYPTzOcLc82NU0bZeBclUCVEkxeWCHcNwBRW2heNyujRLWl35klbQPbFlQjQNNOnj0wpp1K1+KFetqjKm7s4Xqjk7UAiCjAX59VIkPpvvTc7iOFHEWqvfdLHciJDxYNuN7nPZk+bVtj/7KNSdCf2Q== IPFS_CLIENT_PROJECT_SECRET: AgCD3q5miwVO6GjALhdNMsbLQul35EQybREj9bV3IQJtC3ij7b65vpySapMAG1NBx6UMm8cqBRKySPxs85ICAevdtPXp9QtDLndTb/HiuEKC945Pw4R7g2V3RJaKgnKO/WZb2HRnaFq3ol63A4eIC2gcxAcU/M2Cinf3mQ8+Ha4D3ZwRTZBp+Vl7aD0vmDF8DrAaczBYs5qX0e4QYwCMlTviHGw/XNNKoQidl+jL+QGWE0aklOMrS9EBgcGtggYaBaNl5eqPAvhNllfsLIKAe4yt5QTQ2hCCih12/8HifOkAQAQw4l6xtzl3idfCSstwsMwMuAeBS/WUd40Y6tD1gR40AxSYiRC/FfMgDhW/ftqZD5wQq67YM7fD2BjJaM2UPDmzcyCdDqRGtHQNeCc9wf6iksGUv/4e68HRlrDQEw+DYqudeKcDs8jbQhJiecCDLGUr3Bb0Ww7Wi+tvs/rqtGrJzLyEJ5kM3XlEeuvA5qaGoJpEOa/0AnHLysfL3KnK+uA6t9KQTNUlRasmATIiHO0X0xSVJj0w30qgN6My83SbnIcbyno2rzlLktBxc+ZeB7pLb2ZvdFYYNgh05F0BXUMmqNXMAYerLpfZvAG4Y7JYcaPVkXnR16rG0vl0Qv+6/mVXbHV/Udfa8wjUrkbzl17mPGKZjmYdF8GoeGbjHsFM4JWceHsxbrX8d8m9NqPqYDRD420a6cqCFZUkpoq7RTa1hZUJ7HX5hlGyg75DuomDNQ== - IPFS_CLUSTER_ROOT: AgCK8khzwUAWcFtB9tO58fCOxUuE9AQqKFP1tO/0zHWLesuiz2N5cXwiySCeokJ3uo2lFfNdclBitV9Mew+mtbKWEJi1s9bXNN83P+5ctJLwXp75MuqROZJxrnigVrb/3BLwgjo8C1KYnH4BHjYvnsN32TmBneIbtdRqsivvs+KT5Xbe6SFj6RlUH7Du9FIfUq5dmf4Bc4hH6Bopwl0KxBZspSECb66ydQnJVJnImKBNpJwySVdwJA3cqYzH/7KE5NR1nByHpVh6Lv0OO6aikq/XGYrBz+pny1NFcLBNKt8qP280SYa8jSRxc+Gd+632LuDsYVnOJLVgzbJllwnTtGUki9EwNAOsIfU5TblJfDJbZ9nLGsispYPrh/DoZHvt4Y0DxljEZomUR+uj1h0x8WCtEX9eRCaVjXW86gcYQ8ssLjKnXTY1n5axQY0IZtc1QyxKp9LEVUGe/UJRTzzR7J+SJTCeeHI9+5ED8wJrAC2SfWuUHYPczB/z4BHRYRv4enO+AWpr3NaMFVD3uQsGRRq4vgw+z8h9O46T1T3uzrIKyL8WdUuzu9nEsC9MZmtlTYcUqAeZZq/rJ9bsgIrRmo1D/3N6aHWU9MNnv+ZPlJeYzRAIqmQEcrWm6pxhzWGzRUOwpD9FYl9xEgmDn5szk4+ihwa776qUXiRkp/6630Y0hkBsext0Jefpe2iXuLroIhSCCYKswWBmMtEpVYHuhet8tbmjmiNpkXo6ysjVSTZ1BvxuTXXS4+dLzgTR6g== + IPFS_CLUSTER_ROOT_URL: AgCK8khzwUAWcFtB9tO58fCOxUuE9AQqKFP1tO/0zHWLesuiz2N5cXwiySCeokJ3uo2lFfNdclBitV9Mew+mtbKWEJi1s9bXNN83P+5ctJLwXp75MuqROZJxrnigVrb/3BLwgjo8C1KYnH4BHjYvnsN32TmBneIbtdRqsivvs+KT5Xbe6SFj6RlUH7Du9FIfUq5dmf4Bc4hH6Bopwl0KxBZspSECb66ydQnJVJnImKBNpJwySVdwJA3cqYzH/7KE5NR1nByHpVh6Lv0OO6aikq/XGYrBz+pny1NFcLBNKt8qP280SYa8jSRxc+Gd+632LuDsYVnOJLVgzbJllwnTtGUki9EwNAOsIfU5TblJfDJbZ9nLGsispYPrh/DoZHvt4Y0DxljEZomUR+uj1h0x8WCtEX9eRCaVjXW86gcYQ8ssLjKnXTY1n5axQY0IZtc1QyxKp9LEVUGe/UJRTzzR7J+SJTCeeHI9+5ED8wJrAC2SfWuUHYPczB/z4BHRYRv4enO+AWpr3NaMFVD3uQsGRRq4vgw+z8h9O46T1T3uzrIKyL8WdUuzu9nEsC9MZmtlTYcUqAeZZq/rJ9bsgIrRmo1D/3N6aHWU9MNnv+ZPlJeYzRAIqmQEcrWm6pxhzWGzRUOwpD9FYl9xEgmDn5szk4+ihwa776qUXiRkp/6630Y0hkBsext0Jefpe2iXuLroIhSCCYKswWBmMtEpVYHuhet8tbmjmiNpkXo6ysjVSTZ1BvxuTXXS4+dLzgTR6g== IPFS_CLUSTER_USER: AgBwOgRhIkI20NFyEWwNyEhHELvOVLajzvu+Q3LhfjIWbyVSfP18Hx33RpPc6KR7gx5gJI0UswESjzqxIIgv2FjSl9LHx8ZXa/go0iVq6y/hJBk6ersWD3a9nqiafMqo691C19CRHIAZqpUK8NIzgcGjRo9/gXu95MuOac3I1Qs1ILkTRsfwGAbQMIUup2CO0bbpsc582i3phj/OguTrI8i+vfoNOSoJRk6CNNHH9qzIb5U6A0fQkic94IkcGIi05rPmUoU3WFB3ebgs7iD1ZRNV5syMNNjvMC7YV9stYok5TbH1Fqnq+OVlu0bfDKnQ2o8SPx4+lDhKVrEnFKH0AeTNOXqM1ZARMaOeWSell4MQAcnSggAXn3SexSy0D1xNy6MrH7uydqO8r8y52BXRVMc0lWkuxcTFmzjPuxG9IEHzGUQ55Rh39FI5/rizEXWV5VYHTcnl+IAa7LqqzfN4HC0FzxYgQZ4ZBsWX0VhL5ZqJyh1Yo856u1vxnPCCaHGlFyAHKueFxh8iMv0F6LxiSXgP5sJyQ8py7piisDbEJUTXsLWwqmhs/5T1eaOiH3cPUcCJZsY8Z362vwguzz9TAXzDRDhSddAEhbZ5NA7aBPxSo3YKmLLXpDvObW82TelZhbRW5epK0xwPNWVmjnOdcm4V6fj76+aiUPP8Iu74ImFq4EEtEpwB30BI9y2iJI2jdxL21n17bZYke3WatAcCUg== IPFS_CLUSTER_PASSWORD: AgAhHW1lN0qi1JitHn10qC4fF8vdh2J2mBbA9MVs5vMHHSdOhOX/s+JAtFAMVplIqaxBUjM4VsWFDqs4LZLN3I06pTDe4srDenOCvdlT2Pcupo6a/ftlVQ/MidOA68Qz5i0HFBlirOG9ZtdfdTEq41CqgvaP73oXbG7pkgkQqiydVglvKGmWOr8QWwqVLYqI1Jlrmz9OKcyC6AKaix7+FToVw2IJqH4dp73ciuQ5qwlOIOHIcsuGXgnvHaGYcbcOt1FYjzF/NTaKq3q69WvffM6UMCQq1m+RHBVpTAMQInGe2nfLnijAUShWJoozgLbGgU9knStwEQAVDPr2Shp+bjFhAVip3F7ZUQ4GmCkKQPVNccMsgHUib/157kvy2ET6OLVDppG6v5QIrXP5JM/8AIYz8zpwvuqkeumVqR94VRsI8Ryy+vSVjxYsEDuyG892cQ4wMf+dl7SOJINJIFxL0vd5f761lu4aUz0G5aQsbzzSafPlGgEQitm35kAoCbRyVX/2pquXiRsiigdxa7Nl+OJBm/EoNbbISi88eqWlbEOVq5EA1Tu0HC6FU0AW3Xg7G8vGrbHKrXxX5oxTRzNB6crBniAgRAdrPOc/G1ORZhkZQD6wEPUJ9Ny6GPikR9MlZ4v5sMFboapZqGD9IIwxjP+FJlrThMBfXcsqHrqGJWYJx8j5WrJpLUmunI4FBE1NcyHCSQ7rHCBfM46bd/URq19q6XCh0w== diff --git a/e2e/app.e2e.spec.ts b/e2e/app.e2e.spec.ts index 1869fcd1d..09d69e4d7 100644 --- a/e2e/app.e2e.spec.ts +++ b/e2e/app.e2e.spec.ts @@ -8,11 +8,13 @@ import { appConfig } from '../src/common/test.utils'; import { authTestSuite } from './auth'; import { claimTestSuite } from './claim'; import { statusList2021TestSuite } from './status-list'; -import { shutDownIpfsDaemon, spawnIpfsDaemon } from './setup-ipfs'; +import { shutdownIpfsCluster, spawnIpfsCluster } from './setupIpfsCluster'; import { EthereumDIDRegistry } from '../src/ethers/EthereumDIDRegistry'; import { EthereumDIDRegistry__factory } from '../src/ethers/factories/EthereumDIDRegistry__factory'; import { didModuleTestSuite } from './did/did-service'; import { Provider } from '../src/common/provider'; +import { ipfsModuleTestSuite } from './ipfs/ipfs.testSuite'; +import { ChildProcess } from 'child_process'; export let app: INestApplication; @@ -23,6 +25,7 @@ describe('iam-cache-server E2E tests', () => { let didRegistry: EthereumDIDRegistry; network.config.chainId = 73799; let consoleLogSpy: jest.SpyInstance; + let cluster: ChildProcess; async function deployDidRegistry() { const didRegistry = await new EthereumDIDRegistry__factory() @@ -36,17 +39,17 @@ describe('iam-cache-server E2E tests', () => { didRegistry = await loadFixture(deployDidRegistry); process.env.DID_REGISTRY_ADDRESS = didRegistry.address; - process.env.IPFS_CLUSTER_ROOT = 'http://localhost:8080'; - process.env.IPFS_CLUSTER_USER = 'not-required-locally'; - process.env.IPFS_CLUSTER_PASSWORD = 'not-required-locally'; + + cluster = await spawnIpfsCluster(); + process.env.IPFS_CLUSTER_ROOT_URL = 'http://localhost:8080'; + + process.env.IPFS_CLIENT_URL = 'http://mocked'; // CID resolved incorrectly through gateway exposed on cluster. TODO: instead of gateway try to expose IPFS API // have to import dynamically to have opportunity to deploy DID registry before environment configuration validation const { AppModule } = await import('../src/app.module'); const testingModule = await Test.createTestingModule({ imports: [AppModule], }) - .overrideProvider('IPFSClientConfig') - .useValue(await spawnIpfsDaemon()) .overrideProvider(Provider) .useValue(provider) .compile(); @@ -60,7 +63,7 @@ describe('iam-cache-server E2E tests', () => { expect.stringMatching(/^error \[.+\] : .+/) ); await app.close(); - await shutDownIpfsDaemon(); + shutdownIpfsCluster(cluster); }, 60_000); // 1min describe('Modules v1', () => { @@ -68,5 +71,6 @@ describe('iam-cache-server E2E tests', () => { describe('Claim module', claimTestSuite); describe('StatusList2021 module', statusList2021TestSuite); describe('Did module', didModuleTestSuite); + describe('Ipfs module', ipfsModuleTestSuite); }); }); diff --git a/e2e/ipfs/ipfs.testSuite.ts b/e2e/ipfs/ipfs.testSuite.ts new file mode 100644 index 000000000..88e41bc67 --- /dev/null +++ b/e2e/ipfs/ipfs.testSuite.ts @@ -0,0 +1,112 @@ +import { HttpStatus } from '@nestjs/common'; +import { getQueueToken } from '@nestjs/bull'; +import { Queue } from 'bull'; +import request from 'supertest'; +import { Connection, EntityManager, QueryRunner } from 'typeorm'; +import { DidStore as DidStoreCluster } from 'didStoreCluster'; +import { DidStore as DidStoreGateway } from 'didStoreGateway'; +import { app } from '../app.e2e.spec'; +import { randomUser } from '../utils'; + +export const ipfsModuleTestSuite = () => { + let queryRunner: QueryRunner; + let didStoreCluster: DidStoreCluster; + let didStoreInfura: DidStoreGateway; + let pinsQueue: Queue; + const notPinned = { claimType: 'hello world notpinned' }; + const notPinnedCid = + 'bafkreigj4mi6cnegeh6hh6rxdjb63l4d7dowjcxxeyakgnijvoues3svii'; + const notPersistedCid = + 'bafkreih5pe7r3ucfdebiu7wjx3jr35qpbxqkxm5eneb2s2zu6t7yfqllci'; // CID of { claimType: 'hello world not persisted bafybeicg2rebjoofv4kbyovkw7af3rpiitvnl6i7ckcywaq6xjcxnc2mby' } + + beforeEach(async () => { + jest.restoreAllMocks(); + + didStoreCluster = app.get(DidStoreCluster); + didStoreInfura = app.get(DidStoreGateway); + pinsQueue = app.get(getQueueToken('pins')); + + const manager = app.get(EntityManager); + const dbConnection = app.get(Connection); + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + queryRunner = manager.queryRunner = + dbConnection.createQueryRunner('master'); + await queryRunner.startTransaction(); + }); + + afterEach(async () => { + await queryRunner.rollbackTransaction(); + await queryRunner.release(); + }); + + it('should be able to post claim', async () => { + const claimData = { + claimType: 'claim type', + claimTypeVersion: 1, + fields: [], + issuerFields: [], + }; + const requester = await randomUser(); + + const { text: cid } = await request(app.getHttpServer()) + .post(`/v1/ipfs/`) + .set('Cookie', requester.cookies) + .send(claimData) + .expect(HttpStatus.CREATED); + + const get = jest.spyOn(didStoreCluster, 'get'); + const { text: stored } = await request(app.getHttpServer()) + .get(`/v1/ipfs/${cid}`) + .set('Cookie', requester.cookies) + .expect(HttpStatus.OK); + expect(get).toBeCalledTimes(1); + + expect(JSON.parse(stored)).toStrictEqual(claimData); + }); + + it('should return 404 if claim was not persisted in IPFS', async () => { + const didStoreInfuraGet = jest.spyOn(didStoreInfura, 'get'); + const requester = await randomUser(); + + const cid = notPersistedCid; + didStoreInfuraGet.mockRejectedValueOnce({ response: { status: 504 } }); + await request(app.getHttpServer()) + .get(`/v1/ipfs/${cid}`) + .set('Cookie', requester.cookies) + .expect(HttpStatus.NOT_FOUND); + }); + + it('claim persisted in IPFS should be pinned in cluster', async () => { + const didStoreClusterGet = jest.spyOn(didStoreCluster, 'get'); + const didStoreInfuraGet = jest.spyOn(didStoreInfura, 'get'); + + const requester = await randomUser(); + + const claim = JSON.stringify(notPinned); + const cid = notPinnedCid; + + const claimPinned = new Promise((resolve) => { + pinsQueue.on('completed', () => { + resolve(); + }); + }); + didStoreInfuraGet.mockResolvedValueOnce(claim); + await request(app.getHttpServer()) + .get(`/v1/ipfs/${cid}`) + .set('Cookie', requester.cookies); + + await claimPinned; + + expect(didStoreClusterGet).toBeCalledTimes(0); + expect(didStoreInfuraGet).toBeCalledTimes(1); + + await request(app.getHttpServer()) + .get(`/v1/ipfs/${cid}`) + .set('Cookie', requester.cookies); + + expect(didStoreClusterGet).toBeCalledTimes(1); + expect(didStoreInfuraGet).toBeCalledTimes(1); + }); +}; diff --git a/e2e/setup-ipfs.ts b/e2e/setup-ipfs.ts deleted file mode 100644 index a7f807f51..000000000 --- a/e2e/setup-ipfs.ts +++ /dev/null @@ -1,23 +0,0 @@ -import Ctl from 'ipfsd-ctl'; -import path from 'path'; -import ipfsHttpModule from 'ipfs-http-client'; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -let ipfsd: any; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export async function spawnIpfsDaemon(): Promise { - const ipfsBin = path.resolve(__dirname, '../', 'node_modules/.bin', 'jsipfs'); - ipfsd = await Ctl.createController({ - type: 'js', - disposable: true, - test: true, - ipfsBin, - ipfsHttpModule, - }); - return ipfsd.apiAddr; -} - -export async function shutDownIpfsDaemon(): Promise { - return ipfsd && ipfsd.stop(); -} diff --git a/e2e/setupIpfsCluster/docker-compose.yml b/e2e/setupIpfsCluster/docker-compose.yml new file mode 100644 index 000000000..53e06dec7 --- /dev/null +++ b/e2e/setupIpfsCluster/docker-compose.yml @@ -0,0 +1,116 @@ +# This configuration is based on https://github.com/ipfs-cluster/ipfs-cluster/blob/master/docker-compose.yml + +version: '3.4' + +services: + cluster_proxy: + container_name: cluster_proxy + image: nginx:alpine + ports: + - 8080:8080 + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + depends_on: + - cluster0 + - ipfs0 + + ################################################################################## + ## Cluster PEER 0 ################################################################ + ################################################################################## + + ipfs0: + container_name: ipfs0 + image: ipfs/go-ipfs:latest + expose: + - '8080' + # ports: + # - "4001:4001" # ipfs swarm - expose if needed/wanted + # - "5001:5001" # ipfs api - expose if needed/wanted + # - "8080:8080" # ipfs gateway - expose if needed/wanted + volumes: + - ipfs0:/data/ipfs + + cluster0: + container_name: cluster0 + image: ipfs/ipfs-cluster:latest + depends_on: + - ipfs0 + environment: + CLUSTER_PEERNAME: cluster0 + CLUSTER_SECRET: ${CLUSTER_SECRET} # From shell variable if set + CLUSTER_IPFSHTTP_NODEMULTIADDRESS: /dns4/ipfs0/tcp/5001 + CLUSTER_CRDT_TRUSTEDPEERS: '*' # Trust all peers in Cluster + CLUSTER_RESTAPI_HTTPLISTENMULTIADDRESS: /ip4/0.0.0.0/tcp/9094 # Expose API + CLUSTER_MONITORPINGINTERVAL: 2s # Speed up peer discovery + expose: + - '9094' + # ports: + # Open API port (allows ipfs-cluster-ctl usage on host) + # - "127.0.0.1:9094:9094" + # The cluster swarm port would need to be exposed if this container + # was to connect to cluster peers on other hosts. + # But this is just a testing cluster. + # - "9095:9095" # Cluster IPFS Proxy endpoint + # - "9096:9096" # Cluster swarm endpoint + volumes: + - cluster0:/data/ipfs-cluster + + ################################################################################## + ## Cluster PEER 1 ################################################################ + ################################################################################## + + # See Cluster PEER 0 for comments (all removed here and below) + ipfs1: + container_name: ipfs1 + image: ipfs/go-ipfs:latest + volumes: + - ipfs1:/data/ipfs + + cluster1: + container_name: cluster1 + image: ipfs/ipfs-cluster:latest + depends_on: + - ipfs1 + environment: + CLUSTER_PEERNAME: cluster1 + CLUSTER_SECRET: ${CLUSTER_SECRET} + CLUSTER_IPFSHTTP_NODEMULTIADDRESS: /dns4/ipfs1/tcp/5001 + CLUSTER_CRDT_TRUSTEDPEERS: '*' + CLUSTER_MONITORPINGINTERVAL: 2s # Speed up peer discovery + volumes: + - cluster1:/data/ipfs-cluster + + ################################################################################## + ## Cluster PEER 2 ################################################################ + ################################################################################## + + # See Cluster PEER 0 for comments (all removed here and below) + ipfs2: + container_name: ipfs2 + image: ipfs/go-ipfs:latest + volumes: + - ipfs2:/data/ipfs + + cluster2: + container_name: cluster2 + image: ipfs/ipfs-cluster:latest + depends_on: + - ipfs2 + environment: + CLUSTER_PEERNAME: cluster2 + CLUSTER_SECRET: ${CLUSTER_SECRET} + CLUSTER_IPFSHTTP_NODEMULTIADDRESS: /dns4/ipfs2/tcp/5001 + CLUSTER_CRDT_TRUSTEDPEERS: '*' + CLUSTER_MONITORPINGINTERVAL: 2s # Speed up peer discovery + volumes: + - cluster2:/data/ipfs-cluster + # For adding more peers, copy PEER 1 and rename things to ipfs2, cluster2. + # Keep bootstrapping to cluster0. + +volumes: + ipfs0: + ipfs1: + ipfs2: + cluster0: + cluster1: + cluster2: diff --git a/e2e/setupIpfsCluster/index.ts b/e2e/setupIpfsCluster/index.ts new file mode 100644 index 000000000..ab23ba9bb --- /dev/null +++ b/e2e/setupIpfsCluster/index.ts @@ -0,0 +1,53 @@ +import { execFile, execFileSync, ChildProcess } from 'child_process'; +import path from 'path'; +import waitOn from 'wait-on'; + +/** + * Spawn local ipfs cluster + * + * @returns uri of the cluster root + */ +export async function spawnIpfsCluster() { + shutdownNodes(); + + const cluster = execFile( + 'docker', + ['compose', 'up', '--force-recreate', 'cluster_proxy'], + { cwd: path.join(__dirname) }, + (error) => { + if (error) { + throw error; + } + } + ); + + try { + await waitOn({ + // https://ipfscluster.io/documentation/reference/api/ + resources: ['http-get://localhost:8080/id'], + delay: 5000, + // if your internet connection is slow try to increase timeout to be able to download all images + timeout: 30000, + }); + + return cluster; + } catch (error) { + shutdownIpfsCluster(cluster); + throw error; + } +} + +export function shutdownIpfsCluster(cluster?: ChildProcess) { + shutdownNodes(); + + if (cluster !== undefined) { + cluster.kill(); + } +} + +function shutdownNodes() { + execFileSync('docker', ['compose', 'down', '--volumes'], { + cwd: path.join(__dirname), + stdio: 'pipe', + }); +} diff --git a/e2e/setupIpfsCluster/nginx.conf b/e2e/setupIpfsCluster/nginx.conf new file mode 100644 index 000000000..e40c00731 --- /dev/null +++ b/e2e/setupIpfsCluster/nginx.conf @@ -0,0 +1,17 @@ +events { } + +http { + client_max_body_size 100M; + + server { + listen 8080; + + location / { + proxy_pass http://cluster0:9094; + } + + location /ipfs { + proxy_pass http://ipfs0:8080; + } + } +} \ No newline at end of file diff --git a/src/common/cid.pipe.ts b/src/common/cid.pipe.ts new file mode 100644 index 000000000..7cd8ec61f --- /dev/null +++ b/src/common/cid.pipe.ts @@ -0,0 +1,9 @@ +import { PipeTransform, Injectable } from '@nestjs/common'; +import { CID } from 'multiformats/cid'; + +@Injectable() +export class CIDPipe implements PipeTransform { + transform(cid: string) { + return CID.parse(cid); + } +} diff --git a/src/env-vars-validation-schema.ts b/src/env-vars-validation-schema.ts index a83655cc6..574a4051f 100644 --- a/src/env-vars-validation-schema.ts +++ b/src/env-vars-validation-schema.ts @@ -89,14 +89,15 @@ export const envVarsValidationSchema = Joi.object({ ASSETS_SYNC_HISTORY_INTERVAL_IN_HOURS: Joi.number().positive().required(), ASSETS_SYNC_ENABLED: Joi.boolean().required(), - IPFS_CLIENT_HOST: Joi.string().hostname(), + IPFS_CLIENT_URL: Joi.string().uri(), + IPFS_CLIENT_PROTO: Joi.string(), + IPFS_CLIENT_HOST: Joi.string().uri(), IPFS_CLIENT_PORT: Joi.number().port(), - IPFS_CLIENT_PROJECT_ID: Joi.string(), - IPFS_CLIENT_PROJECT_SECRET: Joi.string(), - IPFS_CLUSTER_ROOT: Joi.string(), + IPFS_CLIENT_USER: Joi.string(), + IPFS_CLIENT_PASSWORD: Joi.string(), + IPFS_CLUSTER_ROOT_URL: Joi.string().required(), IPFS_CLUSTER_USER: Joi.string(), IPFS_CLUSTER_PASSWORD: Joi.string(), - DID_SYNC_MODE_FULL: Joi.boolean().required(), DID_SYNC_ENABLED: Joi.boolean().required(), DIDDOC_SYNC_INTERVAL_IN_HOURS: Joi.number().positive().required(), @@ -127,4 +128,6 @@ export const envVarsValidationSchema = Joi.object({ STATUS_LIST_DOMAIN: Joi.string().uri().required(), DISABLE_GET_DIDS_BY_ROLE: Joi.bool().default(false), -}); +}) + .or('IPFS_CLIENT_URL', 'IPFS_CLIENT_HOST') + .with('IPFS_CLIENT_HOST', ['IPFS_CLIENT_PROTO', 'IPFS_CLIENT_PORT']); diff --git a/src/modules/auth/login.strategy.ts b/src/modules/auth/login.strategy.ts index e2b0aab2d..8485e1005 100644 --- a/src/modules/auth/login.strategy.ts +++ b/src/modules/auth/login.strategy.ts @@ -7,6 +7,7 @@ import { URL } from 'url'; import { RoleIssuerResolver } from '../claim/resolvers/issuer.resolver'; import { RoleRevokerResolver } from '../claim/resolvers/revoker.resolver'; import { RoleCredentialResolver } from '../claim/resolvers/credential.resolver'; +import { IpfsGatewayConfig, IPFSGatewayConfigToken } from '../ipfs/ipfs.types'; import { LoginStrategyOptions } from 'passport-did-auth/dist/lib/LoginStrategy'; @Injectable() @@ -23,6 +24,9 @@ export class AuthStrategy extends PassportStrategy(LoginStrategy, 'login') { new URL(configService.get('STRATEGY_CACHE_SERVER')).origin ).href; const loginStrategyOptions: LoginStrategyOptions = { + @Inject(IPFSGatewayConfigToken) ipfsConfig: IpfsGatewayConfig + ) { + let loginStrategyParams: Omit = { name: 'login', rpcUrl: configService.get('ENS_URL'), cacheServerUrl: configService.get('STRATEGY_CACHE_SERVER'), @@ -44,6 +48,14 @@ export class AuthStrategy extends PassportStrategy(LoginStrategy, 'login') { ); if (numberOfBlocksBack) { loginStrategyOptions.numberOfBlocksBack = numberOfBlocksBack; + ipfsUrl: ipfsConfig.url, + }; + const numBlocksBack = configService.get('STRATEGY_NUM_BLOCKS_BACK'); + if (numBlocksBack) { + loginStrategyParams = { + ...loginStrategyParams, + ...{ numberOfBlocksBack: parseInt(numBlocksBack) }, + }; } super(...loginStrategyParams); } diff --git a/src/modules/did/did.module.ts b/src/modules/did/did.module.ts index 242e57866..bc4243e4f 100644 --- a/src/modules/did/did.module.ts +++ b/src/modules/did/did.module.ts @@ -11,10 +11,6 @@ import { DIDResolver } from './did.resolver'; import { DIDService } from './did.service'; import { ethrReg } from '@ew-did-registry/did-ethr-resolver'; import { ConfigService } from '@nestjs/config'; -import { DidStore as DidStoreInfura } from 'didStoreInfura'; -import { IpfsConfig } from '../ipfs/ipfs.types'; -import { PIN_CLAIM_QUEUE_NAME, UPDATE_DOCUMENT_QUEUE_NAME } from './did.types'; -import { PinProcessor } from './pin.processor'; const RegistrySettingsProvider = { provide: 'RegistrySettings', @@ -45,14 +41,7 @@ const RegistrySettingsProvider = { DIDResolver, Provider, RegistrySettingsProvider, - { - provide: DidStoreInfura, - useFactory: (ipfsConfig: IpfsConfig) => { - return new DidStoreInfura(ipfsConfig); - }, - inject: [{ token: 'IPFSClientConfig', optional: false }], - }, ], - exports: [DIDService, RegistrySettingsProvider, DidStoreInfura], + exports: [DIDService, RegistrySettingsProvider], }) export class DIDModule {} diff --git a/src/modules/did/did.processor.ts b/src/modules/did/did.processor.ts index b49fdd9f9..f93c81bfc 100644 --- a/src/modules/did/did.processor.ts +++ b/src/modules/did/did.processor.ts @@ -10,7 +10,6 @@ import { import { ConfigService } from '@nestjs/config'; import { Job, Queue } from 'bull'; import { Logger } from '../logger/logger.service'; -import { DIDDocumentEntity } from './did.entity'; import { DIDService } from './did.service'; import { ADD_DID_DOC_JOB_NAME, diff --git a/src/modules/did/did.service.spec.ts b/src/modules/did/did.service.spec.ts index 4b1066812..63a4e1e6c 100644 --- a/src/modules/did/did.service.spec.ts +++ b/src/modules/did/did.service.spec.ts @@ -2,7 +2,6 @@ import { IDIDDocument } from '@ew-did-registry/did-resolver-interface'; import { addressOf, ethrReg } from '@ew-did-registry/did-ethr-resolver'; import { Methods, Chain } from '@ew-did-registry/did'; -import { DidStore as DidStoreInfura } from 'didStoreInfura'; import { getQueueToken } from '@nestjs/bull'; import { HttpService } from '@nestjs/axios'; import { ConfigService } from '@nestjs/config'; @@ -18,6 +17,7 @@ import { Logger } from '../logger/logger.service'; import { SentryTracingService } from '../sentry/sentry-tracing.service'; import { EthereumDIDRegistry } from '../../ethers/EthereumDIDRegistry'; import { PIN_CLAIM_QUEUE_NAME, UPDATE_DOCUMENT_QUEUE_NAME } from './did.types'; +import { IPFSService } from '../ipfs/ipfs.service'; const { formatBytes32String } = utils; @@ -119,7 +119,7 @@ describe('DidDocumentService', () => { }), inject: [ConfigService], }, - { provide: DidStoreInfura, useValue: MockObject }, + { provide: IPFSService, useValue: MockObject }, ], }).compile(); await module.init(); diff --git a/src/modules/did/did.service.ts b/src/modules/did/did.service.ts index 8c9351d24..64313fbc4 100644 --- a/src/modules/did/did.service.ts +++ b/src/modules/did/did.service.ts @@ -26,7 +26,6 @@ import { DidEventNames, RegistrySettings, } from '@ew-did-registry/did-resolver-interface'; -import { DidStore as DidStoreInfura } from 'didStoreInfura'; import { documentFromLogs, Resolver, @@ -64,7 +63,7 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { private readonly provider: Provider, private readonly sentryTracingService: SentryTracingService, @Inject('RegistrySettings') registrySettings: RegistrySettings, - private readonly didStore: DidStoreInfura + private readonly ipfsService: IPFSService ) { this.logger.setContext(DIDService.name); @@ -327,15 +326,12 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { * @param did DID of the document service endpoints */ public async resolveServiceEndpoints(did: string) { - if (!this.didStore) { - throw new Error(`resolveServiceEndpoints: DIDStore is undefined`); - } const { service } = await this.getById(did); return Promise.all( service .map(({ serviceEndpoint }) => serviceEndpoint) .filter((endpoint) => IPFSService.isCID(endpoint)) - .map((cid) => this.didStore.get(cid)) + .map((cid) => this.ipfsService.get(cid)) ); } @@ -424,7 +420,7 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { return logs; } - private resolveNotCachedClaims( + private async resolveNotCachedClaims( services: IServiceEndpoint[], cachedServices: IClaim[] = [] ): Promise { @@ -442,7 +438,7 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { return { serviceEndpoint, ...rest }; } - const token = await this.didStore.get(serviceEndpoint); + const token = await this.ipfsService.get(serviceEndpoint); if (isJWT(token)) { const decodedData = jwt.decode(token) as { diff --git a/src/modules/ipfs/ipfs.controller.ts b/src/modules/ipfs/ipfs.controller.ts new file mode 100644 index 000000000..fd2c953cb --- /dev/null +++ b/src/modules/ipfs/ipfs.controller.ts @@ -0,0 +1,46 @@ +import { + Body, + Controller, + Get, + Param, + Post, + UseInterceptors, +} from '@nestjs/common'; +import { ApiBody, ApiOperation, ApiParam, ApiTags } from '@nestjs/swagger'; +import { CID } from 'multiformats/cid'; +import { CIDPipe } from '../../common/cid.pipe'; +import { Auth } from '../auth/auth.decorator'; +import { SentryErrorInterceptor } from '../interceptors/sentry-error-interceptor'; +import { IPFSService } from './ipfs.service'; + +@Auth() +@UseInterceptors(SentryErrorInterceptor) +@Controller({ path: 'ipfs', version: '1' }) +export class IPFSController { + constructor(private ipfsService: IPFSService) {} + + @Get('/:cid') + @ApiTags('IPFS') + @ApiOperation({ + summary: 'Returns credential from IPFS Store', + description: 'Returns credential represented by service in DID document.', + }) + @ApiParam({ name: 'cid', type: 'string', required: true }) + public async get(@Param('cid', CIDPipe) cid: CID): Promise { + return this.ipfsService.get(cid.toString()); + } + + @Post() + @ApiTags('IPFS') + @ApiBody({ + type: 'string', + description: 'Stringified credential', + }) + @ApiOperation({ + summary: 'Saves credential in IPFS', + description: 'Saves credential on IPFS and returns its CID', + }) + public async save(@Body() credential: Record) { + return this.ipfsService.save(JSON.stringify(credential)); + } +} diff --git a/src/modules/ipfs/ipfs.module.ts b/src/modules/ipfs/ipfs.module.ts index 722e76af2..2c7b85012 100644 --- a/src/modules/ipfs/ipfs.module.ts +++ b/src/modules/ipfs/ipfs.module.ts @@ -1,38 +1,108 @@ +import { DidStore as DidStoreGateway } from 'didStoreGateway'; +import { DidStore as DidStoreCluster } from 'didStoreCluster'; import { Module, Global } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { IPFSController } from './ipfs.controller'; import { IPFSService } from './ipfs.service'; -import { IpfsConfig } from './ipfs.types'; +import { + IpfsClusterConfig, + IPFSClusterConfigToken, + IpfsGatewayConfig, + IPFSGatewayConfigToken, + PINS_QUEUE, +} from './ipfs.types'; +import { BullModule } from '@nestjs/bull'; +import { PinsProcessor } from './pins.processor'; -const IPFSClientConfigProvider = { - provide: 'IPFSClientConfig', - useFactory: (config: ConfigService): IpfsConfig => { - const IPFS_CLIENT_PORT = config.get('IPFS_CLIENT_PORT'); +const IPFSGatewayConfigProvider = { + provide: IPFSGatewayConfigToken, + useFactory: (config: ConfigService): IpfsGatewayConfig => { + const IPFS_CLIENT_URL = config.get('IPFS_CLIENT_URL'); + const IPFS_CLIENT_PROTO = config.get('IPFS_CLIENT_PROTO'); const IPFS_CLIENT_HOST = config.get('IPFS_CLIENT_HOST'); - const IPFS_CLIENT_PROJECT_SECRET = config.get( - 'IPFS_CLIENT_PROJECT_SECRET' - ); - const IPFS_CLIENT_PROJECT_ID = config.get('IPFS_CLIENT_PROJECT_ID'); - // https://community.infura.io/t/how-to-add-internet-content-from-a-url-using-ipfs-http-client/5188 - const authorization = - 'Basic ' + - Buffer.from( - `${IPFS_CLIENT_PROJECT_ID}:${IPFS_CLIENT_PROJECT_SECRET}` - ).toString('base64'); - return { + const IPFS_CLIENT_PORT = config.get('IPFS_CLIENT_PORT'); + const ipfsConfig: IpfsGatewayConfig = { + url: IPFS_CLIENT_URL, + protocol: IPFS_CLIENT_PROTO, host: IPFS_CLIENT_HOST, port: parseInt(IPFS_CLIENT_PORT), - protocol: 'https', - headers: { - authorization, - }, }; + const IPFS_CLIENT_PASSWORD = config.get('IPFS_CLIENT_PASSWORD'); + const IPFS_CLIENT_USER = config.get('IPFS_CLIENT_USER'); + // https://community.infura.io/t/how-to-add-internet-content-from-a-url-using-ipfs-http-client/5188 + if (IPFS_CLIENT_USER && IPFS_CLIENT_PASSWORD) { + ipfsConfig.headers = { + Authorization: + 'Basic ' + + Buffer.from(`${IPFS_CLIENT_USER}:${IPFS_CLIENT_PASSWORD}`).toString( + 'base64' + ), + // Authorization: 'Basic WHpOY1NoVGF1R1NTQkk6Y0dnMVZ2dld0dkI1QTF5UW84SE8=', + }; + } + return ipfsConfig; + }, + inject: [ConfigService], +}; + +const IPFSClusterConfigProvider = { + provide: IPFSClusterConfigToken, + useFactory: (config: ConfigService): IpfsClusterConfig => { + const IPFS_CLUSTER_ROOT_URL = config.get('IPFS_CLUSTER_ROOT_URL'); + const IPFS_CLUSTER_USER = config.get('IPFS_CLUSTER_USER'); + const IPFS_CLUSTER_PASSWORD = config.get('IPFS_CLUSTER_PASSWORD'); + let ipfsConfig: IpfsClusterConfig; + if (IPFS_CLUSTER_USER && IPFS_CLUSTER_PASSWORD) { + const headers = { + authorization: + 'Basic ' + + Buffer.from(`${IPFS_CLUSTER_USER}:${IPFS_CLUSTER_PASSWORD}`).toString( + 'base64' + ), + }; + ipfsConfig = [IPFS_CLUSTER_ROOT_URL, headers]; + } else { + ipfsConfig = [IPFS_CLUSTER_ROOT_URL]; + } + return ipfsConfig; }, inject: [ConfigService], }; @Global() @Module({ - providers: [IPFSClientConfigProvider, IPFSService], - exports: [IPFSClientConfigProvider, IPFSService], + imports: [ + BullModule.registerQueue({ + name: PINS_QUEUE, + }), + ], + providers: [ + IPFSClusterConfigProvider, + IPFSGatewayConfigProvider, + IPFSService, + { + provide: DidStoreCluster, + useFactory: (ipfsConfig: IpfsClusterConfig) => { + return new DidStoreCluster(...ipfsConfig); + }, + inject: [{ token: IPFSClusterConfigToken, optional: false }], + }, + { + provide: DidStoreGateway, + useFactory: (ipfsConfig: IpfsGatewayConfig) => { + return new DidStoreGateway(ipfsConfig); + }, + inject: [{ token: IPFSGatewayConfigToken, optional: false }], + }, + PinsProcessor, + ], + controllers: [IPFSController], + exports: [ + IPFSClusterConfigProvider, + IPFSGatewayConfigProvider, + IPFSService, + DidStoreCluster, + DidStoreGateway, + ], }) export class IPFSModule {} diff --git a/src/modules/ipfs/ipfs.service.ts b/src/modules/ipfs/ipfs.service.ts index 97884e1c7..78a9aa668 100644 --- a/src/modules/ipfs/ipfs.service.ts +++ b/src/modules/ipfs/ipfs.service.ts @@ -1,6 +1,44 @@ +import { DidStore as DidStoreCluster } from 'didStoreCluster'; +import { DidStore as DidStoreGateway } from 'didStoreGateway'; +import { + HttpException, + HttpStatus, + Injectable, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; import { CID } from 'multiformats/cid'; +import { InjectQueue } from '@nestjs/bull'; +import { Queue } from 'bull'; +import { inspect } from 'util'; +import { PINS_QUEUE, PIN_CLAIM } from './ipfs.types'; +import { Logger } from '../logger/logger.service'; + +@Injectable() +export class IPFSService implements OnModuleInit, OnModuleDestroy { + constructor( + private didStoreCluster: DidStoreCluster, + private didStoreGateway: DidStoreGateway, + @InjectQueue(PINS_QUEUE) private readonly pinsQueue: Queue, + private readonly logger: Logger + ) { + this.logger.setContext(IPFSService.name); + } + + async onModuleInit() { + const jobsCount = await this.pinsQueue.getJobCounts(); + this.logger.info( + `Service endpoints pinning jobs statuses ${inspect(jobsCount, { + depth: 3, + colors: true, + })}` + ); + } + + async onModuleDestroy() { + await this.pinsQueue.close(); + } -export class IPFSService { /** * Check if given value is a valid IPFS CID. * @@ -26,4 +64,47 @@ export class IPFSService { return false; } } + + /** + * Get claim from cluster. If claim isn't found tries to get from gateway + * + * @param cid Content identifier. + * @returns Stringified credential + */ + public async get(cid: string): Promise { + let claim: string; + if (await this.didStoreCluster.isPinned(cid)) { + this.logger.debug(`${cid} was pinned. Getting from cluster`); + claim = await this.didStoreCluster.get(cid); + } else { + this.logger.debug(`${cid} was not pinned. Getting from gateway`); + try { + claim = await this.didStoreGateway.get(cid); + await this.pinsQueue.add(PIN_CLAIM, JSON.stringify({ cid, claim })); + } catch (e) { + // 504 is the expected response code when IPFS gateway is unable to provide content within time limit + // https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#504-gateway-timeout + // In other words, this is the expected response code if the traversal of the DHT fails to find the content + if (e?.response?.status === 504 || e?.response?.status === 404) { + throw new HttpException( + `Claim ${cid} not found`, + HttpStatus.NOT_FOUND + ); + } else { + throw e; + } + } + } + return claim; + } + + /** + * Saves credential on cluster + * + * @param credential Credential being persisted + * @returns CID of the persisted credential + */ + public async save(credential: string): Promise { + return this.didStoreCluster.save(credential); + } } diff --git a/src/modules/ipfs/ipfs.types.ts b/src/modules/ipfs/ipfs.types.ts index 807c93793..02e038b79 100644 --- a/src/modules/ipfs/ipfs.types.ts +++ b/src/modules/ipfs/ipfs.types.ts @@ -1,7 +1,21 @@ -export interface IpfsConfig { - host: string; - port?: number; +import { DidStore as DidStoreCluster } from 'didStoreCluster'; + +export type IpfsClusterConfig = ConstructorParameters; + +// copied from https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-http-client#createoptions, because ipfs-http-client isnt' typed +export type IpfsGatewayConfig = { + url?: string; protocol?: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - headers?: Record; -} + host?: string; + port?: number; + path?: string; + headers?: { + Authorization?: string; + }; +}; + +export const IPFSGatewayConfigToken = Symbol.for('IPFSGatewayConfigToken'); +export const IPFSClusterConfigToken = Symbol.for('IPFSClusterConfigToken'); + +export const PIN_CLAIM = 'pinClaim'; +export const PINS_QUEUE = 'pins'; diff --git a/src/modules/ipfs/pins.processor.ts b/src/modules/ipfs/pins.processor.ts new file mode 100644 index 000000000..46fdd53f4 --- /dev/null +++ b/src/modules/ipfs/pins.processor.ts @@ -0,0 +1,36 @@ +import { OnQueueError, Process, Processor } from '@nestjs/bull'; +import { Job } from 'bull'; +import { DidStore as DidStoreCluster } from 'didStoreCluster'; +import { Logger } from '../logger/logger.service'; +import { PINS_QUEUE, PIN_CLAIM } from './ipfs.types'; + +@Processor(PINS_QUEUE) +export class PinsProcessor { + constructor( + private readonly didStoreCluster: DidStoreCluster, + private readonly logger: Logger + ) { + this.logger.setContext(PinsProcessor.name); + } + + @OnQueueError() + onError(error: Error) { + this.logger.error(error); + } + + @Process(PIN_CLAIM) + public async pinClaim({ data }: Job) { + const { cid: cidGateway, claim } = JSON.parse(data); + this.logger.debug(`Pinning ${claim}`); + try { + const cidCluster = await this.didStoreCluster.save(claim); + await this.didStoreCluster.pin(cidGateway); + this.logger.debug(`${cidGateway} saved on cluster as ${cidCluster}`); + if (claim !== (await this.didStoreCluster.get(cidGateway))) { + throw new Error('Cluster content is not resolved by gateway CID'); + } + } catch (e) { + this.logger.error(e.message); + } + } +}