From afeb20d18e2d1df208322ce2519f869012049bb5 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 19 Oct 2021 14:22:14 +0200
Subject: [PATCH 01/11] wip(pubsub): use multibase in http rpc wire format

This only adds comments marking places that need changes
to restore interop with go-ipfs after changes from
https://github.com/ipfs/go-ipfs/pull/8183 land
---
 packages/ipfs-http-client/src/pubsub/ls.js            | 1 +
 packages/ipfs-http-client/src/pubsub/publish.js       | 1 +
 packages/ipfs-http-client/src/pubsub/subscribe.js     | 3 +++
 packages/ipfs-http-server/src/api/resources/pubsub.js | 5 +++++
 4 files changed, 10 insertions(+)

diff --git a/packages/ipfs-http-client/src/pubsub/ls.js b/packages/ipfs-http-client/src/pubsub/ls.js
index 920e986c64..49853273ab 100644
--- a/packages/ipfs-http-client/src/pubsub/ls.js
+++ b/packages/ipfs-http-client/src/pubsub/ls.js
@@ -17,6 +17,7 @@ export const createLs = configure(api => {
       headers: options.headers
     })).json()
 
+    // TODO: unwrap topic names from multibase
     return Strings || []
   }
   return ls
diff --git a/packages/ipfs-http-client/src/pubsub/publish.js b/packages/ipfs-http-client/src/pubsub/publish.js
index 5c51dca64e..ebf6b565e4 100644
--- a/packages/ipfs-http-client/src/pubsub/publish.js
+++ b/packages/ipfs-http-client/src/pubsub/publish.js
@@ -14,6 +14,7 @@ export const createPublish = configure(api => {
    * @type {PubsubAPI["publish"]}
    */
   async function publish (topic, data, options = {}) {
+    // TODO: wrap topic in multibase
     const searchParams = toUrlSearchParams({
       arg: topic,
       ...options
diff --git a/packages/ipfs-http-client/src/pubsub/subscribe.js b/packages/ipfs-http-client/src/pubsub/subscribe.js
index 7ae5b97a9e..621975a8a0 100644
--- a/packages/ipfs-http-client/src/pubsub/subscribe.js
+++ b/packages/ipfs-http-client/src/pubsub/subscribe.js
@@ -39,6 +39,7 @@ export const createSubscribe = (options, subsTracker) => {
       // is received. If this doesn't happen within 1 second assume success
       const ffWorkaround = setTimeout(() => done(), 1000)
 
+      // TODO: wrap topic in multibase
       // Do this async to not block Firefox
       api.post('pubsub/sub', {
         signal: options.signal,
@@ -94,6 +95,8 @@ async function readMessages (response, { onMessage, onEnd, onError }) {
           continue
         }
 
+        // TODO: multibase data, seqno and topics
+        // TODO: parse string and get peerid bytes using libp2p lib
         onMessage({
           from: uint8ArrayToString(uint8ArrayFromString(msg.from, 'base64pad'), 'base58btc'),
           data: uint8ArrayFromString(msg.data, 'base64pad'),
diff --git a/packages/ipfs-http-server/src/api/resources/pubsub.js b/packages/ipfs-http-server/src/api/resources/pubsub.js
index fa8caef8ed..18a140494a 100644
--- a/packages/ipfs-http-server/src/api/resources/pubsub.js
+++ b/packages/ipfs-http-server/src/api/resources/pubsub.js
@@ -56,6 +56,8 @@ export const subscribeResource = {
        * @type {import('ipfs-core-types/src/pubsub').MessageHandlerFn}
        */
       const handler = (msg) => {
+        // TODO: data, seqno and topicIDs in multibase
+        // TODO: from should use canonical toString from peerid libp2p lib
         output.push({
           from: uint8ArrayToString(uint8ArrayFromString(msg.from, 'base58btc'), 'base64pad'),
           data: uint8ArrayToString(msg.data, 'base64pad'),
@@ -156,6 +158,7 @@ export const publishResource = {
     } = request
 
     try {
+      // TODO: unwrap topic from multibase?
       await ipfs.pubsub.publish(topic, data, {
         signal,
         timeout
@@ -209,6 +212,7 @@ export const lsResource = {
       throw Boom.boomify(err, { message: 'Failed to list subscriptions' })
     }
 
+    // TODO: multibase topic names in Strings array
     return h.response({ Strings: subscriptions })
   }
 }
@@ -252,6 +256,7 @@ export const peersResource = {
 
     let peers
     try {
+      // TODO: unwrap topic from multibase
       peers = await ipfs.pubsub.peers(topic, {
         signal,
         timeout

From 1dcac76f56972fc3519526e93567e39d685033dd Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 19 Oct 2021 23:30:37 +0200
Subject: [PATCH 02/11] fix(pubsub): new wire format in http rpc

---
 packages/ipfs-cli/test/pubsub.spec.js         | 22 ++++-----
 .../src/lib/http-rpc-wire-format.js           | 41 ++++++++++++++++
 packages/ipfs-http-client/src/pubsub/ls.js    |  4 +-
 packages/ipfs-http-client/src/pubsub/peers.js |  3 +-
 .../ipfs-http-client/src/pubsub/publish.js    |  4 +-
 .../ipfs-http-client/src/pubsub/subscribe.js  | 16 +++---
 .../ipfs-http-client/test/utils/factory.js    |  2 +-
 .../src/api/resources/pubsub.js               | 49 ++++++++++++-------
 8 files changed, 97 insertions(+), 44 deletions(-)
 create mode 100644 packages/ipfs-http-client/src/lib/http-rpc-wire-format.js

diff --git a/packages/ipfs-cli/test/pubsub.spec.js b/packages/ipfs-cli/test/pubsub.spec.js
index 34c179cf4a..839aca18e2 100644
--- a/packages/ipfs-cli/test/pubsub.spec.js
+++ b/packages/ipfs-cli/test/pubsub.spec.js
@@ -67,7 +67,7 @@ describe('pubsub', () => {
       timeout: undefined
     }
 
-    it('should list toic peers', async () => {
+    it('should list topic peers', async () => {
       const subName = 'sub-name'
       const peer = 'peer-id'
 
@@ -79,7 +79,7 @@ describe('pubsub', () => {
       expect(out).to.equal(`${peer}\n`)
     })
 
-    it('should list toic peers with a timeout', async () => {
+    it('should list topic peers with a timeout', async () => {
       const subName = 'sub-name'
       const peer = 'peer-id'
 
@@ -101,19 +101,19 @@ describe('pubsub', () => {
     }
 
     it('should publish message', async () => {
-      const subName = 'sub-name'
-      const data = 'data'
+      const subName = 'sub-name-1'
+      const data = 'data\r\nfirst\nZażółć gęślą jaźń😇'
 
-      await cli(`pubsub pub ${subName} ${data}`, { ipfs })
+      await cli(`pubsub pub ${subName} "${data}"`, { ipfs })
 
       expect(ipfs.pubsub.publish.calledWith(subName, uint8ArrayFromString(data), defaultOptions)).to.be.true()
     })
 
     it('should publish message with timeout', async () => {
-      const subName = 'sub-name'
-      const data = 'data'
+      const subName = 'sub-name-2'
+      const data = 'data\r\nsecond\nZażółć gęślą jaźń😇'
 
-      await cli(`pubsub pub ${subName} ${data} --timeout=1s`, { ipfs })
+      await cli(`pubsub pub ${subName} "${data}" --timeout=1s`, { ipfs })
 
       expect(ipfs.pubsub.publish.calledWith(subName, uint8ArrayFromString(data), {
         ...defaultOptions,
@@ -128,9 +128,9 @@ describe('pubsub', () => {
     }
 
     it('should subscribe', async () => {
-      const subName = 'sub-name'
+      const subName = 'sub\nname'
 
-      await cli(`pubsub sub ${subName}`, { ipfs })
+      await cli(`pubsub sub "${subName}"`, { ipfs })
 
       expect(ipfs.pubsub.subscribe.calledWith(subName, sinon.match.func, defaultOptions)).to.be.true()
     })
@@ -138,7 +138,7 @@ describe('pubsub', () => {
     it('should subscribe with a timeout', async () => {
       const subName = 'sub-name'
 
-      await cli(`pubsub sub ${subName} --timeout=1s`, { ipfs })
+      await cli(`pubsub sub "${subName}" --timeout=1s`, { ipfs })
 
       expect(ipfs.pubsub.subscribe.calledWith(subName, sinon.match.func, {
         ...defaultOptions,
diff --git a/packages/ipfs-http-client/src/lib/http-rpc-wire-format.js b/packages/ipfs-http-client/src/lib/http-rpc-wire-format.js
new file mode 100644
index 0000000000..6e28747d1a
--- /dev/null
+++ b/packages/ipfs-http-client/src/lib/http-rpc-wire-format.js
@@ -0,0 +1,41 @@
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { base64url } from 'multiformats/bases/base64'
+
+/* HTTP RPC:
+ * - wraps binary data in multibase. base64url is used to avoid issues
+ *   when a binary data is passed as search param in URL.
+ *   Historical context: https://github.com/ipfs/go-ipfs/issues/7939
+ *   Multibase wrapping introduced in: https://github.com/ipfs/go-ipfs/pull/8183
+ */
+
+/**
+ * @param {Array<string>} strings
+ * @returns {Array<string>} strings
+ */
+const rpcArrayToTextArray = strings => {
+  if (Array.isArray(strings)) {
+    return strings.map(rpcToText)
+  }
+  return strings
+}
+
+/**
+ * @param {string} mb
+ * @returns {string}
+ */
+const rpcToText = mb => uint8ArrayToString(rpcToBytes(mb))
+
+/**
+ * @param {string} mb
+ * @returns {Uint8Array}
+ */
+const rpcToBytes = mb => base64url.decode(mb)
+
+/**
+ * @param {string} text
+ * @returns {string}
+ */
+const textToUrlSafeRpc = text => base64url.encode(uint8ArrayFromString(text))
+
+export { rpcArrayToTextArray, rpcToText, rpcToBytes, textToUrlSafeRpc }
diff --git a/packages/ipfs-http-client/src/pubsub/ls.js b/packages/ipfs-http-client/src/pubsub/ls.js
index 49853273ab..94fb4227a6 100644
--- a/packages/ipfs-http-client/src/pubsub/ls.js
+++ b/packages/ipfs-http-client/src/pubsub/ls.js
@@ -1,5 +1,6 @@
 import { configure } from '../lib/configure.js'
 import { toUrlSearchParams } from '../lib/to-url-search-params.js'
+import { rpcArrayToTextArray } from '../lib/http-rpc-wire-format.js'
 
 /**
  * @typedef {import('../types').HTTPClientExtraOptions} HTTPClientExtraOptions
@@ -17,8 +18,7 @@ export const createLs = configure(api => {
       headers: options.headers
     })).json()
 
-    // TODO: unwrap topic names from multibase
-    return Strings || []
+    return rpcArrayToTextArray(Strings) || []
   }
   return ls
 })
diff --git a/packages/ipfs-http-client/src/pubsub/peers.js b/packages/ipfs-http-client/src/pubsub/peers.js
index 33a0009f59..a7cfff6c0d 100644
--- a/packages/ipfs-http-client/src/pubsub/peers.js
+++ b/packages/ipfs-http-client/src/pubsub/peers.js
@@ -1,5 +1,6 @@
 import { configure } from '../lib/configure.js'
 import { toUrlSearchParams } from '../lib/to-url-search-params.js'
+import { textToUrlSafeRpc } from '../lib/http-rpc-wire-format.js'
 
 /**
  * @typedef {import('../types').HTTPClientExtraOptions} HTTPClientExtraOptions
@@ -14,7 +15,7 @@ export const createPeers = configure(api => {
     const res = await api.post('pubsub/peers', {
       signal: options.signal,
       searchParams: toUrlSearchParams({
-        arg: topic,
+        arg: textToUrlSafeRpc(topic),
         ...options
       }),
       headers: options.headers
diff --git a/packages/ipfs-http-client/src/pubsub/publish.js b/packages/ipfs-http-client/src/pubsub/publish.js
index ebf6b565e4..b393e48893 100644
--- a/packages/ipfs-http-client/src/pubsub/publish.js
+++ b/packages/ipfs-http-client/src/pubsub/publish.js
@@ -2,6 +2,7 @@ import { configure } from '../lib/configure.js'
 import { toUrlSearchParams } from '../lib/to-url-search-params.js'
 import { multipartRequest } from 'ipfs-core-utils/multipart-request'
 import { abortSignal } from '../lib/abort-signal.js'
+import { textToUrlSafeRpc } from '../lib/http-rpc-wire-format.js'
 import { AbortController } from 'native-abort-controller'
 
 /**
@@ -14,9 +15,8 @@ export const createPublish = configure(api => {
    * @type {PubsubAPI["publish"]}
    */
   async function publish (topic, data, options = {}) {
-    // TODO: wrap topic in multibase
     const searchParams = toUrlSearchParams({
-      arg: topic,
+      arg: textToUrlSafeRpc(topic),
       ...options
     })
 
diff --git a/packages/ipfs-http-client/src/pubsub/subscribe.js b/packages/ipfs-http-client/src/pubsub/subscribe.js
index 621975a8a0..468206ead4 100644
--- a/packages/ipfs-http-client/src/pubsub/subscribe.js
+++ b/packages/ipfs-http-client/src/pubsub/subscribe.js
@@ -1,8 +1,7 @@
-import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
-import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import debug from 'debug'
 import { configure } from '../lib/configure.js'
 import { toUrlSearchParams } from '../lib/to-url-search-params.js'
+import { textToUrlSafeRpc, rpcArrayToTextArray, rpcToBytes } from '../lib/http-rpc-wire-format.js'
 const log = debug('ipfs-http-client:pubsub:subscribe')
 
 /**
@@ -39,12 +38,11 @@ export const createSubscribe = (options, subsTracker) => {
       // is received. If this doesn't happen within 1 second assume success
       const ffWorkaround = setTimeout(() => done(), 1000)
 
-      // TODO: wrap topic in multibase
       // Do this async to not block Firefox
       api.post('pubsub/sub', {
         signal: options.signal,
         searchParams: toUrlSearchParams({
-          arg: topic,
+          arg: textToUrlSafeRpc(topic),
           ...options
         }),
         headers: options.headers
@@ -95,13 +93,11 @@ async function readMessages (response, { onMessage, onEnd, onError }) {
           continue
         }
 
-        // TODO: multibase data, seqno and topics
-        // TODO: parse string and get peerid bytes using libp2p lib
         onMessage({
-          from: uint8ArrayToString(uint8ArrayFromString(msg.from, 'base64pad'), 'base58btc'),
-          data: uint8ArrayFromString(msg.data, 'base64pad'),
-          seqno: uint8ArrayFromString(msg.seqno, 'base64pad'),
-          topicIDs: msg.topicIDs
+          from: msg.from,
+          data: rpcToBytes(msg.data),
+          seqno: rpcToBytes(msg.seqno),
+          topicIDs: rpcArrayToTextArray(msg.topicIDs)
         })
       } catch (/** @type {any} */ err) {
         err.message = `Failed to parse pubsub message: ${err.message}`
diff --git a/packages/ipfs-http-client/test/utils/factory.js b/packages/ipfs-http-client/test/utils/factory.js
index 012137867c..401b791e1f 100644
--- a/packages/ipfs-http-client/test/utils/factory.js
+++ b/packages/ipfs-http-client/test/utils/factory.js
@@ -16,7 +16,7 @@ const commonOptions = {
 
 const commonOverrides = {
   go: {
-    ipfsBin: isNode ? path() : undefined
+    ipfsBin: isNode ? (process.env.IPFS_GO_EXEC || path()) : undefined
   }
 }
 
diff --git a/packages/ipfs-http-server/src/api/resources/pubsub.js b/packages/ipfs-http-server/src/api/resources/pubsub.js
index 18a140494a..ec9a0282e7 100644
--- a/packages/ipfs-http-server/src/api/resources/pubsub.js
+++ b/packages/ipfs-http-server/src/api/resources/pubsub.js
@@ -6,6 +6,22 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { streamResponse } from '../../utils/stream-response.js'
 import pushable from 'it-pushable'
+import { base64url } from 'multiformats/bases/base64'
+
+const preDecodeTopicFromHttpRpc = {
+  assign: 'topic',
+  /**
+   * @param {import('../../types').Request} request
+   * @param {import('@hapi/hapi').ResponseToolkit} _h
+   */
+  method: async (request, _h) => {
+    try {
+      return uint8ArrayToString(base64url.decode(request.query.topic))
+    } catch (/** @type {any} */ err) {
+      throw Boom.boomify(err, { message: `Failed to decode topic  from HTTP RPC form ${request.query.topic}` })
+    }
+  }
+}
 
 export const subscribeResource = {
   options: {
@@ -24,7 +40,8 @@ export const subscribeResource = {
           override: true,
           ignoreUndefined: true
         })
-    }
+    },
+    pre: [preDecodeTopicFromHttpRpc]
   },
   /**
    * @param {import('../../types').Request} request
@@ -40,8 +57,8 @@ export const subscribeResource = {
           ipfs
         }
       },
-      query: {
-        topic
+      pre: {
+        topic // decoded version created by preDecodeTopicFromHttpRpc
       }
     } = request
 
@@ -56,13 +73,11 @@ export const subscribeResource = {
        * @type {import('ipfs-core-types/src/pubsub').MessageHandlerFn}
        */
       const handler = (msg) => {
-        // TODO: data, seqno and topicIDs in multibase
-        // TODO: from should use canonical toString from peerid libp2p lib
         output.push({
-          from: uint8ArrayToString(uint8ArrayFromString(msg.from, 'base58btc'), 'base64pad'),
-          data: uint8ArrayToString(msg.data, 'base64pad'),
-          seqno: uint8ArrayToString(msg.seqno, 'base64pad'),
-          topicIDs: msg.topicIDs
+          from: msg.from, // TODO: switch to PeerId.parse(msg.from).toString() when go-ipfs defaults to CIDv1
+          data: base64url.encode(msg.data),
+          seqno: base64url.encode(msg.seqno),
+          topicIDs: msg.topicIDs.map(t => base64url.encode(uint8ArrayFromString(t)))
         })
       }
 
@@ -92,7 +107,7 @@ export const publishResource = {
       parse: false,
       output: 'stream'
     },
-    pre: [{
+    pre: [preDecodeTopicFromHttpRpc, {
       assign: 'data',
       /**
        * @param {import('../../types').Request} request
@@ -149,16 +164,15 @@ export const publishResource = {
         }
       },
       pre: {
+        topic,
         data
       },
       query: {
-        topic,
         timeout
       }
     } = request
 
     try {
-      // TODO: unwrap topic from multibase?
       await ipfs.pubsub.publish(topic, data, {
         signal,
         timeout
@@ -212,8 +226,7 @@ export const lsResource = {
       throw Boom.boomify(err, { message: 'Failed to list subscriptions' })
     }
 
-    // TODO: multibase topic names in Strings array
-    return h.response({ Strings: subscriptions })
+    return h.response({ Strings: subscriptions.map(s => base64url.encode(uint8ArrayFromString(s))) })
   }
 }
 
@@ -232,7 +245,8 @@ export const peersResource = {
           override: true,
           ignoreUndefined: true
         })
-    }
+    },
+    pre: [preDecodeTopicFromHttpRpc]
   },
   /**
    * @param {import('../../types').Request} request
@@ -248,15 +262,16 @@ export const peersResource = {
           ipfs
         }
       },
+      pre: {
+        topic
+      },
       query: {
-        topic,
         timeout
       }
     } = request
 
     let peers
     try {
-      // TODO: unwrap topic from multibase
       peers = await ipfs.pubsub.peers(topic, {
         signal,
         timeout

From ac8ecc8dcb1b4eb99263daf8c40507448f89e24e Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 1 Dec 2021 20:14:21 +0100
Subject: [PATCH 03/11] chore: go-ipfs 0.11.0-rc1

---
 packages/ipfs-core/package.json        | 2 +-
 packages/ipfs-http-client/package.json | 2 +-
 packages/ipfs/package.json             | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json
index 79ae545e2f..c3275f7b5a 100644
--- a/packages/ipfs-core/package.json
+++ b/packages/ipfs-core/package.json
@@ -135,7 +135,7 @@
     "@types/rimraf": "^3.0.1",
     "aegir": "^35.1.1",
     "delay": "^5.0.0",
-    "go-ipfs": "0.9.1",
+    "go-ipfs": "0.11.0-rc1",
     "interface-blockstore-tests": "^2.0.1",
     "interface-ipfs-core": "^0.151.1",
     "ipfsd-ctl": "^10.0.4",
diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json
index 3184361423..f33c841eab 100644
--- a/packages/ipfs-http-client/package.json
+++ b/packages/ipfs-http-client/package.json
@@ -76,7 +76,7 @@
   "devDependencies": {
     "aegir": "^35.1.1",
     "delay": "^5.0.0",
-    "go-ipfs": "0.9.1",
+    "go-ipfs": "0.11.0-rc1",
     "ipfsd-ctl": "^10.0.4",
     "it-all": "^1.0.4",
     "it-first": "^1.0.4",
diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json
index e7593aecd1..d4467d85f2 100644
--- a/packages/ipfs/package.json
+++ b/packages/ipfs/package.json
@@ -64,7 +64,7 @@
     "test:interface:client": "aegir test -f test/interface-client.js",
     "test:interface:http-js": "aegir test -f test/interface-http-js.js",
     "test:interface:http-go": "aegir test -f test/interface-http-go.js",
-    "test:interop": "cross-env IPFS_JS_EXEC=$PWD/src/cli.js IPFS_JS_MODULE=$PWD/dist IPFS_JS_HTTP_MODULE=$PWD/node_modules/ipfs-http-client IPFS_REUSEPORT=false ipfs-interop",
+    "test:interop": "cross-env IPFS_JS_EXEC=$PWD/src/cli.js IPFS_JS_MODULE=$PWD/dist IPFS_JS_HTTP_MODULE=$PWD/node_modules/ipfs-http-client LIBP2P_TCP_REUSEPORT=false ipfs-interop",
     "test:external": "aegir test-dependant",
     "clean": "rimraf ./dist",
     "dep-check": "aegir dep-check -i ipfs-core-types -i @types/* -i npm-run-all -i copyfiles"
@@ -84,12 +84,12 @@
     "copyfiles": "^2.4.1",
     "cross-env": "^7.0.0",
     "electron-webrtc": "^0.3.0",
-    "go-ipfs": "0.9.1",
+    "go-ipfs": "0.11.0-rc1",
     "interface-ipfs-core": "^0.151.1",
     "ipfs-client": "^0.7.1",
     "ipfs-core-types": "^0.8.1",
     "ipfs-http-client": "^53.0.1",
-    "ipfs-interop": "^7.0.2",
+    "ipfs-interop": "ipfs/interop#feat/pubsub-require-multibase",
     "ipfs-utils": "^9.0.2",
     "ipfsd-ctl": "^10.0.4",
     "iso-url": "^1.0.0",

From a536257a6b912edc16d12d34eac4c9ef49d4cf22 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Fri, 3 Dec 2021 15:44:49 +0100
Subject: [PATCH 04/11] chore: fix package.json
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

resolve conflicts via web interface, they said
it will be faster, they said 🙃
---
 packages/ipfs-core/package.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json
index ce3de81b2a..23b5314a1f 100644
--- a/packages/ipfs-core/package.json
+++ b/packages/ipfs-core/package.json
@@ -137,7 +137,6 @@
     "aegir": "^36.0.1",
     "delay": "^5.0.0",
     "go-ipfs": "0.11.0-rc1",
-    "go-ipfs": "0.11.0-rc1",
     "interface-blockstore-tests": "^2.0.1",
     "interface-ipfs-core": "^0.152.2",
     "ipfsd-ctl": "^10.0.4",

From 58cce0e4346635bb0a74a38858928efd947f08f3 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Mon, 6 Dec 2021 02:46:02 +0100
Subject: [PATCH 05/11] chore: switch interop to pubsub+dht branch

---
 packages/ipfs/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json
index 91f8c93b9c..0926d2ef6c 100644
--- a/packages/ipfs/package.json
+++ b/packages/ipfs/package.json
@@ -89,7 +89,7 @@
     "ipfs-client": "^0.7.4",
     "ipfs-core-types": "^0.8.4",
     "ipfs-http-client": "^54.0.2",
-    "ipfs-interop": "ipfs/interop#feat/pubsub-require-multibase",
+    "ipfs-interop": "ipfs/interop#feat/js-pubsub-and-dht",
     "ipfs-utils": "^9.0.2",
     "ipfsd-ctl": "^10.0.4",
     "iso-url": "^1.0.0",

From 1a8a4b5e92e93bb56cc512d312f566923e52e4aa Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 7 Dec 2021 14:58:41 +0100
Subject: [PATCH 06/11] test: pubsub multibase in ipfs-http-server

---
 .../ipfs-http-server/test/inject/pubsub.js    | 25 +++++++++++--------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/packages/ipfs-http-server/test/inject/pubsub.js b/packages/ipfs-http-server/test/inject/pubsub.js
index fdcc45edbd..d15af82562 100644
--- a/packages/ipfs-http-server/test/inject/pubsub.js
+++ b/packages/ipfs-http-server/test/inject/pubsub.js
@@ -9,6 +9,8 @@ import sinon from 'sinon'
 import { AbortSignal } from 'native-abort-controller'
 import { randomBytes } from 'iso-random-stream'
 import streamToPromise from 'stream-to-promise'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { base64url } from 'multiformats/bases/base64'
 
 const sendData = async (data) => {
   const form = new FormData()
@@ -20,11 +22,14 @@ const sendData = async (data) => {
     payload
   }
 }
+const textToUrlSafeRpc = text => base64url.encode(uint8ArrayFromString(text))
 
 describe('/pubsub', () => {
   const buf = Buffer.from('some message')
   const topic = 'nonScents'
+  const topicRpcEnc = textToUrlSafeRpc(topic)
   const topicNotSubscribed = 'somethingRandom'
+  const topicNotSubscribedRpcEnc = textToUrlSafeRpc(topicNotSubscribed)
 
   let ipfs
 
@@ -62,7 +67,7 @@ describe('/pubsub', () => {
 
       const res = await http({
         method: 'POST',
-        url: `/api/v0/pubsub/sub?arg=${topic}`
+        url: `/api/v0/pubsub/sub?arg=${topicRpcEnc}`
       }, { ipfs })
 
       expect(res).to.have.property('statusCode', 200)
@@ -92,7 +97,7 @@ describe('/pubsub', () => {
     it('returns 200 with topic and buffer', async () => {
       const res = await http({
         method: 'POST',
-        url: `/api/v0/pubsub/pub?arg=${topic}`,
+        url: `/api/v0/pubsub/pub?arg=${topicRpcEnc}`,
         ...await sendData(buf)
       }, { ipfs })
 
@@ -104,7 +109,7 @@ describe('/pubsub', () => {
       const buf = randomBytes(10)
       const res = await http({
         method: 'POST',
-        url: `/api/v0/pubsub/pub?arg=${topic}`,
+        url: `/api/v0/pubsub/pub?arg=${topicRpcEnc}`,
         ...await sendData(buf)
       }, { ipfs })
 
@@ -115,7 +120,7 @@ describe('/pubsub', () => {
     it('returns 400 with topic and empty buffer', async () => {
       const res = await http({
         method: 'POST',
-        url: `/api/v0/pubsub/pub?arg=${topic}`,
+        url: `/api/v0/pubsub/pub?arg=${topicRpcEnc}`,
         ...await sendData(Buffer.from(''))
       }, { ipfs })
 
@@ -125,7 +130,7 @@ describe('/pubsub', () => {
     it('accepts a timeout', async () => {
       const res = await http({
         method: 'POST',
-        url: `/api/v0/pubsub/pub?arg=${topic}&timeout=1s`,
+        url: `/api/v0/pubsub/pub?arg=${topicRpcEnc}&timeout=1s`,
         ...await sendData(buf)
       }, { ipfs })
 
@@ -158,7 +163,7 @@ describe('/pubsub', () => {
       }, { ipfs })
 
       expect(res).to.have.property('statusCode', 200)
-      expect(res).to.have.deep.nested.property('result.Strings', [topic])
+      expect(res).to.have.deep.nested.property('result.Strings', [topicRpcEnc])
     })
 
     it('accepts a timeout', async () => {
@@ -175,7 +180,7 @@ describe('/pubsub', () => {
       }, { ipfs })
 
       expect(res).to.have.property('statusCode', 200)
-      expect(res).to.have.deep.nested.property('result.Strings', [topic])
+      expect(res).to.have.deep.nested.property('result.Strings', [topicRpcEnc])
     })
   })
 
@@ -194,7 +199,7 @@ describe('/pubsub', () => {
 
       const res = await http({
         method: 'POST',
-        url: `/api/v0/pubsub/peers?arg=${topicNotSubscribed}`
+        url: `/api/v0/pubsub/peers?arg=${topicNotSubscribedRpcEnc}`
       }, { ipfs })
 
       expect(res).to.have.property('statusCode', 200)
@@ -208,7 +213,7 @@ describe('/pubsub', () => {
 
       const res = await http({
         method: 'POST',
-        url: `/api/v0/pubsub/peers?arg=${topic}`
+        url: `/api/v0/pubsub/peers?arg=${topicRpcEnc}`
       }, { ipfs })
 
       expect(res).to.have.property('statusCode', 200)
@@ -227,7 +232,7 @@ describe('/pubsub', () => {
 
       const res = await http({
         method: 'POST',
-        url: `/api/v0/pubsub/peers?arg=${topic}&timeout=1s`
+        url: `/api/v0/pubsub/peers?arg=${topicRpcEnc}&timeout=1s`
       }, { ipfs })
 
       expect(res).to.have.property('statusCode', 200)

From ecd3b430e2b2897034bb7654eb43f331556d9c40 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 7 Dec 2021 15:00:37 +0100
Subject: [PATCH 07/11] chore: go-ipfs 0.11.0-rc2

---
 packages/ipfs-core/package.json        | 2 +-
 packages/ipfs-http-client/package.json | 2 +-
 packages/ipfs/package.json             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json
index 23b5314a1f..ef9e4602ea 100644
--- a/packages/ipfs-core/package.json
+++ b/packages/ipfs-core/package.json
@@ -136,7 +136,7 @@
     "@types/rimraf": "^3.0.1",
     "aegir": "^36.0.1",
     "delay": "^5.0.0",
-    "go-ipfs": "0.11.0-rc1",
+    "go-ipfs": "0.11.0-rc2",
     "interface-blockstore-tests": "^2.0.1",
     "interface-ipfs-core": "^0.152.2",
     "ipfsd-ctl": "^10.0.4",
diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json
index 107c31239e..560925c224 100644
--- a/packages/ipfs-http-client/package.json
+++ b/packages/ipfs-http-client/package.json
@@ -77,7 +77,7 @@
   "devDependencies": {
     "aegir": "^36.0.1",
     "delay": "^5.0.0",
-    "go-ipfs": "0.11.0-rc1",
+    "go-ipfs": "0.11.0-rc2",
     "ipfsd-ctl": "^10.0.4",
     "it-all": "^1.0.4",
     "it-first": "^1.0.4",
diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json
index 0926d2ef6c..3227f9affd 100644
--- a/packages/ipfs/package.json
+++ b/packages/ipfs/package.json
@@ -84,7 +84,7 @@
     "copyfiles": "^2.4.1",
     "cross-env": "^7.0.0",
     "electron-webrtc": "^0.3.0",
-    "go-ipfs": "0.11.0-rc1",
+    "go-ipfs": "0.11.0-rc2",
     "interface-ipfs-core": "^0.152.2",
     "ipfs-client": "^0.7.4",
     "ipfs-core-types": "^0.8.4",

From 746dbef82951de451b820047a20af902c88e3ba3 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 7 Dec 2021 19:46:10 +0100
Subject: [PATCH 08/11] refactor: sharding config in go-ipfs 0.11

Experimental.ShardingEnabled was replaced by autosharding.
To simulate the old behavior ("shard everything") one needs to set
Internal.UnixFSShardingSizeThreshold to 1B.

This is a temporary fix for sharding tests until js-ipfs implements
https://github.com/ipfs/js-ipfs-unixfs/issues/149
---
 packages/interface-ipfs-core/src/add.js         | 6 +++---
 packages/interface-ipfs-core/src/files/cp.js    | 6 +++---
 packages/interface-ipfs-core/src/files/ls.js    | 6 +++---
 packages/interface-ipfs-core/src/files/mkdir.js | 6 +++---
 packages/interface-ipfs-core/src/files/mv.js    | 6 +++---
 packages/interface-ipfs-core/src/files/read.js  | 6 +++---
 packages/interface-ipfs-core/src/files/rm.js    | 6 +++---
 packages/interface-ipfs-core/src/files/stat.js  | 6 +++---
 packages/interface-ipfs-core/src/files/write.js | 6 +++---
 9 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/packages/interface-ipfs-core/src/add.js b/packages/interface-ipfs-core/src/add.js
index d9176bd5ff..a0b582978b 100644
--- a/packages/interface-ipfs-core/src/add.js
+++ b/packages/interface-ipfs-core/src/add.js
@@ -463,9 +463,9 @@ export function testAdd (factory, options) {
               sharding: true
             },
             config: {
-              // enable sharding for go
-              Experimental: {
-                ShardingEnabled: true
+              // enable sharding for go with automatic threshold dropped to the minimum so it shards everything
+              Internal: {
+                UnixFSShardingSizeThreshold: '1B'
               }
             }
           }
diff --git a/packages/interface-ipfs-core/src/files/cp.js b/packages/interface-ipfs-core/src/files/cp.js
index 43bc24c50a..101b16c1b6 100644
--- a/packages/interface-ipfs-core/src/files/cp.js
+++ b/packages/interface-ipfs-core/src/files/cp.js
@@ -358,9 +358,9 @@ export function testCp (factory, options) {
               sharding: true
             },
             config: {
-              // enable sharding for go
-              Experimental: {
-                ShardingEnabled: true
+              // enable sharding for go with automatic threshold dropped to the minimum so it shards everything
+              Internal: {
+                UnixFSShardingSizeThreshold: '1B'
               }
             }
           }
diff --git a/packages/interface-ipfs-core/src/files/ls.js b/packages/interface-ipfs-core/src/files/ls.js
index 9b91df2298..8af8d2e791 100644
--- a/packages/interface-ipfs-core/src/files/ls.js
+++ b/packages/interface-ipfs-core/src/files/ls.js
@@ -174,9 +174,9 @@ export function testLs (factory, options) {
               sharding: true
             },
             config: {
-              // enable sharding for go
-              Experimental: {
-                ShardingEnabled: true
+              // enable sharding for go with automatic threshold dropped to the minimum so it shards everything
+              Internal: {
+                UnixFSShardingSizeThreshold: '1B'
               }
             }
           }
diff --git a/packages/interface-ipfs-core/src/files/mkdir.js b/packages/interface-ipfs-core/src/files/mkdir.js
index a4ca2825fb..17c056990a 100644
--- a/packages/interface-ipfs-core/src/files/mkdir.js
+++ b/packages/interface-ipfs-core/src/files/mkdir.js
@@ -235,9 +235,9 @@ export function testMkdir (factory, options) {
               sharding: true
             },
             config: {
-              // enable sharding for go
-              Experimental: {
-                ShardingEnabled: true
+              // enable sharding for go with automatic threshold dropped to the minimum so it shards everything
+              Internal: {
+                UnixFSShardingSizeThreshold: '1B'
               }
             }
           }
diff --git a/packages/interface-ipfs-core/src/files/mv.js b/packages/interface-ipfs-core/src/files/mv.js
index e3b8758ad6..e7dcf20b66 100644
--- a/packages/interface-ipfs-core/src/files/mv.js
+++ b/packages/interface-ipfs-core/src/files/mv.js
@@ -120,9 +120,9 @@ export function testMv (factory, options) {
               sharding: true
             },
             config: {
-              // enable sharding for go
-              Experimental: {
-                ShardingEnabled: true
+              // enable sharding for go with automatic threshold dropped to the minimum so it shards everything
+              Internal: {
+                UnixFSShardingSizeThreshold: '1B'
               }
             }
           }
diff --git a/packages/interface-ipfs-core/src/files/read.js b/packages/interface-ipfs-core/src/files/read.js
index 571ae774f9..3aabf8ace7 100644
--- a/packages/interface-ipfs-core/src/files/read.js
+++ b/packages/interface-ipfs-core/src/files/read.js
@@ -124,9 +124,9 @@ export function testRead (factory, options) {
               sharding: true
             },
             config: {
-              // enable sharding for go
-              Experimental: {
-                ShardingEnabled: true
+              // enable sharding for go with automatic threshold dropped to the minimum so it shards everything
+              Internal: {
+                UnixFSShardingSizeThreshold: '1B'
               }
             }
           }
diff --git a/packages/interface-ipfs-core/src/files/rm.js b/packages/interface-ipfs-core/src/files/rm.js
index e4e181243a..43d0000d84 100644
--- a/packages/interface-ipfs-core/src/files/rm.js
+++ b/packages/interface-ipfs-core/src/files/rm.js
@@ -143,9 +143,9 @@ export function testRm (factory, options) {
               sharding: true
             },
             config: {
-              // enable sharding for go
-              Experimental: {
-                ShardingEnabled: true
+              // enable sharding for go with automatic threshold dropped to the minimum so it shards everything
+              Internal: {
+                UnixFSShardingSizeThreshold: '1B'
               }
             }
           }
diff --git a/packages/interface-ipfs-core/src/files/stat.js b/packages/interface-ipfs-core/src/files/stat.js
index d5fc7ec91e..08956b82f1 100644
--- a/packages/interface-ipfs-core/src/files/stat.js
+++ b/packages/interface-ipfs-core/src/files/stat.js
@@ -382,9 +382,9 @@ export function testStat (factory, options) {
               sharding: true
             },
             config: {
-              // enable sharding for go
-              Experimental: {
-                ShardingEnabled: true
+              // enable sharding for go with automatic threshold dropped to the minimum so it shards everything
+              Internal: {
+                UnixFSShardingSizeThreshold: '1B'
               }
             }
           }
diff --git a/packages/interface-ipfs-core/src/files/write.js b/packages/interface-ipfs-core/src/files/write.js
index 9004ebca14..aab974cbf0 100644
--- a/packages/interface-ipfs-core/src/files/write.js
+++ b/packages/interface-ipfs-core/src/files/write.js
@@ -661,9 +661,9 @@ export function testWrite (factory, options) {
               sharding: true
             },
             config: {
-              // enable sharding for go
-              Experimental: {
-                ShardingEnabled: true
+              // enable sharding for go with automatic threshold dropped to the minimum so it shards everything
+              Internal: {
+                UnixFSShardingSizeThreshold: '1B'
               }
             }
           }

From cad816defcb70adcc1fc2a75e8de9ca0b567bcae Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 14 Dec 2021 02:43:20 +0100
Subject: [PATCH 09/11] fix: regression in /api/v0/files/rm

This works around https://github.com/ipfs/go-ipfs/issues/8606
---
 packages/ipfs-http-client/src/files/rm.js | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/packages/ipfs-http-client/src/files/rm.js b/packages/ipfs-http-client/src/files/rm.js
index 8f4ec45817..03aa688a22 100644
--- a/packages/ipfs-http-client/src/files/rm.js
+++ b/packages/ipfs-http-client/src/files/rm.js
@@ -1,5 +1,6 @@
 import { configure } from '../lib/configure.js'
 import { toUrlSearchParams } from '../lib/to-url-search-params.js'
+import HTTP from 'ipfs-utils/src/http.js'
 
 /**
  * @typedef {import('../types').HTTPClientExtraOptions} HTTPClientExtraOptions
@@ -20,7 +21,15 @@ export const createRm = configure(api => {
       headers: options.headers
     })
 
-    await res.text()
+    const body = await res.text()
+    // we don't expect text body to be ever present
+    // (if so, it means an error such as https://github.com/ipfs/go-ipfs/issues/8606)
+    if (body !== '') {
+      /** @type {Error} */
+      const error = new HTTP.HTTPError(res)
+      error.message = body
+      throw error
+    }
   }
   return rm
 })

From e2b7a9c59e0447b3caf33b0343ad46593944f495 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 14 Dec 2021 02:43:52 +0100
Subject: [PATCH 10/11] chore: go-ipfs 0.11.0

---
 packages/ipfs-core/package.json        | 2 +-
 packages/ipfs-http-client/package.json | 2 +-
 packages/ipfs/package.json             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json
index 7a75971e08..a5f55514c2 100644
--- a/packages/ipfs-core/package.json
+++ b/packages/ipfs-core/package.json
@@ -136,7 +136,7 @@
     "@types/rimraf": "^3.0.1",
     "aegir": "^36.0.1",
     "delay": "^5.0.0",
-    "go-ipfs": "0.11.0-rc2",
+    "go-ipfs": "0.11.0",
     "interface-blockstore-tests": "^2.0.1",
     "interface-ipfs-core": "^0.152.2",
     "ipfsd-ctl": "^10.0.4",
diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json
index 560925c224..edd5d7e03e 100644
--- a/packages/ipfs-http-client/package.json
+++ b/packages/ipfs-http-client/package.json
@@ -77,7 +77,7 @@
   "devDependencies": {
     "aegir": "^36.0.1",
     "delay": "^5.0.0",
-    "go-ipfs": "0.11.0-rc2",
+    "go-ipfs": "0.11.0",
     "ipfsd-ctl": "^10.0.4",
     "it-all": "^1.0.4",
     "it-first": "^1.0.4",
diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json
index 3227f9affd..c99cc2e349 100644
--- a/packages/ipfs/package.json
+++ b/packages/ipfs/package.json
@@ -84,7 +84,7 @@
     "copyfiles": "^2.4.1",
     "cross-env": "^7.0.0",
     "electron-webrtc": "^0.3.0",
-    "go-ipfs": "0.11.0-rc2",
+    "go-ipfs": "0.11.0",
     "interface-ipfs-core": "^0.152.2",
     "ipfs-client": "^0.7.4",
     "ipfs-core-types": "^0.8.4",

From 7bcc9734dc28bb5f2ccf841b0711e796f0765ba1 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 14 Dec 2021 22:35:31 +0100
Subject: [PATCH 11/11] chore: disable pubsub examples

rationale:
https://github.com/ipfs/js-ipfs/pull/3922#issuecomment-994074036
---
 .github/workflows/test.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7c0c09832c..de184f0f7e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -331,9 +331,9 @@ jobs:
           - name: ipfs browser exchange files
             repo: https://github.com/ipfs-examples/js-ipfs-browser-exchange-files.git
             deps: ipfs-core@$PWD/packages/ipfs-core/dist,ipfs@$PWD/packages/ipfs/dist,ipfs-core-types@$PWD/packages/ipfs-core-types/dist,ipfs-http-client@$PWD/packages/ipfs-http-client/dist
-          - name: ipfs browser ipns publish
-            repo: https://github.com/ipfs-examples/js-ipfs-browser-ipns-publish.git
-            deps: ipfs-core@$PWD/packages/ipfs-core/dist,ipfs-http-client@$PWD/packages/ipfs-http-client/dist
+          #- name: ipfs browser ipns publish TODO: re-enable after example bumped to go-ipfs 0.11 and ipfs-http-client from https://github.com/ipfs/js-ipfs/pull/3922
+          #  repo: https://github.com/ipfs-examples/js-ipfs-browser-ipns-publish.git
+          #  deps: ipfs-core@$PWD/packages/ipfs-core/dist,ipfs-http-client@$PWD/packages/ipfs-http-client/dist
           - name: ipfs browser mfs
             repo: https://github.com/ipfs-examples/js-ipfs-browser-mfs.git
             deps: ipfs-core@$PWD/packages/ipfs-core/dist
@@ -373,9 +373,9 @@ jobs:
           - name: ipfs custom libp2p
             repo: https://github.com/ipfs-examples/js-ipfs-custom-libp2p.git
             deps: ipfs-core@$PWD/packages/ipfs-core/dist
-          - name: ipfs-http-client browser pubsub
-            repo: https://github.com/ipfs-examples/js-ipfs-http-client-browser-pubsub.git
-            deps: ipfs-http-client@$PWD/packages/ipfs-http-client/dist,ipfs@$PWD/packages/ipfs/dist
+          #- name: ipfs-http-client browser pubsub TODO: re-enable after example bumped to go-ipfs 0.11 and ipfs-http-client from https://github.com/ipfs/js-ipfs/pull/3922
+          #  repo: https://github.com/ipfs-examples/js-ipfs-http-client-browser-pubsub.git
+          #  deps: ipfs-http-client@$PWD/packages/ipfs-http-client/dist,ipfs@$PWD/packages/ipfs/dist
           - name: ipfs-http-client bundle webpack
             repo: https://github.com/ipfs-examples/js-ipfs-http-client-bundle-webpack.git
             deps: ipfs-http-client@$PWD/packages/ipfs-http-client/dist,ipfs@$PWD/packages/ipfs/dist