feat: store pins in datastore instead of a DAG

Adds a `.pins` datastore to `ipfs-repo` and uses that to store pins as cbor
binary keyed by base64 stringified multihashes (n.b. not CIDs).

Each pin has several fields:

```javascript
{
  cid: // buffer, the full CID pinned
  type: // string, 'recursive' or 'direct'
  comments: // string, human-readable comments for the pin
}
```

BREAKING CHANGES:

* pins are now stored in a datastore, a repo migration will be necessary
* ipfs.pins.add now returns an async generator
* ipfs.pins.rm now returns an async generator

Depends on:

- [ ] ipfs/js-ipfs-repo#221

Author: achingbrain

examples/custom-ipfs-repo/package.json

@@ -12,7 +12,7 @@
   "dependencies": {
     "datastore-fs": "^0.9.1",
     "ipfs": "^0.41.0",
-    "ipfs-repo": "^0.30.1",
+    "ipfs-repo": "github:ipfs/js-ipfs-repo#store-pins-in-datastore",
     "it-all": "^1.0.1"
   },
   "devDependencies": {

packages/interface-ipfs-core/SPEC/PIN.md

@@ -8,11 +8,11 @@
 
 > Adds an IPFS object to the pinset and also stores it to the IPFS repo. pinset is the set of hashes currently pinned (not gc'able).
 
-##### `ipfs.pin.add(hash, [options])`
+##### `ipfs.pin.add(source, [options])`
 
 Where:
 
-- `hash` is an IPFS multihash.
+- `source` is a [CID], an array of CIDs or an (async) iterable that yields CIDs
 - `options` is an object that can contain the following keys
   - `recursive` (`boolean`) - Recursively pin the object linked. Type: bool. Default: `true`
   - `timeout` (`number`|`string`) - Throw an error if the request does not complete within the specified milliseconds timeout. If `timeout` is a string, the value is parsed as a [human readable duration](https://www.npmjs.com/package/parse-duration). There is no timeout by default.
@@ -21,9 +21,9 @@ Where:
 
 | Type | Description |
 | -------- | -------- |
-| `Promise<{ cid: CID }>` | An array of objects that represent the files that were pinned |
+| `AsyncIterable<CID>` | An async iterable that yields objects containing the CIDs that were pinned |
 
-an array of objects is returned, each of the form:
+Each yielded object has the form:
 
 ```JavaScript
 {
@@ -77,26 +77,27 @@ A great source of [examples][] can be found in the tests for this API.
 
 > Remove a hash from the pinset
 
-##### `ipfs.pin.rm(hash, [options])`
+##### `ipfs.pin.rm(source, [options])`
 
 Where:
-- `hash` is a multihash.
+- `source` is a [CID], an array of CIDs or an (async) iterable that yields CIDs
 - `options` is an object that can contain the following keys
   - 'recursive' - Recursively unpin the object linked. Type: bool. Default: `true`
 
 **Returns**
 
 | Type | Description |
 | -------- | -------- |
-| `Promise<{ cid: CID }>` | An array of unpinned objects |
+| `AsyncIterable<{ cid: CID }>` | An async iterable that yields objects containing the CIDs that were unpinned |
 
 **Example:**
 
 ```JavaScript
-const pinset = await ipfs.pin.rm('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
-console.log(pinset)
+for await (const unpinned of ipfs.pin.rm(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) {
+  console.log(unpinned)
+}
 // prints the hashes that were unpinned
-// [ { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') } ]
+// { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') }
 ```
 
 A great source of [examples][] can be found in the tests for this API.

packages/interface-ipfs-core/package.json

@@ -45,6 +45,7 @@
     "is-ipfs": "^0.6.1",
     "it-all": "^1.0.1",
     "it-concat": "^1.0.0",
+    "it-drain": "^1.0.0",
     "it-last": "^1.0.1",
     "it-pushable": "^1.3.1",
     "multiaddr": "^7.2.1",

packages/interface-ipfs-core/src/block/rm.js

@@ -4,6 +4,8 @@
 const { getDescribe, getIt, expect } = require('../utils/mocha')
 const hat = require('hat')
 const all = require('it-all')
+const last = require('it-last')
+const drain = require('it-drain')
 
 /** @typedef { import("ipfsd-ctl/src/factory") } Factory */
 /**
@@ -142,13 +144,12 @@ module.exports = (common, options) => {
         format: 'raw',
         hashAlg: 'sha2-256'
       })
-      await ipfs.pin.add(cid.toString())
+      await drain(ipfs.pin.add(cid))
 
-      const result = await all(ipfs.block.rm(cid))
+      const result = await last(ipfs.block.rm(cid))
 
-      expect(result).to.be.an('array').and.to.have.lengthOf(1)
-      expect(result[0]).to.have.property('error')
-      expect(result[0].error.message).to.include('pinned')
+      expect(result).to.have.property('error').that.is.an('Error')
+        .with.property('message').that.includes('pinned')
     })
   })
 }

packages/interface-ipfs-core/src/pin/add.js

@@ -28,8 +28,8 @@ module.exports = (common, options) => {
     after(() => common.clean())
 
     it('should add a pin', async () => {
-      const pinset = await ipfs.pin.add(fixtures.files[0].cid, { recursive: false })
-      expect(pinset.map(p => p.cid.toString())).to.include(fixtures.files[0].cid)
+      const pinset = await all(ipfs.pin.add(fixtures.files[0].cid, { recursive: false }))
+      expect(pinset.map(p => p.cid)).to.deep.include(fixtures.files[0].cid)
     })
   })
 }

packages/interface-ipfs-core/src/pin/index.js

@@ -4,7 +4,8 @@ const { createSuite } = require('../utils/suite')
 const tests = {
   ls: require('./ls'),
   rm: require('./rm'),
-  add: require('./add')
+  add: require('./add'),
+  core: require('./pins')
 }
 
 module.exports = createSuite(tests)

packages/interface-ipfs-core/src/pin/ls.js

@@ -4,6 +4,7 @@
 const { fixtures } = require('./utils')
 const { getDescribe, getIt, expect } = require('../utils/mocha')
 const all = require('it-all')
+const drain = require('it-drain')
 
 /** @typedef { import("ipfsd-ctl/src/factory") } Factory */
 /**
@@ -23,22 +24,21 @@ module.exports = (common, options) => {
       ipfs = (await common.spawn()).api
       // two files wrapped in directories, only root CID pinned recursively
       const dir = fixtures.directory.files.map((file) => ({ path: file.path, content: file.data }))
-      await all(ipfs.add(dir, { pin: false, cidVersion: 0 }))
-      await ipfs.pin.add(fixtures.directory.cid, { recursive: true })
+      await drain(ipfs.add(dir, { pin: false, cidVersion: 0 }))
+      await drain(ipfs.pin.add(fixtures.directory.cid, { recursive: true }))
       // a file (CID pinned recursively)
-      await all(ipfs.add(fixtures.files[0].data, { pin: false, cidVersion: 0 }))
-      await ipfs.pin.add(fixtures.files[0].cid, { recursive: true })
+      await drain(ipfs.add(fixtures.files[0].data, { pin: false, cidVersion: 0 }))
+      await drain(ipfs.pin.add(fixtures.files[0].cid, { recursive: true }))
       // a single CID (pinned directly)
-      await all(ipfs.add(fixtures.files[1].data, { pin: false, cidVersion: 0 }))
-      await ipfs.pin.add(fixtures.files[1].cid, { recursive: false })
+      await drain(ipfs.add(fixtures.files[1].data, { pin: false, cidVersion: 0 }))
+      await drain(ipfs.pin.add(fixtures.files[1].cid, { recursive: false }))
     })
 
     after(() => common.clean())
 
     // 1st, because ipfs.add pins automatically
     it('should list all recursive pins', async () => {
-      const pinset = (await all(ipfs.pin.ls({ type: 'recursive' })))
-        .map(p => ({ ...p, cid: p.cid.toString() }))
+      const pinset = await all(ipfs.pin.ls({ type: 'recursive' }))
 
       expect(pinset).to.deep.include({
         type: 'recursive',
@@ -51,8 +51,7 @@ module.exports = (common, options) => {
     })
 
     it('should list all indirect pins', async () => {
-      const pinset = (await all(ipfs.pin.ls({ type: 'indirect' })))
-        .map(p => ({ ...p, cid: p.cid.toString() }))
+      const pinset = await all(ipfs.pin.ls({ type: 'indirect' }))
 
       expect(pinset).to.not.deep.include({
         type: 'recursive',
@@ -77,8 +76,7 @@ module.exports = (common, options) => {
     })
 
     it('should list all types of pins', async () => {
-      const pinset = (await all(ipfs.pin.ls()))
-        .map(p => ({ ...p, cid: p.cid.toString() }))
+      const pinset = await all(ipfs.pin.ls())
 
       expect(pinset).to.not.be.empty()
       // check the three "roots"
@@ -107,15 +105,19 @@ module.exports = (common, options) => {
     it('should list all direct pins', async () => {
       const pinset = await all(ipfs.pin.ls({ type: 'direct' }))
       expect(pinset).to.have.lengthOf(1)
-      expect(pinset[0].type).to.equal('direct')
-      expect(pinset[0].cid.toString()).to.equal(fixtures.files[1].cid)
+      expect(pinset).to.deep.include({
+        type: 'direct',
+        cid: fixtures.files[1].cid
+      })
     })
 
     it('should list pins for a specific hash', async () => {
       const pinset = await all(ipfs.pin.ls(fixtures.files[0].cid))
       expect(pinset).to.have.lengthOf(1)
-      expect(pinset[0].type).to.equal('recursive')
-      expect(pinset[0].cid.toString()).to.equal(fixtures.files[0].cid)
+      expect(pinset).to.deep.include({
+        type: 'recursive',
+        cid: fixtures.files[0].cid
+      })
     })
 
     it('should throw an error on missing direct pins for existing path', () => {
@@ -136,22 +138,26 @@ module.exports = (common, options) => {
     it('should list indirect pins for a specific path', async () => {
       const pinset = await all(ipfs.pin.ls(`/ipfs/${fixtures.directory.cid}/files/ipfs.txt`, { type: 'indirect' }))
       expect(pinset).to.have.lengthOf(1)
-      expect(pinset[0].type).to.equal(`indirect through ${fixtures.directory.cid}`)
-      expect(pinset[0].cid.toString()).to.equal(fixtures.directory.files[1].cid)
+      expect(pinset).to.deep.include({
+        type: `indirect through ${fixtures.directory.cid}`,
+        cid: fixtures.directory.files[1].cid
+      })
     })
 
     it('should list recursive pins for a specific hash', async () => {
       const pinset = await all(ipfs.pin.ls(fixtures.files[0].cid, { type: 'recursive' }))
       expect(pinset).to.have.lengthOf(1)
-      expect(pinset[0].type).to.equal('recursive')
-      expect(pinset[0].cid.toString()).to.equal(fixtures.files[0].cid)
+      expect(pinset).to.deep.include({
+        type: 'recursive',
+        cid: fixtures.files[0].cid
+      })
     })
 
     it('should list pins for multiple CIDs', async () => {
       const pinset = await all(ipfs.pin.ls([fixtures.files[0].cid, fixtures.files[1].cid]))
-      const cids = pinset.map(p => p.cid.toString())
-      expect(cids).to.include(fixtures.files[0].cid)
-      expect(cids).to.include(fixtures.files[1].cid)
+      const cids = pinset.map(p => p.cid)
+      expect(cids).to.deep.include(fixtures.files[0].cid)
+      expect(cids).to.deep.include(fixtures.files[1].cid)
     })
   })
 }