diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 359eb975..28068134 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -3,7 +3,7 @@ name: test & maybe release on: push: branches: - - master + - main pull_request: workflow_dispatch: @@ -19,9 +19,10 @@ concurrency: jobs: js-test-and-release: - uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 + uses: ipdxco/unified-github-workflows/.github/workflows/js-test-and-release.yml@v1.0 secrets: DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/README.md b/README.md index a617dbb7..9d0fa234 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ +# js-ipfs-unixfs + [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-unixfs/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) > JS implementation of the IPFS UnixFS diff --git a/package.json b/package.json index f0b522d5..3021f1eb 100644 --- a/package.json +++ b/package.json @@ -12,36 +12,9 @@ "url": "https://github.com/ipfs/js-ipfs-unixfs/issues" }, "private": true, - "scripts": { - "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", - "test": "aegir run test", - "test:node": "aegir run test:node", - "test:chrome": "aegir run test:chrome", - "test:chrome-webworker": "aegir run test:chrome-webworker", - "test:firefox": "aegir run test:firefox", - "test:firefox-webworker": "aegir run test:firefox-webworker", - "test:electron-main": "aegir run test:electron-main", - "test:electron-renderer": "aegir run test:electron-renderer", - "clean": "aegir run clean", - "generate": "aegir run generate", - "build": "aegir run build", - "lint": "aegir run lint", - "dep-check": "aegir run dep-check", - "release": "run-s build docs:no-publish npm:release docs", - "npm:release": "aegir release", - "docs": "aegir docs", - "docs:no-publish": "aegir docs --publish false" - }, - "devDependencies": { - "aegir": "^42.2.2", - "npm-run-all": "^4.1.5" - }, - "workspaces": [ - "packages/*" - ], "release": { "branches": [ - "master" + "main" ], "plugins": [ [ @@ -123,5 +96,32 @@ "@semantic-release/github", "@semantic-release/git" ] - } + }, + "scripts": { + "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", + "test": "aegir run test", + "test:node": "aegir run test:node", + "test:chrome": "aegir run test:chrome", + "test:chrome-webworker": "aegir run test:chrome-webworker", + "test:firefox": "aegir run test:firefox", + "test:firefox-webworker": "aegir run test:firefox-webworker", + "test:electron-main": "aegir run test:electron-main", + "test:electron-renderer": "aegir run test:electron-renderer", + "clean": "aegir run clean", + "generate": "aegir run generate", + "build": "aegir run build", + "lint": "aegir run lint", + "dep-check": "aegir run dep-check", + "release": "run-s build docs:no-publish npm:release docs", + "npm:release": "aegir release", + "docs": "aegir docs", + "docs:no-publish": "aegir docs --publish false" + }, + "devDependencies": { + "aegir": "^42.2.2", + "npm-run-all": "^4.1.5" + }, + "workspaces": [ + "packages/*" + ] } diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 4883e49e..db0ca47b 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,36 @@ +## ipfs-unixfs-exporter [13.5.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.4.0...ipfs-unixfs-exporter-13.5.0) (2024-02-02) + + +### Features + +* add json resolver ([#400](https://github.com/ipfs/js-ipfs-unixfs/issues/400)) ([81e85c8](https://github.com/ipfs/js-ipfs-unixfs/commit/81e85c8a6798a8f76e11bdf2eebe9bfa2d9cb5a4)), closes [#397](https://github.com/ipfs/js-ipfs-unixfs/issues/397) + +## ipfs-unixfs-exporter [13.4.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.3.1...ipfs-unixfs-exporter-13.4.0) (2024-01-19) + + +### Features + +* add blockReadConcurrency option to exporter ([#361](https://github.com/ipfs/js-ipfs-unixfs/issues/361)) ([295077e](https://github.com/ipfs/js-ipfs-unixfs/commit/295077ea70c99fd6c4ca0ef8c304781de07120c7)), closes [#359](https://github.com/ipfs/js-ipfs-unixfs/issues/359) + +## ipfs-unixfs-exporter [13.3.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.3.0...ipfs-unixfs-exporter-13.3.1) (2024-01-19) + + +### Trivial Changes + +* update project config ([125f4d7](https://github.com/ipfs/js-ipfs-unixfs/commit/125f4d7dba9e04914f16560fb1f5002e298eaba1)) + + +### Dependencies + +* **dev:** bump aegir from 41.3.5 to 42.2.2 ([#399](https://github.com/ipfs/js-ipfs-unixfs/issues/399)) ([9d6c7cb](https://github.com/ipfs/js-ipfs-unixfs/commit/9d6c7cb9904e40e1fb4d92735fac62f5bc04559e)) + + + +### Dependencies + +* **ipfs-unixfs:** upgraded to 11.1.3 +* **ipfs-unixfs-importer:** upgraded to 15.2.4 + ## [ipfs-unixfs-exporter-v13.3.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-v13.2.5...ipfs-unixfs-exporter-v13.3.0) (2024-01-16) diff --git a/packages/ipfs-unixfs-exporter/README.md b/packages/ipfs-unixfs-exporter/README.md index 09e866df..9ddd6bcb 100644 --- a/packages/ipfs-unixfs-exporter/README.md +++ b/packages/ipfs-unixfs-exporter/README.md @@ -1,14 +1,29 @@ -# ipfs-unixfs-exporter +# ipfs-unixfs-exporter [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-unixfs/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) > JavaScript implementation of the UnixFs exporter used by IPFS # About + + The UnixFS Exporter provides a means to read DAGs from a blockstore given a CID. ## Example @@ -88,9 +103,3 @@ Please be aware that all interactions related to this repo are subject to the IP Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) - -[dag API]: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md - -[blockstore]: https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-blockstore#readme - -[UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index d397711a..c007b0c0 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,9 +1,9 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.3.0", + "version": "13.5.0", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/master/packages/ipfs-unixfs-exporter#readme", + "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/js-ipfs-unixfs.git" @@ -50,44 +50,46 @@ "dep-check": "aegir dep-check" }, "dependencies": { - "@ipld/dag-cbor": "^9.0.0", - "@ipld/dag-json": "^10.1.7", - "@ipld/dag-pb": "^4.0.0", - "@multiformats/murmur3": "^2.0.0", + "@ipld/dag-cbor": "^9.2.0", + "@ipld/dag-json": "^10.2.0", + "@ipld/dag-pb": "^4.1.0", + "@multiformats/murmur3": "^2.1.8", "err-code": "^3.0.1", - "hamt-sharding": "^3.0.0", - "interface-blockstore": "^5.0.0", + "hamt-sharding": "^3.0.6", + "interface-blockstore": "^5.2.10", "ipfs-unixfs": "^11.0.0", - "it-filter": "^3.0.2", - "it-last": "^3.0.2", - "it-map": "^3.0.3", - "it-parallel": "^3.0.0", + "it-filter": "^3.0.4", + "it-last": "^3.0.4", + "it-map": "^3.0.5", + "it-parallel": "^3.0.6", "it-pipe": "^3.0.1", - "it-pushable": "^3.1.0", - "multiformats": "^13.0.0", + "it-pushable": "^3.2.3", + "multiformats": "^13.1.0", "p-queue": "^8.0.1", "progress-events": "^1.0.0" }, "devDependencies": { - "@types/readable-stream": "^4.0.1", - "@types/sinon": "^17.0.2", - "aegir": "^42.2.2", - "blockstore-core": "^4.0.1", + "@types/readable-stream": "^4.0.11", + "@types/sinon": "^17.0.3", + "aegir": "^42.2.5", + "blockstore-core": "^4.4.0", "delay": "^6.0.0", "ipfs-unixfs-importer": "^15.0.0", "iso-random-stream": "^2.0.2", - "it-all": "^3.0.2", - "it-buffer-stream": "^3.0.0", - "it-first": "^3.0.2", - "it-to-buffer": "^4.0.2", + "it-all": "^3.0.4", + "it-buffer-stream": "^3.0.6", + "it-drain": "^3.0.5", + "it-first": "^3.0.4", + "it-to-buffer": "^4.0.5", "merge-options": "^3.0.4", - "readable-stream": "^4.4.0", + "readable-stream": "^4.5.2", "sinon": "^17.0.1", - "uint8arrays": "^5.0.0", + "uint8arrays": "^5.0.3", "wherearewe": "^2.0.1" }, "browser": { "fs": false, "readable-stream": false - } + }, + "sideEffects": false } diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index 5d31070d..b1adc319 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -94,9 +94,40 @@ export type ExporterProgressEvents = ProgressEvent<'unixfs:exporter:walk:raw', ExportWalk> export interface ExporterOptions extends ProgressOptions { + /** + * An optional offset to start reading at. + * + * If the CID resolves to a file this will be a byte offset within that file, + * otherwise if it's a directory it will be a directory entry offset within + * the directory listing. (default: undefined) + */ offset?: number + + /** + * An optional length to read. + * + * If the CID resolves to a file this will be the number of bytes read from + * the file, otherwise if it's a directory it will be the number of directory + * entries read from the directory listing. (default: undefined) + */ length?: number + + /** + * This signal can be used to abort any long-lived operations such as fetching + * blocks from the network. (default: undefined) + */ signal?: AbortSignal + + /** + * When a DAG layer is encountered, all child nodes are loaded in parallel but + * processed as they arrive. This allows us to load sibling nodes in advance + * of yielding their bytes. Pass a value here to control the number of blocks + * loaded in parallel. If a strict depth-first traversal is required, this + * value should be set to `1`, otherwise the traversal order will tend to + * resemble a breadth-first fan-out and yield a have stable ordering. + * (default: undefined) + */ + blockReadConcurrency?: number } export interface Exportable { @@ -143,6 +174,8 @@ export interface Exportable { size: bigint /** + * @example File content + * * When `entry` is a file or a `raw` node, `offset` and/or `length` arguments can be passed to `entry.content()` to return slices of data: * * ```javascript @@ -162,6 +195,8 @@ export interface Exportable { * return data * ``` * + * @example Directory content + * * If `entry` is a directory, passing `offset` and/or `length` to `entry.content()` will limit the number of files returned from the directory. * * ```javascript @@ -176,7 +211,6 @@ export interface Exportable { * * // `entries` contains the first 5 files/directories in the directory * ``` - * */ content(options?: ExporterOptions): AsyncGenerator } diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.ts b/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.ts index a653a1ad..0631af98 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.ts @@ -1,67 +1,12 @@ import * as dagCbor from '@ipld/dag-cbor' -import errCode from 'err-code' -import { CID } from 'multiformats/cid' +import { resolveObjectPath } from '../utils/resolve-object-path.js' import type { Resolver } from '../index.js' const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { const block = await blockstore.get(cid, options) const object = dagCbor.decode(block) - let subObject = object - let subPath = path - while (toResolve.length > 0) { - const prop = toResolve[0] - - if (prop in subObject) { - // remove the bit of the path we have resolved - toResolve.shift() - subPath = `${subPath}/${prop}` - - const subObjectCid = CID.asCID(subObject[prop]) - if (subObjectCid != null) { - return { - entry: { - type: 'object', - name, - path, - cid, - node: block, - depth, - size: BigInt(block.length), - content: async function * () { - yield object - } - }, - next: { - cid: subObjectCid, - name: prop, - path: subPath, - toResolve - } - } - } - - subObject = subObject[prop] - } else { - // cannot resolve further - throw errCode(new Error(`No property named ${prop} found in cbor node ${cid}`), 'ERR_NO_PROP') - } - } - - return { - entry: { - type: 'object', - name, - path, - cid, - node: block, - depth, - size: BigInt(block.length), - content: async function * () { - yield object - } - } - } + return resolveObjectPath(object, block, cid, name, path, toResolve, depth) } export default resolve diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/dag-json.ts b/packages/ipfs-unixfs-exporter/src/resolvers/dag-json.ts index d701501d..c206d7af 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/dag-json.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/dag-json.ts @@ -1,67 +1,12 @@ import * as dagJson from '@ipld/dag-json' -import errCode from 'err-code' -import { CID } from 'multiformats/cid' +import { resolveObjectPath } from '../utils/resolve-object-path.js' import type { Resolver } from '../index.js' const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { const block = await blockstore.get(cid, options) const object = dagJson.decode(block) - let subObject = object - let subPath = path - while (toResolve.length > 0) { - const prop = toResolve[0] - - if (prop in subObject) { - // remove the bit of the path we have resolved - toResolve.shift() - subPath = `${subPath}/${prop}` - - const subObjectCid = CID.asCID(subObject[prop]) - if (subObjectCid != null) { - return { - entry: { - type: 'object', - name, - path, - cid, - node: block, - depth, - size: BigInt(block.length), - content: async function * () { - yield object - } - }, - next: { - cid: subObjectCid, - name: prop, - path: subPath, - toResolve - } - } - } - - subObject = subObject[prop] - } else { - // cannot resolve further - throw errCode(new Error(`No property named ${prop} found in dag-json node ${cid}`), 'ERR_NO_PROP') - } - } - - return { - entry: { - type: 'object', - name, - path, - cid, - node: block, - depth, - size: BigInt(block.length), - content: async function * () { - yield object - } - } - } + return resolveObjectPath(object, block, cid, name, path, toResolve, depth) } export default resolve diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/index.ts index 93038d32..c314fa67 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/index.ts @@ -2,11 +2,13 @@ import * as dagCbor from '@ipld/dag-cbor' import * as dagJson from '@ipld/dag-json' import * as dagPb from '@ipld/dag-pb' import errCode from 'err-code' +import * as json from 'multiformats/codecs/json' import * as raw from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' import dagCborResolver from './dag-cbor.js' import dagJsonResolver from './dag-json.js' import identifyResolver from './identity.js' +import jsonResolver from './json.js' import rawResolver from './raw.js' import dagPbResolver from './unixfs-v1/index.js' import type { Resolve, Resolver } from '../index.js' @@ -16,7 +18,8 @@ const resolvers: Record = { [raw.code]: rawResolver, [dagCbor.code]: dagCborResolver, [dagJson.code]: dagJsonResolver, - [identity.code]: identifyResolver + [identity.code]: identifyResolver, + [json.code]: jsonResolver } const resolve: Resolve = async (cid, name, path, toResolve, depth, blockstore, options) => { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/json.ts b/packages/ipfs-unixfs-exporter/src/resolvers/json.ts new file mode 100644 index 00000000..90c5a174 --- /dev/null +++ b/packages/ipfs-unixfs-exporter/src/resolvers/json.ts @@ -0,0 +1,12 @@ +import * as json from 'multiformats/codecs/json' +import { resolveObjectPath } from '../utils/resolve-object-path.js' +import type { Resolver } from '../index.js' + +const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { + const block = await blockstore.get(cid, options) + const object = json.decode(block) + + return resolveObjectPath(object, block, cid, name, path, toResolve, depth) +} + +export default resolve diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts index bfa1d61d..afab2634 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts @@ -25,7 +25,10 @@ const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, de return result.entry } }), - source => parallel(source, { ordered: true }), + source => parallel(source, { + ordered: true, + concurrency: options.blockReadConcurrency + }), source => filter(source, entry => entry != null) ) } diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts index 1da18056..f65a449a 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts @@ -84,7 +84,8 @@ async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8A } }), (source) => parallel(source, { - ordered: true + ordered: true, + concurrency: options.blockReadConcurrency }), async (source) => { for await (const { link, block, blockStart } of source) { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index 9e59d7c9..1c482c68 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -62,7 +62,10 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de } } }), - source => parallel(source, { ordered: true }) + source => parallel(source, { + ordered: true, + concurrency: options.blockReadConcurrency + }) ) for await (const { entries } of results) { diff --git a/packages/ipfs-unixfs-exporter/src/utils/resolve-object-path.ts b/packages/ipfs-unixfs-exporter/src/utils/resolve-object-path.ts new file mode 100644 index 00000000..addb7066 --- /dev/null +++ b/packages/ipfs-unixfs-exporter/src/utils/resolve-object-path.ts @@ -0,0 +1,62 @@ +import errCode from 'err-code' +import { CID } from 'multiformats/cid' +import type { ResolveResult } from '../index.js' + +export function resolveObjectPath (object: any, block: Uint8Array, cid: CID, name: string, path: string, toResolve: string[], depth: number): ResolveResult { + let subObject = object + let subPath = path + + while (toResolve.length > 0) { + const prop = toResolve[0] + + if (prop in subObject) { + // remove the bit of the path we have resolved + toResolve.shift() + subPath = `${subPath}/${prop}` + + const subObjectCid = CID.asCID(subObject[prop]) + if (subObjectCid != null) { + return { + entry: { + type: 'object', + name, + path, + cid, + node: block, + depth, + size: BigInt(block.length), + content: async function * () { + yield object + } + }, + next: { + cid: subObjectCid, + name: prop, + path: subPath, + toResolve + } + } + } + + subObject = subObject[prop] + } else { + // cannot resolve further + throw errCode(new Error(`No property named ${prop} found in node ${cid}`), 'ERR_NO_PROP') + } + } + + return { + entry: { + type: 'object', + name, + path, + cid, + node: block, + depth, + size: BigInt(block.length), + content: async function * () { + yield object + } + } + } +} diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index f1c5fd95..c1bce7d9 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -12,14 +12,17 @@ import { fixedSize } from 'ipfs-unixfs-importer/chunker' import { balanced, type FileLayout, flat, trickle } from 'ipfs-unixfs-importer/layout' import all from 'it-all' import randomBytes from 'it-buffer-stream' +import drain from 'it-drain' import first from 'it-first' import last from 'it-last' import toBuffer from 'it-to-buffer' import { CID } from 'multiformats/cid' +import * as json from 'multiformats/codecs/json' import * as raw from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' import { sha256 } from 'multiformats/hashes/sha2' import { Readable } from 'readable-stream' +import Sinon from 'sinon' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' @@ -976,7 +979,7 @@ describe('exporter', () => { expect(data).to.deep.equal(smallFile) }) - it('errors when exporting a non-existent key from a cbor node', async () => { + it('errors when exporting a non-existent key from a dag-cbor node', async () => { const node = { foo: 'bar' } @@ -992,7 +995,7 @@ describe('exporter', () => { } }) - it('exports a cbor node', async () => { + it('exports a dag-cbor node', async () => { const node = { foo: 'bar' } @@ -1009,7 +1012,7 @@ describe('exporter', () => { return expect(first(exported.content())).to.eventually.deep.equal(node) }) - it('errors when exporting a non-existent key from a json node', async () => { + it('errors when exporting a non-existent key from a dag-json node', async () => { const node = { foo: 'bar' } @@ -1025,7 +1028,7 @@ describe('exporter', () => { } }) - it('exports a json node', async () => { + it('exports a dag-json node', async () => { const node = { foo: 'bar' } @@ -1042,6 +1045,39 @@ describe('exporter', () => { return expect(first(exported.content())).to.eventually.deep.equal(node) }) + it('errors when exporting a non-existent key from a json node', async () => { + const node = { + foo: 'bar' + } + + const jsonBlock = json.encode(node) + const cid = CID.createV1(json.code, await sha256.digest(jsonBlock)) + await block.put(cid, jsonBlock) + + try { + await exporter(`${cid}/baz`, block) + } catch (err: any) { + expect(err.code).to.equal('ERR_NO_PROP') + } + }) + + it('exports a json node', async () => { + const node = { + foo: 'bar' + } + + const jsonBlock = json.encode(node) + const cid = CID.createV1(json.code, await sha256.digest(jsonBlock)) + await block.put(cid, jsonBlock) + const exported = await exporter(`${cid}`, block) + + if (exported.type !== 'object') { + throw new Error('Unexpected type') + } + + return expect(first(exported.content())).to.eventually.deep.equal(node) + }) + it('errors when exporting a node with no resolver', async () => { const cid = CID.create(1, 0x78, CID.parse('zdj7WkRPAX9o9nb9zPbXzwG7JEs78uyhwbUs8JSUayB98DWWY').multihash) @@ -1343,4 +1379,228 @@ describe('exporter', () => { dataSizeInBytes *= 10 } }) + + it('should allow control of block read concurrency for files', async () => { + // create a multi-layered DAG of a manageable size + const imported = await first(importer([{ + path: '1.2MiB.txt', + content: asAsyncIterable(smallFile) + }], block, { + rawLeaves: true, + chunker: fixedSize({ chunkSize: 50 }), + layout: balanced({ maxChildrenPerNode: 2 }) + })) + + if (imported == null) { + throw new Error('Nothing imported') + } + + const node = dagPb.decode(await block.get(imported.cid)) + expect(node.Links).to.have.lengthOf(2, 'imported node had too many children') + + const child1 = dagPb.decode(await block.get(node.Links[0].Hash)) + expect(child1.Links).to.have.lengthOf(2, 'layer 1 node had too many children') + + const child2 = dagPb.decode(await block.get(node.Links[1].Hash)) + expect(child2.Links).to.have.lengthOf(2, 'layer 1 node had too many children') + + // should be raw nodes + expect(child1.Links[0].Hash.code).to.equal(raw.code, 'layer 2 node had wrong codec') + expect(child1.Links[1].Hash.code).to.equal(raw.code, 'layer 2 node had wrong codec') + expect(child2.Links[0].Hash.code).to.equal(raw.code, 'layer 2 node had wrong codec') + expect(child2.Links[1].Hash.code).to.equal(raw.code, 'layer 2 node had wrong codec') + + // export file + const file = await exporter(imported.cid, block) + + // export file data with default settings + const blockReadSpy = Sinon.spy(block, 'get') + const contentWithDefaultBlockConcurrency = await toBuffer(file.content()) + + // blocks should be loaded in default order - a whole level of sibling nodes at a time + expect(blockReadSpy.getCalls().map(call => call.args[0].toString())).to.deep.equal([ + node.Links[0].Hash.toString(), + node.Links[1].Hash.toString(), + child1.Links[0].Hash.toString(), + child1.Links[1].Hash.toString(), + child2.Links[0].Hash.toString(), + child2.Links[1].Hash.toString() + ]) + + // export file data overriding read concurrency + blockReadSpy.resetHistory() + const contentWitSmallBlockConcurrency = await toBuffer(file.content({ + blockReadConcurrency: 1 + })) + + // blocks should be loaded in traversal order + expect(blockReadSpy.getCalls().map(call => call.args[0].toString())).to.deep.equal([ + node.Links[0].Hash.toString(), + child1.Links[0].Hash.toString(), + child1.Links[1].Hash.toString(), + node.Links[1].Hash.toString(), + child2.Links[0].Hash.toString(), + child2.Links[1].Hash.toString() + ]) + + // ensure exported bytes are the same + expect(contentWithDefaultBlockConcurrency).to.equalBytes(contentWitSmallBlockConcurrency) + }) + + it('should allow control of block read concurrency for directories', async () => { + const entries = 1024 + + // create a largeish directory + const imported = await last(importer((async function * () { + for (let i = 0; i < entries; i++) { + yield { + path: `file-${i}.txt`, + content: Uint8Array.from([i]) + } + } + })(), block, { + wrapWithDirectory: true + })) + + if (imported == null) { + throw new Error('Nothing imported') + } + + const node = dagPb.decode(await block.get(imported.cid)) + expect(node.Links).to.have.lengthOf(entries, 'imported node had too many children') + + for (const link of node.Links) { + // should be raw nodes + expect(link.Hash.code).to.equal(raw.code, 'child node had wrong codec') + } + + // export directory + const directory = await exporter(imported.cid, block) + + // export file data with default settings + const originalGet = block.get.bind(block) + + const expectedInvocations: string[] = [] + + for (const link of node.Links) { + expectedInvocations.push(`${link.Hash.toString()}-start`) + expectedInvocations.push(`${link.Hash.toString()}-end`) + } + + const actualInvocations: string[] = [] + + block.get = async (cid) => { + actualInvocations.push(`${cid.toString()}-start`) + + // introduce a small delay - if running in parallel actualInvocations will + // be: + // `foo-start`, `bar-start`, `baz-start`, `foo-end`, `bar-end`, `baz-end` + // if in series it will be: + // `foo-start`, `foo-end`, `bar-start`, `bar-end`, `baz-start`, `baz-end` + await delay(1) + + actualInvocations.push(`${cid.toString()}-end`) + + return originalGet(cid) + } + + const blockReadSpy = Sinon.spy(block, 'get') + await drain(directory.content({ + blockReadConcurrency: 1 + })) + + // blocks should be loaded in default order - a whole level of sibling nodes at a time + expect(blockReadSpy.getCalls().map(call => call.args[0].toString())).to.deep.equal( + node.Links.map(link => link.Hash.toString()) + ) + + expect(actualInvocations).to.deep.equal(expectedInvocations) + }) + + it('should allow control of block read concurrency for HAMT sharded directories', async () => { + const entries = 1024 + + // create a sharded directory + const imported = await last(importer((async function * () { + for (let i = 0; i < entries; i++) { + yield { + path: `file-${i}.txt`, + content: Uint8Array.from([i]) + } + } + })(), block, { + wrapWithDirectory: true, + shardSplitThresholdBytes: 10 + })) + + if (imported == null) { + throw new Error('Nothing imported') + } + + const node = dagPb.decode(await block.get(imported.cid)) + const data = UnixFS.unmarshal(node.Data ?? new Uint8Array(0)) + expect(data.type).to.equal('hamt-sharded-directory') + + // traverse the shard, collect all the CIDs + async function collectCIDs (node: PBNode): Promise { + const children: CID[] = [] + + for (const link of node.Links) { + children.push(link.Hash) + + if (link.Hash.code === dagPb.code) { + const buf = await block.get(link.Hash) + const childNode = dagPb.decode(buf) + + children.push(...(await collectCIDs(childNode))) + } + } + + return children + } + + const children: CID[] = await collectCIDs(node) + + // export directory + const directory = await exporter(imported.cid, block) + + // export file data with default settings + const originalGet = block.get.bind(block) + + const expectedInvocations: string[] = [] + + for (const cid of children) { + expectedInvocations.push(`${cid.toString()}-start`) + expectedInvocations.push(`${cid.toString()}-end`) + } + + const actualInvocations: string[] = [] + + block.get = async (cid) => { + actualInvocations.push(`${cid.toString()}-start`) + + // introduce a small delay - if running in parallel actualInvocations will + // be: + // `foo-start`, `bar-start`, `baz-start`, `foo-end`, `bar-end`, `baz-end` + // if in series it will be: + // `foo-start`, `foo-end`, `bar-start`, `bar-end`, `baz-start`, `baz-end` + await delay(1) + + actualInvocations.push(`${cid.toString()}-end`) + + return originalGet(cid) + } + + const blockReadSpy = Sinon.spy(block, 'get') + await drain(directory.content({ + blockReadConcurrency: 1 + })) + + // blocks should be loaded in default order - a whole level of sibling nodes at a time + expect(blockReadSpy.getCalls().map(call => call.args[0].toString())).to.deep.equal( + children.map(link => link.toString()) + ) + + expect(actualInvocations).to.deep.equal(expectedInvocations) + }) }) diff --git a/packages/ipfs-unixfs-exporter/typedoc.json b/packages/ipfs-unixfs-exporter/typedoc.json index 3be48369..f599dc72 100644 --- a/packages/ipfs-unixfs-exporter/typedoc.json +++ b/packages/ipfs-unixfs-exporter/typedoc.json @@ -1,6 +1,5 @@ { "entryPoints": [ "./src/index.ts" - ], - "readme": "none" + ] } diff --git a/packages/ipfs-unixfs-importer/CHANGELOG.md b/packages/ipfs-unixfs-importer/CHANGELOG.md index 8c4ff20e..db8c9ac2 100644 --- a/packages/ipfs-unixfs-importer/CHANGELOG.md +++ b/packages/ipfs-unixfs-importer/CHANGELOG.md @@ -1,3 +1,21 @@ +## ipfs-unixfs-importer [15.2.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.2.3...ipfs-unixfs-importer-15.2.4) (2024-01-19) + + +### Trivial Changes + +* update project config ([125f4d7](https://github.com/ipfs/js-ipfs-unixfs/commit/125f4d7dba9e04914f16560fb1f5002e298eaba1)) + + +### Dependencies + +* **dev:** bump aegir from 41.3.5 to 42.2.2 ([#399](https://github.com/ipfs/js-ipfs-unixfs/issues/399)) ([9d6c7cb](https://github.com/ipfs/js-ipfs-unixfs/commit/9d6c7cb9904e40e1fb4d92735fac62f5bc04559e)) + + + +### Dependencies + +* **ipfs-unixfs:** upgraded to 11.1.3 + ## [ipfs-unixfs-importer-v15.2.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-v15.2.2...ipfs-unixfs-importer-v15.2.3) (2023-12-28) diff --git a/packages/ipfs-unixfs-importer/README.md b/packages/ipfs-unixfs-importer/README.md index c8544ca3..7ac03ff0 100644 --- a/packages/ipfs-unixfs-importer/README.md +++ b/packages/ipfs-unixfs-importer/README.md @@ -1,14 +1,29 @@ -# ipfs-unixfs-importer +# ipfs-unixfs-importer [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-unixfs/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) > JavaScript implementation of the UnixFs importer used by IPFS # About + + ## Example Let's create a little directory to import: @@ -105,11 +120,3 @@ Please be aware that all interactions related to this repo are subject to the IP Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) - -[blockstore]: https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-blockstore#readme - -[UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs - -[IPLD]: https://github.com/ipld/js-ipld - -[CID]: https://github.com/multiformats/js-cid diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 0c837827..47793b6f 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -1,9 +1,9 @@ { "name": "ipfs-unixfs-importer", - "version": "15.2.3", + "version": "15.2.4", "description": "JavaScript implementation of the UnixFs importer used by IPFS", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/master/packages/ipfs-unixfs-importer#readme", + "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-importer#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/js-ipfs-unixfs.git" @@ -74,30 +74,31 @@ "dep-check": "aegir dep-check" }, "dependencies": { - "@ipld/dag-pb": "^4.0.0", - "@multiformats/murmur3": "^2.0.0", + "@ipld/dag-pb": "^4.1.0", + "@multiformats/murmur3": "^2.1.8", "err-code": "^3.0.1", - "hamt-sharding": "^3.0.0", - "interface-blockstore": "^5.0.0", - "interface-store": "^5.0.1", + "hamt-sharding": "^3.0.6", + "interface-blockstore": "^5.2.10", + "interface-store": "^5.1.8", "ipfs-unixfs": "^11.0.0", - "it-all": "^3.0.2", - "it-batch": "^3.0.2", - "it-first": "^3.0.2", - "it-parallel-batch": "^3.0.1", - "multiformats": "^13.0.0", + "it-all": "^3.0.4", + "it-batch": "^3.0.4", + "it-first": "^3.0.4", + "it-parallel-batch": "^3.0.4", + "multiformats": "^13.1.0", "progress-events": "^1.0.0", - "rabin-wasm": "^0.1.4", - "uint8arraylist": "^2.4.3", - "uint8arrays": "^5.0.0" + "rabin-wasm": "^0.1.5", + "uint8arraylist": "^2.4.8", + "uint8arrays": "^5.0.3" }, "devDependencies": { - "aegir": "^42.2.2", - "blockstore-core": "^4.0.1", - "it-last": "^3.0.2", + "aegir": "^42.2.5", + "blockstore-core": "^4.4.0", + "it-last": "^3.0.4", "wherearewe": "^2.0.1" }, "browser": { "fs": false - } + }, + "sideEffects": false } diff --git a/packages/ipfs-unixfs/CHANGELOG.md b/packages/ipfs-unixfs/CHANGELOG.md index 5bf1d80e..837f7b11 100644 --- a/packages/ipfs-unixfs/CHANGELOG.md +++ b/packages/ipfs-unixfs/CHANGELOG.md @@ -1,3 +1,15 @@ +## ipfs-unixfs [11.1.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.1.3...ipfs-unixfs-11.1.4) (2024-04-05) + + +### Bug Fixes + +* add sideEffects false to package.json to enable tree shaking ([#402](https://github.com/ipfs/js-ipfs-unixfs/issues/402)) ([aea58c4](https://github.com/ipfs/js-ipfs-unixfs/commit/aea58c40a4a2457ddf44454befa1eb25d4caa016)) + + +### Trivial Changes + +* rename master to main ([0cdfcd6](https://github.com/ipfs/js-ipfs-unixfs/commit/0cdfcd674513b21aab7e27b446a6f2181c9ba842)) + ## ipfs-unixfs [11.1.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.1.2...ipfs-unixfs-11.1.3) (2024-01-19) diff --git a/packages/ipfs-unixfs/README.md b/packages/ipfs-unixfs/README.md index 53b8db35..eb6a4ccc 100644 --- a/packages/ipfs-unixfs/README.md +++ b/packages/ipfs-unixfs/README.md @@ -1,14 +1,29 @@ -# ipfs-unixfs +# ipfs-unixfs [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-unixfs/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-unixfs/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml?query=branch%3Amain) > JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG) # About + + This module contains the protobuf definition of the UnixFS data structure found at the root of all UnixFS DAGs. The UnixFS spec can be found in the [ipfs/specs repository](http://github.com/ipfs/specs) diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index d190ddf2..fdeb9c62 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -1,9 +1,9 @@ { "name": "ipfs-unixfs", - "version": "11.1.3", + "version": "11.1.4", "description": "JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG)", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/master/packages/ipfs-unixfs#readme", + "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs#readme", "repository": { "type": "git", "url": "git+https://github.com/ipfs/js-ipfs-unixfs.git" @@ -55,15 +55,16 @@ }, "dependencies": { "err-code": "^3.0.1", - "protons-runtime": "^5.0.0", - "uint8arraylist": "^2.4.3" + "protons-runtime": "^5.4.0", + "uint8arraylist": "^2.4.8" }, "devDependencies": { - "aegir": "^42.2.2", - "protons": "^7.0.2", - "uint8arrays": "^5.0.0" + "aegir": "^42.2.5", + "protons": "^7.5.0", + "uint8arrays": "^5.0.3" }, "browser": { "fs": false - } + }, + "sideEffects": false } diff --git a/packages/ipfs-unixfs/typedoc.json b/packages/ipfs-unixfs/typedoc.json index 3be48369..f599dc72 100644 --- a/packages/ipfs-unixfs/typedoc.json +++ b/packages/ipfs-unixfs/typedoc.json @@ -1,6 +1,5 @@ { "entryPoints": [ "./src/index.ts" - ], - "readme": "none" + ] }