Skip to content

Commit 47f295b

Browse files
achingbrainachingbrain
authored
chore: tighten up input types (#133)
We normalise content to `AsyncIterator<Uint8Array>` so there's no point accepting more than that, it also leads to type ballooning around passing input to the importer.
1 parent 172548b commit 47f295b

26 files changed

+303
-205
lines changed

‎package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"version": "1.0.0",
44
"description": "JS implementation of the IPFS UnixFS",
55
"scripts": {
6-
"reset": "lerna run clean && rimraf packages/*/node_modules node_modules",
6+
"reset": "lerna run clean && rimraf packages/*/node_modules node_modules packages/*/package-lock.json package-lock.json",
77
"test": "lerna run test",
88
"coverage": "lerna run coverage",
99
"prepare": "lerna run prepare",
@@ -51,6 +51,7 @@
5151
"packages/*"
5252
],
5353
"engines": {
54+
"node": ">=14.0.0",
5455
"npm": ">=7.0.0"
5556
}
5657
}

‎packages/ipfs-unixfs-exporter/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,15 @@
2828
"url": "https://github.com/ipfs/js-ipfs-unixfs/issues"
2929
},
3030
"engines": {
31-
"node": ">=10.0.0",
32-
"npm": ">=4.0.0"
31+
"node": ">=14.0.0",
32+
"npm": ">=7.0.0"
3333
},
3434
"homepage": "https://github.com/ipfs/js-ipfs-unixfs#readme",
3535
"devDependencies": {
3636
"@types/mocha": "^8.2.1",
3737
"@types/sinon": "^9.0.10",
3838
"abort-controller": "^3.0.0",
39-
"aegir": "^32.0.0",
39+
"aegir": "^32.1.0",
4040
"copy": "^0.3.2",
4141
"crypto-browserify": "^3.12.0",
4242
"detect-node": "^2.0.4",

‎packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.js

+4-3
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ const {
2020
} = require('ipld-dag-pb')
2121
const blockApi = require('./helpers/block')
2222
const uint8ArrayConcat = require('uint8arrays/concat')
23+
const asAsyncIterable = require('./helpers/as-async-iterable')
2324

2425
/**
2526
* @typedef {import('cids')} CID
@@ -49,14 +50,14 @@ describe('exporter sharded', function () {
4950
const createShardWithFileNames = (numFiles, fileName) => {
5051
const files = new Array(numFiles).fill(0).map((_, index) => ({
5152
path: fileName(index),
52-
content: Uint8Array.from([0, 1, 2, 3, 4, index])
53+
content: asAsyncIterable(Uint8Array.from([0, 1, 2, 3, 4, index]))
5354
}))
5455

5556
return createShardWithFiles(files)
5657
}
5758

5859
/**
59-
* @param {{ path: string, content: Uint8Array }[] } files
60+
* @param {{ path: string, content: AsyncIterable<Uint8Array> }[] } files
6061
*/
6162
const createShardWithFiles = async (files) => {
6263
const result = await last(importer(files, block, {
@@ -88,7 +89,7 @@ describe('exporter sharded', function () {
8889

8990
const imported = await all(importer(Object.keys(files).map(path => ({
9091
path,
91-
content: files[path].content
92+
content: asAsyncIterable(files[path].content)
9293
})), block, {
9394
wrapWithDirectory: true,
9495
shardSplitThreshold: SHARD_SPLIT_THRESHOLD

‎packages/ipfs-unixfs-exporter/test/exporter-subtree.spec.js

+5-4
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ const last = require('it-last')
1212
const blockApi = require('./helpers/block')
1313
const randomBytes = require('it-buffer-stream')
1414
const uint8ArrayConcat = require('uint8arrays/concat')
15+
const asAsyncIterable = require('./helpers/as-async-iterable')
1516

1617
const ONE_MEG = Math.pow(1024, 2)
1718

@@ -36,7 +37,7 @@ describe('exporter subtree', () => {
3637
content: randomBytes(ONE_MEG)
3738
}, {
3839
path: './level-1/200Bytes.txt',
39-
content
40+
content: asAsyncIterable(content)
4041
}], block))
4142

4243
if (!imported) {
@@ -64,7 +65,7 @@ describe('exporter subtree', () => {
6465
content: randomBytes(ONE_MEG)
6566
}, {
6667
path: './level-1/200Bytes.txt',
67-
content
68+
content: asAsyncIterable(content)
6869
}, {
6970
path: './level-1/level-2'
7071
}], block))
@@ -121,12 +122,12 @@ describe('exporter subtree', () => {
121122
content: randomBytes(ONE_MEG)
122123
}, {
123124
path: './level-1/200Bytes.txt',
124-
content
125+
content: asAsyncIterable(content)
125126
}, {
126127
path: './level-1/level-2'
127128
}, {
128129
path: './level-1/level-2/200Bytes.txt',
129-
content
130+
content: asAsyncIterable(content)
130131
}], block))
131132

132133
if (!imported) {

‎packages/ipfs-unixfs-exporter/test/exporter.spec.js

+11-10
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ const blockApi = require('./helpers/block')
2525
const uint8ArrayFromString = require('uint8arrays/from-string')
2626
const uint8ArrayToString = require('uint8arrays/to-string')
2727
const uint8ArrayConcat = require('uint8arrays/concat')
28+
const asAsyncIterable = require('./helpers/as-async-iterable')
2829

2930
const ONE_MEG = Math.pow(1024, 2)
3031

@@ -79,7 +80,7 @@ describe('exporter', () => {
7980
async function addTestFile ({ file, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves }) {
8081
const result = await all(importer([{
8182
path,
82-
content: file
83+
content: asAsyncIterable(file)
8384
}], block, {
8485
strategy,
8586
rawLeaves,
@@ -195,7 +196,7 @@ describe('exporter', () => {
195196

196197
const files = await all(importer([{
197198
path: filePath,
198-
content: smallFile
199+
content: asAsyncIterable(smallFile)
199200
}], block))
200201

201202
const path = `/ipfs/${files[1].cid.toBaseEncodedString()}/${fileName}`
@@ -211,7 +212,7 @@ describe('exporter', () => {
211212

212213
const files = await all(importer([{
213214
path: filePath,
214-
content: smallFile
215+
content: asAsyncIterable(smallFile)
215216
}], block))
216217

217218
const path = `/ipfs/${files[1].cid.toBaseEncodedString()}/${fileName}`
@@ -613,7 +614,7 @@ describe('exporter', () => {
613614
it('exports a large file > 1mb imported with raw leaves', async () => {
614615
const imported = await first(importer([{
615616
path: '1.2MiB.txt',
616-
content: bigFile
617+
content: asAsyncIterable(bigFile)
617618
}], block, {
618619
rawLeaves: true
619620
}))
@@ -890,7 +891,7 @@ describe('exporter', () => {
890891

891892
const imported = await first(importer([{
892893
path: '200Bytes.txt',
893-
content: bigFile
894+
content: asAsyncIterable(bigFile)
894895
}], block, {
895896
rawLeaves: true
896897
}))
@@ -915,7 +916,7 @@ describe('exporter', () => {
915916
it('exports a raw leaf', async () => {
916917
const imported = await first(importer([{
917918
path: '200Bytes.txt',
918-
content: smallFile
919+
content: asAsyncIterable(smallFile)
919920
}], block, {
920921
rawLeaves: true
921922
}))
@@ -1022,7 +1023,7 @@ describe('exporter', () => {
10221023
it('exports a node with depth', async () => {
10231024
const imported = await all(importer([{
10241025
path: '/foo/bar/baz.txt',
1025-
content: uint8ArrayFromString('hello world')
1026+
content: asAsyncIterable(uint8ArrayFromString('hello world'))
10261027
}], block))
10271028

10281029
const exported = await exporter(imported[0].cid, ipld)
@@ -1033,13 +1034,13 @@ describe('exporter', () => {
10331034
it('exports a node recursively with depth', async () => {
10341035
const dir = await last(importer([{
10351036
path: '/foo/bar/baz.txt',
1036-
content: uint8ArrayFromString('hello world')
1037+
content: asAsyncIterable(uint8ArrayFromString('hello world'))
10371038
}, {
10381039
path: '/foo/qux.txt',
1039-
content: uint8ArrayFromString('hello world')
1040+
content: asAsyncIterable(uint8ArrayFromString('hello world'))
10401041
}, {
10411042
path: '/foo/bar/quux.txt',
1042-
content: uint8ArrayFromString('hello world')
1043+
content: asAsyncIterable(uint8ArrayFromString('hello world'))
10431044
}], block))
10441045

10451046
if (!dir) {

‎packages/ipfs-unixfs-exporter/test/helpers/as-async-iterable.js

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
'use strict'
2+
3+
/**
4+
* @param {Uint8Array | Uint8Array[]} arr
5+
*/
6+
async function * asAsyncIterable (arr) {
7+
if (!Array.isArray(arr)) {
8+
arr = [arr]
9+
}
10+
11+
yield * arr
12+
}
13+
14+
module.exports = asAsyncIterable

‎packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.js

+8-7
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ const blockApi = require('./helpers/block')
1515
const uint8ArrayFromString = require('uint8arrays/from-string')
1616
const uint8ArrayToString = require('uint8arrays/to-string')
1717
const uint8ArrayConcat = require('uint8arrays/concat')
18+
const asAsyncIterable = require('./helpers/as-async-iterable')
1819

1920
/**
2021
* @typedef {import('../src').UnixFSEntry} UnixFSEntry
@@ -37,7 +38,7 @@ describe('builder: directory sharding', () => {
3738
const content = uint8ArrayFromString('i have the best bytes')
3839
const nodes = await all(importer([{
3940
path: 'a/b',
40-
content
41+
content: asAsyncIterable(content)
4142
}], block, {
4243
shardSplitThreshold: Infinity // never shard
4344
}))
@@ -73,7 +74,7 @@ describe('builder: directory sharding', () => {
7374
it('yields a sharded dir', async () => {
7475
const nodes = await all(importer([{
7576
path: 'a/b',
76-
content: uint8ArrayFromString('i have the best bytes')
77+
content: asAsyncIterable(uint8ArrayFromString('i have the best bytes'))
7778
}], block, {
7879
shardSplitThreshold: 0 // always shard
7980
}))
@@ -95,7 +96,7 @@ describe('builder: directory sharding', () => {
9596
const content = 'i have the best bytes'
9697
const nodes = await all(importer([{
9798
path: 'a/b',
98-
content: uint8ArrayFromString(content)
99+
content: asAsyncIterable(uint8ArrayFromString(content))
99100
}], block, {
100101
shardSplitThreshold: Infinity // never shard
101102
}))
@@ -132,7 +133,7 @@ describe('builder: directory sharding', () => {
132133
const content = 'i have the best bytes'
133134
const nodes = await all(importer([{
134135
path: 'a/b',
135-
content: uint8ArrayFromString(content)
136+
content: asAsyncIterable(uint8ArrayFromString(content))
136137
}], block, {
137138
shardSplitThreshold: 0 // always shard
138139
}))
@@ -177,7 +178,7 @@ describe('builder: directory sharding', () => {
177178
for (let i = 0; i < maxDirs; i++) {
178179
yield {
179180
path: 'big/' + i.toString().padStart(4, '0'),
180-
content: uint8ArrayFromString(i.toString())
181+
content: asAsyncIterable(uint8ArrayFromString(i.toString()))
181182
}
182183
}
183184
}
@@ -196,7 +197,7 @@ describe('builder: directory sharding', () => {
196197
for (let i = 0; i < maxDirs; i++) {
197198
yield {
198199
path: 'big/' + i.toString().padStart(4, '0'),
199-
content: uint8ArrayFromString(i.toString())
200+
content: asAsyncIterable(uint8ArrayFromString(i.toString()))
200201
}
201202
}
202203
}
@@ -249,7 +250,7 @@ describe('builder: directory sharding', () => {
249250

250251
yield {
251252
path: dir.concat(i.toString().padStart(4, '0')).join('/'),
252-
content: uint8ArrayFromString(i.toString())
253+
content: asAsyncIterable(uint8ArrayFromString(i.toString()))
253254
}
254255

255256
pending--

‎packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.js

+5-4
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const blockApi = require('./helpers/block')
1313
const uint8ArrayFromString = require('uint8arrays/from-string')
1414
const uint8ArrayToString = require('uint8arrays/to-string')
1515
const uint8ArrayConcat = require('uint8arrays/concat')
16+
const asAsyncIterable = require('./helpers/as-async-iterable')
1617

1718
describe('import and export: directory', () => {
1819
const rootHash = 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK'
@@ -31,16 +32,16 @@ describe('import and export: directory', () => {
3132

3233
const source = [{
3334
path: 'a/b/c/d/e',
34-
content: uint8ArrayFromString('banana')
35+
content: asAsyncIterable(uint8ArrayFromString('banana'))
3536
}, {
3637
path: 'a/b/c/d/f',
37-
content: uint8ArrayFromString('strawberry')
38+
content: asAsyncIterable(uint8ArrayFromString('strawberry'))
3839
}, {
3940
path: 'a/b/g',
40-
content: uint8ArrayFromString('ice')
41+
content: asAsyncIterable(uint8ArrayFromString('ice'))
4142
}, {
4243
path: 'a/b/h',
43-
content: uint8ArrayFromString('cream')
44+
content: asAsyncIterable(uint8ArrayFromString('cream'))
4445
}]
4546

4647
const files = await all(importer(source, block))

‎packages/ipfs-unixfs-exporter/test/import-export.spec.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const loadFixture = require('aegir/utils/fixtures')
1313
const isNode = require('detect-node')
1414
const bigFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/1.2MiB.txt')
1515
const blockApi = require('./helpers/block')
16+
const asAsyncIterable = require('./helpers/as-async-iterable')
1617

1718
const { importer } = require('ipfs-unixfs-importer')
1819
const { exporter } = require('../src')
@@ -42,7 +43,7 @@ describe('import and export', function () {
4243

4344
it('imports and exports', async () => {
4445
const path = `${strategy}-big.dat`
45-
const values = [{ path: path, content: bigFile }]
46+
const values = [{ path: path, content: asAsyncIterable(bigFile) }]
4647

4748
// @ts-ignore
4849
for await (const file of importer(values, block, importerOptions)) {

0 commit comments

Comments
 (0)