Skip to content

Instantly share code, notes, and snippets.

@Gozala
Last active Jan 6, 2022
Embed
What would you like to do?
Sketch of unixfs-importer API
import type { CID} from "multiformats"
import { Mtime } from "ipfs-unixfs"
export interface Lib {
/**
* Creates a directory importer that can be used to build up directory of
* files. It takes an optional queuing strategy parameter that can be used
* to specify how how many blocks can be queue internally before backpressure
* is applied and writer is blocked. This allows users to specify how much
* memory can be used by the import task.
*/
createDirectoryImporter(strategy?:QueuingStrategy<Block>, options?:WriteOptions):DirectoryImporter
}
interface DirectoryImporter {
/**
* Writer provides an API that can be used to incrementally write it's files.
* It is
*/
writer: DirectoryWriter
/**
* Stream of blocks that are created when writing this directory. Stream uses
* queuing startegy speficied at construction. If it's intenral queue is full
* writer will get blocked until blocks from this stream are consumed.
*/
blocks: ReadableStream<Block>
}
/**
* Writer interface allows user to add files to a directory. It will write
* new blocks into importers block stream and return result containing file
* CID other relevant metadata.
*
* When importers internal queue is full writer will block until some blocks
* are read (or discarded).
*/
interface DirectoryWriter {
/**
* Returns the desired size required to fill the writer's internal queue.
*/
readonly desiredSize: number
/**
* Takes an object that implements a web File API and returns result that
* contains corresponding CID (TODO: Should we also return all the block CIDs).
*
* Attempt to ovewrite existing path will produce an OverwriteError.
*/
write(file:File, options?:WriteOptions):Result<ImportedFile, OverwriteError>
// implementation will look something like
// async write(file:File, {chunker, builder, metadata}:WriteOptions=this.options) {
// const dagBuilder = builder.openFile(file.name, metadata)
// for await (const chunk of chunker.chunkFile(file.stream())) {
// const block = await createLeafFileBlock(chunk)
// await this.enqueue(block)
// const blocks = await dagBuilder.write({
// cid: block.cid,
// size: block.bytes.byteLength
// })
// for (const block of blocks) {
// await this.enqueue(block)
// }
// }
// const root = await dagBuilder.close()
// this.enqueue(block)
// // ....
// }
/**
* Finalizes writer and returns root CID corresponding to this derectory.
*/
close():Result<ImportedDirectory, CloseError>
}
interface ImportedEntry {
cid: CID
size: number
path: string
unixfs: FileNode | RawNode
}
interface ImportedDirectory extends ImportedEntry {
type: "directory"
}
interface ImportedFile extends ImportedEntry {
type: "file"
}
interface FileNode extends PBNode {
type: 'file'
filesize: number
mode?: number
mtime?: Mtime
}
interface RawNode extends PBNode {
type: 'raw'
data: Uint8Array
filesize: number
}
interface PBNode {
data: Uint8Array
blocksizes: number[]
hashType?: number
fanout?: number
}
interface WriteOptions {
chunker: Chunker
builder: FileBuilder
}
interface Chunker {
// Note: chunkFile can just be a generator that yields chunks, we just define
// simpler FileChunker interface.
chunkFile(file:File):FileChunker
}
interface FileChunker extends AsyncIterableIterator<Uint8Array> {
next():Promise<{done:false, value:Uint8Array}|{done:true, value:void}>
return():Promise<{done:true, value:void}>
}
/**
* An interface for building an UnixFS node.
*/
interface FileBuilder {
/**
* Creates a file writer with a given metadata and path.
*/
openFile(path:string, metadata:FileMetadata):FileDAGBuilder
}
interface FileMetadata {
mode?: number
mtime?: Mtime
}
/**
* Interface for assembling file nodes.
*/
// TODO: I do not like current API which is really confusing. As
// far as I can tell it takes stream of file blocks and combines them
// into a file node.
// https://github.com/ipfs/js-ipfs-unixfs/blob/273a141b5ee3805bd0ef2dc8ed7870f8c6c8a820/packages/ipfs-unixfs-importer/src/types.ts#L49
//
// - I believe only cid & size is enough for builder, passing other things
// could lead to increased memory consumption.
// - reduce parameter is also seems fairly static and doesn't need to be passed
// in as far as I can tell.
// TODO: Current implementation has some logic around whether chunker produced
// a single block or not. And if only block is a raw block and file has a
// metadata code pull data from blockstore to repackage it as pb node.
// Need to make sure that we can support it in more cleaner manner.
interface FileDAGBuilder {
/**
* Importer will call this function for each file chunk (leaf node) with a
* it's CID & size. File writer can create intermidiery blocks as necessary
* and return them.
*/
// TODO: The only reason this needs to be async (AFAIK) is because we want to
// compute hash for the CID. It may make more sense to return `PBNode[]`
// instead and let importer do the rest. On the othe hand to create root node
// `close` would need to know CIDs of returned nodes so that may not be
// ideal either.
write(chunk:FileChunk): Result<Block[]>
/**
* Once all file chunks are written importer will call close so that writer
* can produce a root file block.
*/
close():Result<Block>
}
interface FileChunk {
size: number
cid: CID
}
// @see https://github.com/ipld/js-car/blob/a53d5c77d30f998b45a534e20d0f174574c58cd5/api.ts#L5
interface Block {
cid: CID
bytes: Uint8Array
}
// This is just better typed promise
interface Result<T extends unknown, X = Error> {
catch <U = void>(handle:(error:X) => U | PromiseLike<U>):Result<U, never>
then <U = void> (handle:(value:T) => U | PromiseLike<U>):Result<U, X>
finally(handle:() => void): Result<T, X>
}
/**
* Error is thrown on attempt to overwrite existing file.
*/
interface OverwriteError extends Error {
path: string
}
interface CloseError extends Error {
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment