Sketch of unixfs-importer API

api.v0.ts

import type { CID} from "multiformats"
import { Mtime } from "ipfs-unixfs"


export interface Lib {
  /**
   * Creates a directory importer that can be used to build up directory of
   * files. It takes an optional queuing strategy parameter that can be used
   * to specify how how many blocks can be queue internally before backpressure
   * is applied and writer is blocked. This allows users to specify how much
   * memory can be used by the import task.
   */
  createDirectoryImporter(strategy?:QueuingStrategy<Block>, options?:WriteOptions):DirectoryImporter
}

interface DirectoryImporter {
  /**
   * Writer provides an API that can be used to incrementally write it's files.
   * It is 
   */
  writer: DirectoryWriter

  /**
   * Stream of blocks that are created when writing this directory. Stream uses
   * queuing startegy speficied at construction. If it's intenral queue is full
   * writer will get blocked until blocks from this stream are consumed.
   */
  blocks: ReadableStream<Block>
}


/**
 * Writer interface allows user to add files to a directory. It will write
 * new blocks into importers block stream and return result containing file
 * CID other relevant metadata.
 * 
 * When importers internal queue is full writer will block until some blocks
 * are read (or discarded).
 */
interface DirectoryWriter {
  /**
   * Returns the desired size required to fill the writer's internal queue.
   */
  readonly desiredSize: number

  /**
   * Takes an object that implements a web File API and returns result that
   * contains corresponding CID (TODO: Should we also return all the block CIDs).
   * 
   * Attempt to ovewrite existing path will produce an OverwriteError.
   */

  write(file:File, options?:WriteOptions):Result<ImportedFile, OverwriteError>
  // implementation will look something like
  // async write(file:File, {chunker, builder, metadata}:WriteOptions=this.options) {
  //   const dagBuilder = builder.openFile(file.name, metadata)
  //   for await (const chunk of chunker.chunkFile(file.stream())) {
  //     const block = await createLeafFileBlock(chunk)
  //     await this.enqueue(block)
      
  //     const blocks = await dagBuilder.write({
  //       cid: block.cid,
  //       size: block.bytes.byteLength
  //     })

  //     for (const block of blocks) {
  //       await this.enqueue(block)
  //     }
  //   }
  //   const root = await dagBuilder.close()
  //   this.enqueue(block)
  //   // ....
  // }

  /**
   * Finalizes writer and returns root CID corresponding to this derectory.
   */
  close():Result<ImportedDirectory, CloseError>
}

interface ImportedEntry {
  cid: CID
  size: number
  path: string

  unixfs: FileNode | RawNode
}

interface ImportedDirectory extends ImportedEntry {
  type: "directory"
}

interface ImportedFile extends ImportedEntry {
  type: "file"
}

interface FileNode extends PBNode {
  type: 'file'
  filesize: number

  mode?: number
  mtime?: Mtime
}

interface RawNode extends PBNode {
  type: 'raw'
  data: Uint8Array
  filesize: number
}

interface PBNode {
  data: Uint8Array
  blocksizes: number[]

  hashType?: number
  fanout?: number
}
interface WriteOptions {
  chunker: Chunker
  
  builder: FileBuilder
}

interface Chunker {
  // Note: chunkFile can just be a generator that yields chunks, we just define
  // simpler FileChunker interface.
  chunkFile(file:File):FileChunker
}

interface FileChunker extends AsyncIterableIterator<Uint8Array> {
  next():Promise<{done:false, value:Uint8Array}|{done:true, value:void}>
  return():Promise<{done:true, value:void}>
}

/**
 * An interface for building an UnixFS node.
 */

interface FileBuilder {
  /**
   * Creates a file writer with a given metadata and path.
   */
  openFile(path:string, metadata:FileMetadata):FileDAGBuilder
}

interface FileMetadata {
  mode?: number
  mtime?: Mtime
}

/**
 * Interface for assembling file nodes.
 */
// TODO: I do not like current API which is really confusing. As
  // far as I can tell it takes stream of file blocks and combines them
  // into a file node.
  // https://github.com/ipfs/js-ipfs-unixfs/blob/273a141b5ee3805bd0ef2dc8ed7870f8c6c8a820/packages/ipfs-unixfs-importer/src/types.ts#L49
  //
  // - I believe only cid & size is enough for builder, passing other things
  //   could lead to increased memory consumption.
  // - reduce parameter is also seems fairly static and doesn't need to be passed
  //   in as far as I can tell.


  // TODO: Current implementation has some logic around whether chunker produced
  // a single block or not. And if only block is a raw block and file has a
  // metadata code pull data from blockstore to repackage it as pb node.
  // Need to make sure that we can support it in more cleaner manner.

interface FileDAGBuilder {
  /**
   * Importer will call this function for each file chunk (leaf node) with a
   * it's CID & size. File writer can create intermidiery blocks as necessary
   * and return them.
   */
  // TODO: The only reason this needs to be async (AFAIK) is because we want to
  // compute hash for the CID. It may make more sense to return `PBNode[]`
  // instead and let importer do the rest. On the othe hand to create root node
  // `close` would need to know CIDs of returned nodes so that may not be
  // ideal either.
  write(chunk:FileChunk): Result<Block[]>
  /**
   * Once all file chunks are written importer will call close so that writer
   * can produce a root file block.
   */
  close():Result<Block>
}


interface FileChunk {
  size: number
  cid: CID
}

// @see https://github.com/ipld/js-car/blob/a53d5c77d30f998b45a534e20d0f174574c58cd5/api.ts#L5
interface Block {
  cid: CID
  bytes: Uint8Array
}



// This is just better typed promise
interface Result<T extends unknown, X = Error> {
  catch <U = void>(handle:(error:X) => U | PromiseLike<U>):Result<U, never>
  then <U = void> (handle:(value:T) => U | PromiseLike<U>):Result<U, X>
  finally(handle:() => void): Result<T, X>
}

/**
 * Error is thrown on attempt to overwrite existing file.
 */
interface OverwriteError extends Error {
  path: string
}

interface CloseError extends Error {

}