refactor(batching): use generator for memory usage

package.json

@@ -61,7 +61,6 @@
     "@web3-storage/blob-index": "^1.0.2",
     "@web3-storage/content-claims": "^5.1.0",
     "multiformats": "^13.1.0",
-    "multipart-byte-range": "^3.0.1",
     "p-defer": "^4.0.1",
     "p-queue": "^8.0.1",
     "uint8arraylist": "^2.4.8",

src/api.ts

@@ -1,11 +1,70 @@
 import { ByteView, MultihashDigest } from 'multiformats'
 import { Failure, Result, URI, DID } from '@ucanto/interface'
-import { Range } from 'multipart-byte-range'
 import { QueryError } from '@storacha/indexing-service-client/api'
 
 export { ByteView, MultihashDigest } from 'multiformats'
 export { Failure, Result, URI, DID, Principal } from '@ucanto/interface'
-export { Range, SuffixRange, AbsoluteRange } from 'multipart-byte-range'
+
+/**
+ * An absolute byte range to extract - always an array of two values
+ * corresponding to the first and last bytes (both inclusive). e.g.
+ * 
+ * ```
+ * [100, 200]
+ * ```
+ */
+export type AbsoluteRange = [first: number, last: number]
+
+/**
+ * A suffix byte range - always an array of one value corresponding to the
+ * first byte to start extraction from (inclusive). e.g.
+ * 
+ * ```
+ * [900]
+ * ```
+ * 
+ * If it is unknown how large a resource is, the last `n` bytes
+ * can be requested by specifying a negative value:
+ * 
+ * ```
+ * [-100]
+ * ```
+ */
+export type SuffixRange = [first: number]
+
+/**
+ * Byte range to extract - an array of one or two values corresponding to the
+ * first and last bytes (both inclusive). e.g.
+ * 
+ * ```
+ * [100, 200]
+ * ```
+ * 
+ * Omitting the second value requests all remaining bytes of the resource. e.g.
+ * 
+ * ```
+ * [900]
+ * ```
+ * 
+ * Alternatively, if it's unknown how large a resource is, the last `n` bytes
+ * can be requested by specifying a negative value:
+ * 
+ * ```
+ * [-100]
+ * ```
+ */
+export type Range = AbsoluteRange | SuffixRange
+
+export type ByteGetter = (range: AbsoluteRange) => Promise<ReadableStream<Uint8Array>>
+
+export interface EncoderOptions {
+  /** Mime type of each part. */
+  contentType?: string
+  /** Total size of the object in bytes. */
+  totalSize?: number
+  /** Stream queuing strategy. */
+  strategy?: QueuingStrategy<Uint8Array>
+}
 
 export interface Abortable {
   signal: AbortSignal

src/fetcher/batching.js

@@ -157,7 +157,7 @@ class BatchingFetcher {
  */
 export const create = (locator) => new BatchingFetcher(locator)
 
-/** @typedef {{range: import('multipart-byte-range').AbsoluteRange, digest: API.MultihashDigest, orig: API.Range | undefined}} ResolvedRange */
+/** @typedef {{range: API.AbsoluteRange, digest: API.MultihashDigest, orig: API.Range | undefined}} ResolvedRange */
 
 /**
  * Fetch blobs from the passed locations. The locations MUST share a common
@@ -188,7 +188,7 @@ export const fetchBlobs = withResultSpan('fetchBlobs',
         let found = false
         for (const l of s.location) {
           if (l.toString() === url.toString()) {
-          /** @type {import('multipart-byte-range').AbsoluteRange} */
+          /** @type {API.AbsoluteRange} */
             let resolvedRange = [s.range.offset, s.range.offset + s.range.length - 1]
             if (range) {
               const relRange = resolveRange(range, s.range.length)

src/fetcher/lib.js

@@ -1,7 +1,7 @@
 /**
- * @param {import('multipart-byte-range').Range} range
+ * @param {import('../api.js').Range} range
  * @param {number} totalSize
- * @returns {import('multipart-byte-range').AbsoluteRange}
+ * @returns {import('../api.js').AbsoluteRange}
  */
 export const resolveRange = (range, totalSize) => {
   let last = range[1]