Various Fetch / Downloading functions or Objects

fetchArrayBuffer.js

async function fetchArrayBuffer( url ){
    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    // Validate the connection, else stop & return null
    const res = await window.fetch( url );
    if( res.status !== 200 ){
        return null;
    }

    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    // Download Buffer
    const buf = await res.arrayBuffer();
    if( !buf ){
        console.warn('Unable to download array buffer: ' + url);
        return null;
    }

    return buf;
}

fetchAsyncTexture.js

import { Texture, ClampToEdgeWrapping, RepeatWrapping } from 'three';
import fetchImage from './fetchImage.js';

export default function fetchAsyncTexture( url, flipY=true, isRepeat=false ){
    const tex       = new Texture();
    tex.wrapT       = tex.wrapS = ( isRepeat )? ClampToEdgeWrapping : RepeatWrapping;
    tex.flipY       = flipY;

    fetchImage( url ).then( ( img )=>{
        tex.image       = img;
        tex.needsUpdate = true;
    });

    return tex;
}


export async function fetchTexture(
  url: string,
  props?: {flipY?: boolean, isRepeat?: boolean, SRGB?: boolean},
): Promise<Texture | void> {
  const p = {
    flipY: true,
    repeat: true,
    SRGB: false,
    ...props,
  };

  // Get response
  const res = await window.fetch(url);
  if (!res.ok) {
    throw err(res.status.toString());
  }

  // Download Binary
  const blob = await res.blob();
  if (!blob) {
    throw new Error('Unable to download image blob');
  }

  if (blob.size === 0) {
    throw new Error('Zero bytes was streamed');
  }

  // Convert to image
  const img = await window.createImageBitmap(blob, {
    colorSpaceConversion: 'none',
    imageOrientation: p.flipY ? 'flipY' : 'none',
  });

  // Setup ThreeJS Texture
  const tex = new THREE.Texture(img);
  tex.wrapT = tex.wrapS = p.repeat
    ? THREE.RepeatWrapping
    : THREE.ClampToEdgeWrapping;

  tex.colorSpace = p.SRGB ? THREE.SRGBColorSpace : THREE.LinearSRGBColorSpace;
  tex.needsUpdate = true; // Needed, else it may render as black
  return tex;
}

fetchAudio.js

async function fetchAudio2(url: string): Promise<AudioBuffer | void> {
  // Get response
  const res = await window.fetch(url);
  if (!res.ok) {
    throw err(res.status.toString());
  }

  // Download Binary
  const aryBuf = await res.arrayBuffer();
  if (!aryBuf) {
    throw err('Unable to download audio array buffer');
  }

  if (aryBuf.byteLength === 0) {
    throw err('Zero bytes for audio download');
  }

  // Decode Audio
  const ctx = new AudioContext();
  const audio: AudioBuffer = await ctx.decodeAudioData(aryBuf);
  if (audio.length === 0) {
    throw err('Audio decoded with zero length');
  }

  // const playSound = ctx.createBufferSource();
  // playSound.buffer = audio;
  // playSound.connect(ctx.destination);
  // playSound.start(ctx.currentTime);

  return audio;
}

async function fetchAudio(url: string): Promise<Any | void> {
  // Get response
  const res = await window.fetch(url);
  if (!res.ok) {
    throw err(res.status.toString());
  }

  // Download Binary
  const blob = await res.blob();
  if (!blob) {
    throw err('Unable to download audio array buffer');
  }

  if (blob.size === 0) {
    throw err('Zero bytes for audio download');
  }

  const urlObject = window.URL.createObjectURL(blob);

  return urlObject;
}

FetchBatch.js

function nanoID( t=21 ){
    const r = crypto.getRandomValues( new Uint8Array( t ) );
    let n, e = '';
    for( ;t--; ){
        n  = 63 & r[ t ];
        e += ( n < 36 )? n.toString( 36 ) : 
             ( n < 62 )? ( n - 26 ).toString( 36 ).toUpperCase() : 
             ( n < 63 )? '_' : '-';
    }
    return e;
}

/*
    const fetchBatch = new FetchBatch();
    fetchBatch.onDownload = itm => console.log( itm );
    fetchBatch.onComplete = list => console.log( list );

    fetchBatch
        .add( 'http://blabla.bin', 'buffer', 'extraDataForCallback' )
        .add( 'http://blabla.bin2', 'buffer', 'extraDataForCallback2' )
        .start();
*/
export default class FetchBatch{
    // #region MAIN
    queue        = [];          // List of items to download
    listComplete = [];          // List of items that are done
    asyncLimit   = 2;           // How many active downloads
    onDownload   = null;        // Callback for each download
    onComplete   = null;        // Callback when all the downloads are done
    _abortStack  = new Map();   // Save Abort Controllers, Also doubles as how many fetches are currently active
    _itemCount   = 0;           // How many items in the batch
    _doneCount   = 0;           // How many items done being processed
    _isRunning   = false;       // Is Downloading queue currently running
    // #endregion

    // #region METHODS
    getSize(){ return this.queue.length; };

    abort(){
        for( const v of this._abortStack.values() ) v.abort();
        this._abortStack.clear();
        return this;
    }

    /** type = buffer | json */
    add( url, type, extra=null ){
        if( !this._isRunning ){
            this.queue.push( { url, type, extra, payload:null } );   // Save to the queue
            this._itemCount++;
        }
        return this;
    }

    start(){
        if( this._isRunning ) return this;
        this._isRunning = true;

        const min = Math.min( this.asyncLimit, this.queue.length );
        for( let i = 0; i < min; i++ ){
            this._next( this.queue.shift() );
        }

        return this;
    }
    // #endregion

    async _next( itm ){
        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        if( !itm ){
            if( this._doneCount === this._itemCount ){
                this._isRunning = false;
                if( this.onComplete ) this.onComplete( this.listComplete );
            }
            return;
        }

        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        const abortId   = nanoID( 12 );
        const abortCtrl = new AbortController();
        this._abortStack.set( abortId, abortCtrl );

        try{
            const res = await fetch( itm.url, { signal: abortCtrl.signal } );
            if( res.status === 200 ){
                let payload;

                switch( itm.type ){
                    case 'buffer'   : itm.payload = await res.arrayBuffer();    break;
                    case 'json'     : itm.payload = await res.json();           break;
                }

                this.listComplete.push( itm );
                if( this.onDownload ) this.onDownload( itm );
            }else{
                console.error( 'Error Downloading: %s for %s', res.status, itm.url );
            }
        }catch( ex ){
            if( ex.name !== 'AbortError' ){
                console.error( 'Error downloading arraybuffer/json :', ex.message );
                console.error( '---', itm.url );
                console.error( '--- stack trace', ex.stack );
            }
        }

        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        // Download the next item on the queue.
        this._doneCount++;
        this._abortStack.delete( abortId );
        this._next( this.queue.shift() );
    }
}

fetchCancelProgress.js

function fetchArrayBufferProgress(
  url: string,
  onProgress: (loaded: number, total: number) => void,
): [Promise<ArrayBuffer | null>, AbortController] {
  const abortCtrl = new AbortController();
  const promise = window
    .fetch(url, {signal: abortCtrl.signal})
    .then(async res => {
      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      // Validate the connection, else stop & return null
      if (res.status !== 200) {
        return null;
      }

      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      // Stream Data
      const reader = res.body.getReader();
      const contentBytes = parseInt(res.headers.get('Content-Length'), 10);
      const chunks: Array<Uint8Array> = [];
      let loadedBytes = 0;
      let chunk: {done: boolean, value: Uint8Array};

      while (true) {
        // eslint-disable-next-line no-await-in-loop
        chunk = await reader.read();
        if (chunk.done) {
          break;
        }

        chunks.push(chunk.value);
        loadedBytes += chunk.value.byteLength;

        onProgress(loadedBytes, contentBytes);
      }

      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      // Merge chunks into single arraybuffer
      const byteArray = new Uint8Array(loadedBytes);
      let bIdx = 0;
      for (const c of chunks) {
        byteArray.set(c, bIdx);
        bIdx += c.length;
      }

      return byteArray.buffer;
    });

  return [promise, abortCtrl];
}

fetchImage.js

async function fetchImage( url ){
    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    // Get response
    const res = await fetch( url );
    if( !res.ok ){ Promise.reject( new Error( res.status ) ); return; } // throw new Error(400);

    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    // Download Binary
    const blob = await res.blob();
    if( !blob ){ Promise.reject( new Error( 'Unable to download image blob' ) ); return; }

    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    // Convert to image
    return await window.createImageBitmap( blob );
}



function fetchImage( url ){
    return new Promise( async ( resolve, reject )=>{
        const img        = new Image();
        img.crossOrigin	 = 'anonymous';
        img.onload       = ()=>{ resolve( img ); };
        img.onerror      = ()=>{ reject( 'Error loading object url into image' ); };
        img.src          = url;
    });
}

function fetchImage( url ){
    return new Promise( async ( resolve, reject )=>{
        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        // Get response
        const res = await fetch( url );
        if( !res.ok ){ reject( res.status ); return; }

        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        // Download Binary
        const blob = await res.blob();
        if( !blob ){ reject( 'Unable to download image blob' ); return; }

        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        // Convert to image
        // TODO: look into window.createImageBitmap(blob);
        const obj  = URL.createObjectURL( blob );
        const img  = new Image();
    
        img.crossOrigin	 = 'anonymous';
        img.onload       = ()=>{ URL.revokeObjectURL( obj ); resolve( img ); };
        img.onerror      = ()=>{ URL.revokeObjectURL( obj ); reject( 'Error loading object url into image' ); };
        img.src          = obj;
    });
}

fetchJson.js

export default function fetchJson( url ){
    return new Promise( async ( resolve, reject )=>{
        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        // Get response
        const res = await fetch( url );
        if( !res.ok ){ reject( res.status ); return; }

        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        // Download Binary
        const json = await res.json();
        if( !json ){ reject( 'Unable to download json' ); return; }

        resolve( json );
    });
}

fetchProgress.js

async function fetchProgress( url, onProgress=null ){
    const res = await fetch( url );
    if( res.status !== 200 ) return null;

    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    // Download Data
    const reader        = res.body.getReader();
    const contentbytes  = parseInt( res.headers.get( 'Content-Length' ) );
    const chunks        = [];
    let loadedBytes     = 0;
    let chunk;

    while( true ){
        chunk = await reader.read();
        if( chunk.done ) break;

        chunks.push( chunk.value );
        loadedBytes += chunk.value.byteLength;

        if( onProgress ) onProgress( ( loadedBytes / contentbytes ) * 100, loadedBytes, contentbytes );
        
        console.log( 'Loaded', loadedBytes, contentbytes,  ( loadedBytes / contentbytes ) * 100 )
    }

    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    // Merge chunks into a single ArrayBuffer
    const byteArray = new Uint8Array( loadedBytes );
    let   bIdx      = 0;
    for( let c of chunks ){
        byteArray.set( c, bIdx );
        bIdx += c.length;
    }

    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    //let toText = new TextDecoder( 'utf-8' ).decode( byteArray );
    return byteArray.buffer;
}

FetchQueue.js

function nanoID( t=21 ){
    const r = crypto.getRandomValues( new Uint8Array( t ) );
    let n, e = '';
    for( ;t--; ){
        n  = 63 & r[ t ];
        e += ( n < 36 )? n.toString( 36 ) : 
             ( n < 62 )? ( n - 26 ).toString( 36 ).toUpperCase() : 
             ( n < 63 )? '_' : '-';
    }
    return e;
}

/*
    const fetchQueue = new FetchQueue();
    fetchQueue.push( 
        'http://blabla.bin', 
        'buffer', 
        ( payload, url, extra )=>{}, 
        'extraDataForCallback'
    );
*/
export default class FetchQueue{
    // #region MAIN
    queue       = [];           // List of items to download
    asyncLimit  = 2;            // How many active downloads
    _abortStack = new Map();    // Save Abort Controllers, Also doubles as how many fetches are currently active
    // #endregion

    // #region METHODS
    getSize(){ return this.queue.length; };

    abort(){
        for( const v of this._abortStack.values() ) v.abort();
        this._abortStack.clear();
        return this;
    }

    clearQueue(){ this.queue.length = 0; return this; }

    /** type = buffer | json */
    push( url, type, cb, extra=null ){
        this.queue.push( { url, type, cb, extra } );   // Save to the queue
        this._next();                                  // Start download process if one hasn't started
        return this;
    }
    // #endregion

    async _next(){
        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        if( this._abortStack.size >= this.asyncLimit || this.queue.length === 0 ) return;

        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        const abortId   = nanoID( 12 );
        const abortCtrl = new AbortController();
        this._abortStack.set( abortId, abortCtrl );

        let itm;
        try{
            itm = this.queue.shift();
            const res = await fetch( itm.url, { signal: abortCtrl.signal } );
            if( res.status === 200 ){
                let payload;

                switch( itm.type ){
                    case 'buffer'   : payload = await res.arrayBuffer();    break;
                    case 'json'     : payload = await res.json();           break;
                }

                itm.cb( payload, itm.url, itm.extra );
            }else{
                console.error( 'Error Downloading: %s for %s', res.status, itm.url );
            }
        }catch( ex ){
            if( ex.name !== 'AbortError' ){
                console.error( 'Error downloading arraybuffer/json :', ex.message );
                console.error( '---', itm.url );
                console.error( '--- stack trace', ex.stack );
            }
        }

        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        // Download the next item on the queue.
        this._abortStack.delete( abortId );
        this._next();
    }
}

fetchTexture.js

import * as THREE from 'three';
import fetchImage from './fetchImage.js';

export default async function fetchTexture( url, flipY=true, isRepeat=false ){
    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    // Download image
    const img = await fetchImage( url );
    if( !img ) return null;

    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    // Make it a texture
    const tex       = new THREE.Texture( img );
    tex.wrapT       = tex.wrapS = ( isRepeat )? THREE.ClampToEdgeWrapping : THREE.RepeatWrapping;
    tex.flipY       = flipY;
    tex.colorSpace  = THREE.LinearSRGBColorSpace; // THREE.SRGBColorSpace, THREE.NoColorSpace
    tex.needsUpdate = true; // Needed, else it may render as black
    return tex;
}