S3 client direct uploading written in Svelte

Dropzone.svelte

<script>
  import { createEventDispatcher } from "svelte";
  import ButtonSubmit from "../ButtonSubmit.svelte";
  import QueueStatus from "./queueStatus";

  export let fileCount = 0;
  export let status;

  let dragActive = false;
  let fileSelect;
  let firstFile = null;
  let outerBox;
  let processing = false;

  const dispatch = createEventDispatcher();

  const scanFiles = (items) => {
    return new Promise((resolve) => {
      Promise.all(
        items.map((entry) => {
          if (entry.isFile) {
            return entry;
          } else if (entry.isDirectory) {
            return getDirectoryEntries(entry.createReader());
          }
        })
      ).then((entries) => {
        if (entries.flat().filter((e) => e.isFile).length === entries.length) {
          return resolve(entries);
        }

        return resolve(scanFiles(entries.flat()));
      });
    });
  };

  const getDirectoryEntries = (reader, entries = []) => {
    return new Promise((resolve) => {
      readEntries(reader).then((tail) => {
        if (tail.length === 0) return resolve(entries);

        return resolve(getDirectoryEntries(reader, [...entries, ...tail]));
      });
    });
  };

  const readEntries = async (directoryReader) => {
    return await new Promise((resolve, reject) => {
      directoryReader.readEntries(resolve, reject);
    });
  };

  const processDroppedItems = async (items) => {
    const entries = [...new Array(items.length)]
      .map((_, i) => {
        const item = items[i].webkitGetAsEntry();
        return item ?? null;
      })
      .filter((i) => i !== null);

    return await Promise.all(
      (await scanFiles(entries)).map((entry) => {
        return new Promise((resolve) => entry.file((f) => resolve(f)));
      })
    );
  };

  const handleDrop = async (event) => {
    processing = true;

    const files = await processDroppedItems(event.dataTransfer.items);

    dispatch("files-selected", files);
    firstFile = files.length > 1 ? null : files[0];
    dragActive = false;
    processing = false;
  };

  const handleDragleave = (event) => {
    if (event.relatedTarget === outerBox) dragActive = false;
  };

  const handleFileSelected = () => {
    const fileList = fileSelect.files ?? new FileList();
    const files = [...new Array(fileList.length)].map((_, i) => {
      return fileList[i];
    });

    dispatch("files-selected", files);
  };

  const handleOpenFileInput = () => fileSelect.click();

  const startUpload = () => dispatch("start-upload");

  const cancelUpload = () => {
    firstFile = null;
    dispatch("cancel-upload");
  };
</script>

<div bind:this={outerBox}>
  <div
    on:dragover|preventDefault={() => (dragActive = true)}
    on:dragleave={handleDragleave}
    on:drop|preventDefault={handleDrop}
    style="border: 4px dotted #ddd; {dragActive
      ? 'background-color: rgba(0, 0, 0, 0.02)'
      : 'background-color: rgba(0, 0, 0, 0.04)'}"
    class="d-flex flex-column justify-content-center align-items-center p-4 rounded-sm"
  >
    {#if status === QueueStatus.Running}
      <i class="fs-xxl fal fa-hourglass-end" />
    {:else}
      <i class="fs-xxl fal fa-upload" />
    {/if}

    {#if status === QueueStatus.Waiting || status === QueueStatus.Complete || status === QueueStatus.Failed}
      <div class="flex-grow-1 py-2 fs-lg">Drop files here</div>
      <div class="pb-4">or</div>
      <input
        bind:this={fileSelect}
        on:input={handleFileSelected}
        class="d-none"
        type="file"
        multiple
      />
      <ButtonSubmit
        on:click={handleOpenFileInput}
        classes="btn btn-primary"
        type="button"
        loading={processing}
        label="Select Files"
      />
    {:else}
      <div class="flex-grow-1 py-2 fs-lg">
        {status === QueueStatus.Running
          ? firstFile !== null
            ? firstFile.name
            : ""
          : "Ready"}
      </div>
      <div class="pb-4 {status === QueueStatus.Running ? 'invisible' : ''}">
        {fileCount}
        {fileCount > 1 ? "files" : "file"} to upload{firstFile !== null
          ? `: ${firstFile.name}`
          : ""}
      </div>
      <div class="d-flex align-items-center">
        {#if status === QueueStatus.Running}
          <div class="spinner-border text-primary" role="status">
            <span class="sr-only">Loading...</span>
          </div>
        {:else}
          <button
            on:click={startUpload}
            class="mr-1 btn btn-primary"
            type="button"
          >
            Start Upload
          </button>
          <button
            on:click={cancelUpload}
            class="ml-1 btn btn-outline-secondary"
            type="button"
          >
            Cancel
          </button>
        {/if}
      </div>
    {/if}
  </div>
</div>

Index.svelte

<script>
  import axios from "axios";
  import { createEventDispatcher, onMount } from "svelte";

  import Dropzone from "./Dropzone.svelte";
  import Progress from "./Progress.svelte";
  import QueueStatus from "./queueStatus";

  export let apiEndpoint;
  export let filePartSize = 6144000; // 6MB
  export let filePartRetryAttempts = 3;
  export let partParallelUploads = 2;
  export let parallelUploads = 1;

  let failedUploads = [];
  let fileCounts = {};
  let filesToUpload = [];
  let status = QueueStatus.Waiting;

  const dispatch = createEventDispatcher();

  onMount(() => {
    handleResetUpload();
  });

  const handleResetUpload = () => {
    failedUploads = [];
    fileCounts = {
      total: 0,
      completed: 0,
      totalSize: 0,
      completedSize: 0,
      completedProgress: {},
    };
    filesToUpload = [];
    status = QueueStatus.Waiting;
  };

  const handleFilesSelected = (event) => {
    if (status === QueueStatus.Running) return;

    const queue = event.detail;
    const size = +parallelUploads;

    filesToUpload = [...new Array(Math.ceil(queue.length / size))]
      .map((_, i) => queue.slice(i * size, i * size + size))
      .concat(filesToUpload);

    initFileCounts();
    status = QueueStatus.Ready;
  };

  const handleStartUpload = () => {
    status = QueueStatus.Running;
    dispatch("statusChanged", status);

    filesToUpload
      .reduce((promiseChain, queueGroup) => {
        return promiseChain.then((chainResults) => {
          return queueUpload(queueGroup, chainResults);
        });
      }, Promise.resolve([]))
      .then(completeResults);
  };

  const handleUploadProgress = (progressEvent, id) => {
    fileCounts.completedProgress[id] = {
      loaded: progressEvent.loaded,
      total: progressEvent.total,
    };

    fileCounts.completedSize = Object.keys(fileCounts.completedProgress).reduce(
      (acc, cur) => acc + fileCounts.completedProgress[cur].loaded,
      0
    );
  };

  const handleUploadRegress = (failedId) => {
    if (!fileCounts.completedProgress[failedId]) return;

    fileCounts.completedSize =
      fileCounts.completedSize - fileCounts.completedProgress[failedId].loaded;

    fileCounts.completedProgress = Object.keys(fileCounts.completedProgress)
      .filter((id) => id !== failedId)
      .reduce((acc, cur) => {
        acc[cur] = fileCounts.completedProgress[cur];

        return acc;
      }, {});
  };

  const initFileCounts = () => {
    failedUploads = [];
    fileCounts.completedProgress = {};
    fileCounts.completed = 0;
    fileCounts.completedSize = 0;
    fileCounts.total = filesToUpload.flat().length;
    fileCounts.totalSize = filesToUpload.flat().reduce((a, c) => a + c.size, 0);

    dispatch("uploadFailure", []);
  };

  const queueUpload = (queueGroup, chainResults) => {
    return Promise.all(queueGroup.map((file) => uploadFile(file))).then(
      (files) => {
        fileCounts.completed += files.length;

        return [...chainResults, ...files];
      }
    );
  };

  const completeResults = (results) => {
    filesToUpload = [];
    failedUploads = results.filter((r) => !r[1]);

    if (failedUploads.length > 0) {
      status = QueueStatus.Failed;
      dispatch("uploadFailure", failedUploads);
    } else {
      status = QueueStatus.Complete;
      dispatch("uploadSuccess", results);
    }

    dispatch("statusChanged", status);
  };

  const uploadFile = async (file) => {
    const parts = splitFileToParts(file);
    const size = +partParallelUploads;
    const { key, uploadId } = await fetchUploadMeta(file);

    return [...new Array(Math.ceil(parts.length / size))]
      .map((_, i) => parts.slice(i * size, i * size + size))
      .reduce((promiseChain, queueGroup) => {
        return promiseChain.then((chainResults) =>
          uploadPartGroup({
            file,
            key,
            parts: queueGroup,
            uploadId,
            results: chainResults,
          })
        );
      }, Promise.resolve([]))
      .then((completed) => {
        return completeFileUpload({
          file,
          key,
          parts: completed,
          uploadId,
        })
          .then(() => [null, file])
          .catch(() => [file, null]);
      })
      .catch(() => {
        return abortFileUpload({ key, uploadId }).finally(() => [file, null]);
      });
  };

  const uploadPartGroup = async ({ file, key, parts, uploadId, results }) => {
    const presignedParts = await Promise.all(
      parts.map((part) =>
        fetchUploadPresignedUrl({ key, part, uploadId }).then((data) => ({
          ...part,
          url: data.url,
          axiosConfig: {
            headers: { "Content-Type": file.type },
            onUploadProgress: (progress) =>
              handleUploadProgress(progress, `${file.name}_${part.number}`),
          },
        }))
      )
    );

    return Promise.all(
      presignedParts.map((part) =>
        [...Array(+filePartRetryAttempts)].reduce(async (promiseChain, _) => {
          const attempted = await promiseChain;

          return uploadPart({ file, key, part: attempted, uploadId });
        }, Promise.resolve(part))
      )
    ).then((parts) => [...results, ...parts]);
  };

  const uploadPart = ({ file, key, part, uploadId }) => {
    // if (process.env.MIX_ENV === "demo") return mockUploadPart(part);

    return new Promise((resolve, reject) => {
      if (part.uploadSuccessful) {
        resolve(part);
        return;
      }

      axios
        .put(part.url, part.blob, part.axiosConfig)
        .then((resp) => {
          part.etag = resp.headers.etag ?? "";
          part.uploadSuccessful = true;
          resolve(part);
        })
        .catch(() => {
          part.failed++;
          handleUploadRegress(`${file.name}_${part.number}`);

          if (part.failed >= +filePartRetryAttempts) {
            part.uploadSuccessful = false;
            reject(part);
          } else {
            resolve(part);
          }
        });
    });
  };

  const abortFileUpload = ({ key, uploadId }) => {
    return axios.delete(apiEndpoint, { data: { uploadId, key } });
  };

  const completeFileUpload = ({ file, key, parts, uploadId }) => {
    return axios.put(apiEndpoint, {
      uploadId,
      key,
      parts: parts.map((p) => ({ ETag: p.etag, PartNumber: p.number })),
      fileSize: file.size,
      fileName: file.name,
      fileType: file.type,
    });
  };

  const splitFileToParts = (file) => {
    const chunkTotal = Math.ceil(file.size / +filePartSize);

    return [...new Array(chunkTotal)].map((_, i) => {
      const part = i + 1;
      const partStart = (part - 1) * +filePartSize;
      const partEnd = part * +filePartSize;
      const partBlob =
        part < chunkTotal
          ? file.slice(partStart, partEnd)
          : file.slice(partStart);

      return {
        blob: partBlob,
        number: part,
        failed: 0,
        uploadSuccess: false,
      };
    });
  };

  const fetchUploadMeta = async (file) => {
    const resp = await axios.post(apiEndpoint, { filename: file.name });

    return { key: resp.data.key, uploadId: resp.data.upload_id };
  };

  const fetchUploadPresignedUrl = async ({ key, part, uploadId }) => {
    const resp = await axios.get(apiEndpoint, {
      params: { key, uploadId, partNumber: part.number },
    });

    return { url: resp.data.url };
  };

  /*
  const mockUploadPart = (part) => {
    const times = [500, 600, 700, 800, 900, 1000];

    return new Promise((resolve) => {
      setTimeout(() => {
        part.etag = "0000";
        part.uploadSuccessful = true;
        resolve(part);
      }, times[Math.floor(Math.random() * times.length)]);
    });
  };
  */
</script>

<Dropzone
  on:files-selected={handleFilesSelected}
  on:start-upload={handleStartUpload}
  on:cancel-upload={handleResetUpload}
  fileCount={fileCounts.total}
  {status}
/>

{#if status !== QueueStatus.Waiting && status !== QueueStatus.Ready}
  <Progress {...fileCounts} />
{/if}

Progress.svelte

<script>
  export let completed = 0;
  export let completedSize = 0;
  export let total = 0;
  export let totalSize = 0;

  $: percentComplete = Math.ceil((completedSize / totalSize) * 100);
</script>

<div class="text-center text-muted">
  <span class="fs-sm">
    {completed} / {total} files uploaded
  </span>
  <div class="mt-1 progress progress-md">
    <div
      class="progress-bar bg-primary-gradient"
      role="progressbar"
      style="width: {percentComplete}%;"
      aria-valuenow={percentComplete}
      aria-valuemin="0"
      aria-valuemax="100"
    >
      {percentComplete}%
    </div>
  </div>
</div>

queueStatus.js

export default {
  Waiting: 0,
  Ready: 1,
  Running: 2,
  Complete: 3,
  Failed: 4,
};