Merge CSV Files with same Column names and order

merge.ts

import { readdir, readFile, appendFile } from "fs/promises";
import { join } from "path";

export interface Progress {
  type: string; // Progress type
  timestamp: number;
  percentage?: number;
  message: string;
  meta?: Record<string, unknown>;
  isError?: boolean;
}

interface MergeCSVFilesProps {
  onProgress: (input: Progress) => void;
  sourceDirectory: string;
  targetFileName: string;
}

async function mergeCSVFiles({ onProgress, sourceDirectory, targetFileName }: MergeCSVFilesProps): Promise<void> {
  const basePath = app.isPackaged === true ? process.resourcesPath : process.cwd();
  const directoryPath = join(basePath, sourceDirectory);

  // Read the list of files in the directory
  const fileNames = await readdir(directoryPath);

  const csvFilePaths = fileNames
    .filter((fileName) => fileName.endsWith(".csv"))
    .map((fileName) => join(directoryPath, fileName));

  const targetPath = join(directoryPath, `${targetFileName}.csv`);
  let header = "";

  if (csvFilePaths.length > 0) {
    const firstFileData = await readFile(csvFilePaths[0], { encoding: "utf8" });
    header = `${firstFileData.split("\n")[0]}\n`;
    await appendFile(targetPath, header, { encoding: "utf-8" });
  }

  // Merge CSVs
  for (let i = 0; i < csvFilePaths.length; i += 1) {
    // eslint-disable-next-line no-await-in-loop
    const data = await readFile(csvFilePaths[i], { encoding: "utf8" });
    // eslint-disable-next-line no-await-in-loop
    await appendFile(targetPath, `${data.replace(header, "")}\n`, { encoding: "utf-8" });
    onProgress({
      timestamp: new Date().getTime(),
      type: ProgressType.MandiScraping,
      message: `Copied ${i + 1} out of ${csvFilePaths.length} mandi price files.`,
    });
  }
}