repair-drizzle-migration-conflict.ts

// MIT licensed (© 2024)
// https://opensource.org/license/mit
//
// Source: https://gist.github.com/gburtini/7e34842c567dd80ee834de74e7b79edd

import fs from "fs";

import config from "../../drizzle.config";
import path from "path";
import { exec, execSync } from "child_process";

if (!config.out || !fs.existsSync(config.out)) {
  console.error(`Directory ${config.out} does not exist`);
  console.error("Maybe your drizzle.config.ts can't be found.");
  process.exit(1);
}

const files = fs.readdirSync(config.out).sort();
const rawJournal = fs.readFileSync(
  path.join(config.out, "meta", "_journal.json"),
  "utf8",
);

// a conflict is determined when two files have the same migration number
function findAllConflict(files: string[]) {
  const conflicts = [];

  let foundConflict = false;
  let lastMigration = null;

  for (const file of files) {
    const match = file.match(/^(\d+)_/);
    if (!match?.[1]) continue;
    const migration = parseInt(match[1]);

    if (migration === lastMigration || foundConflict) {
      // everything after the first conflict is a conflict
      foundConflict = true;
      conflicts.push(file);
    }
    lastMigration = migration;
  }
  return conflicts;
}

const conflicts = findAllConflict(files);
const firstConflict = conflicts[0];
if (!firstConflict) {
  console.log("No migration conflicts found.");
  process.exit(0);
}

console.error(`Migration conflict found starting at ${pad(firstConflict)}`);

// if we have a journal, it may have git conflict markers. don't touch it if it doesn't, so we don't accidentally throw anything away.
if (rawJournal.includes("<<<<<<<")) {
  if (process.env.FORCE_FIX) {
    console.log("Resetting journal file to the state of the current branch.");
    execSync(`git checkout --theirs ${path.join(config.out, "meta")}`);
  } else {
    console.error("Journal file contains git conflict markers.");
    console.error("Restore the journal to the state _before_ this change.");
    console.error(
      "That is, return to the state that was in main or the parent branch.",
    );
    console.error("As a rule, never merge anything in the meta directory.");
    process.exit(1);
  }
}

// read the journal to identify which migrations to keep
const reloadedJournal = fs.readFileSync(
  path.join(config.out, "meta", "_journal.json"),
  "utf8",
);

const journal = JSON.parse(reloadedJournal) as {
  entries: { idx: number; tag: string }[];
};
for (const conflict of conflicts) {
  const [idx] = conflict.split("_");
  const tag = conflict.replace(".sql", "");
  if (!idx || !tag) {
    console.error(`Could not parse migration number and tag from ${conflict}`);
    process.exit(1);
  }

  const journalEntry = journal.entries.find(
    (entry) => entry.idx === parseInt(idx),
  ) as {
    idx: number;
    tag: string;
  };

  if (!journalEntry) {
    console.error(`Migration ${pad(idx)} not found in journal.`);
    if (process.env.FORCE_FIX) {
      execSync("rm -f " + path.join(config.out, conflict));
      console.log(`Removed migration ${pad(idx)}: ${conflict}`);
      continue;
    } else {
      process.exit(1);
    }
  } else if (journalEntry.tag === tag) {
    console.log(`Keeping migration ${pad(idx)}: ${journalEntry.tag}`);
    continue;
  } else {
    if (process.env.FORCE_FIX) {
      execSync("rm -f " + path.join(config.out, conflict));
      console.log(`Removed migration ${pad(idx)}: ${conflict}`);
      continue;
    } else {
      console.error(
        `Migration ${pad(idx)} is tagged ${journalEntry.tag} in journal.`,
      );
      console.error(`Expected tag ${tag} for migration ${pad(idx)}.`);

      process.exit(1);
    }
  }
}

if (process.env.FORCE_FIX) {
  execSync("pnpm run db:generate");
  console.log("All conflicts resolved. Migrations regenerated.");
} else {
  console.log("All conflicts resolved. Regenerate migrations for your changes.");
}

console.log(
  "When you open the PR, ensure there are no changes to historical entries in the journal or meta snapshot files.",
);

function pad(num: number | string, length = 4) {
  return num.toString().padStart(length, "0");
}

This is great!

I found a little bug - findAllConflict should include the previous file after its first match. As it is, it ignores whichever file is alphabetically first. I made a fork with the bugfix - gist link

I've also posted it on this drizzle issue for visibility - drizzle-team/drizzle-orm#2488 (comment) - I think something like this ought to be built into drizzle kit

Thank you so much for doing this!

Oh, interesting, we may have arbitrarily kept one, but I think you need to keep the one that is actually already in main (because it may have already run). I'd have to go play with it to be sure.

Agreed that it should be built into drizzle-kit.

For what it is worth, one thing we found useful to basically entirely mitigate this challenge was passing the --name parameter to drizzle-kit generate.

Doing this results in these conflicting properly (they no longer have random names, so 0001-migration conflicts with 0001-migration directly; vs. the default when they'd be 0001-random-word and 0001-another-word. So, our db:generate command is now actually:

"db:generate": "drizzle-kit generate --name migration",

In any case, glad you found some value in it.

Cross-posted here as well - drizzle-team/drizzle-orm#1104

If you agree with the bugfix & update your PR to reflect it, I'll update the comments to refer to your gist instead of my fork

Just seeing your comment - good to know about the --name parameter! All other db migration tools I've ever used auto-generated their migration names, so I prefer your solution

but I think you need to keep the one that is actually already in main (because it may have already run)

I agree, but that's not necessarily the one that gets excluded in this case - it just excludes the first one alphabetically (since you sort them on line 18)

And in any case, the conflicts array should include the one in "main" (i.e. the one in _journal.json) so it can be output on line 105

oh wait a minute - I totally missed your point about --name. If you just always use the name "migration" as in 0009-migration.sql or whatever then you wouldn't ever need to run this script with FORCE_FIX=false

Still, I think FORCE_FIX=true is useful, especially when used mid-merge

I guess there could be a modified version of this script that would work better with --name

• It would always force-fix
• In case of a merge conflict it would git checkout --theirs the .sql scripts as well as the meta stuff - the whole config.out directory

And I guess the argument against --name - using random filenames instead - is that uninformed/junior devs might try to manually reconcile merge conflicts in bad ways. Like there's a world where someone merges stuff into an existing migration.sql file and runs drizzle-kit generate and it works so they try to push it

It's a stretch, I know. The more I think about drizzle using random filenames, the less sense it makes. I can't think of another orm that does that - they're usually timestamp-based

I wrote a new script for usage with the --name parameter - gist. This is what I'll actually recommend to my team

I also found a new bug in the original script - snapshots without matching _journal.json entries should be deleted. Here's my patch