pin balenaOS submodules

update_gitmodules.mjs

// update_gitmodules.mjs
import fs from 'fs/promises';
import path from 'path';
import { fileURLToPath } from 'url';
import readline from 'readline';
import { parse } from 'ini';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const githubOrg = 'balena-os';
const repoYamlMatch = 'yocto-based OS image';
const prBranch = 'kyle/update-gitmodules';

const rl = readline.createInterface({
  input: process.stdin,
  output: process.stdout
});

function prompt(question) {
  return new Promise((resolve) => {
    rl.question(question, resolve);
  });
}

function delay(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

async function checkRepoYaml(octokit, owner, repo) {
  try {
    const { data } = await octokit.repos.getContent({
      owner,
      repo,
      path: 'repo.yml',
    });
    const content = Buffer.from(data.content, 'base64').toString('utf-8');
    return content.includes(repoYamlMatch);
  } catch (error) {
    if (error.status === 404) {
      console.log(`${repo}: repo.yml not found, skipping...`);
      return false;
    }
    throw error;
  }
}

async function checkGitModules(octokit, owner, repo) {
  try {
    const { data } = await octokit.repos.getContent({
      owner,
      repo,
      path: '.gitmodules',
    });
    const content = Buffer.from(data.content, 'base64').toString('utf-8');
    
    const gitmodules = parse(content);

    for (const section in gitmodules) {
      if (section.startsWith('submodule ')) {
        const submodule = gitmodules[section];
        if (!submodule.branch) {
          return true;
        }
      }
    }

    return false;
  } catch (error) {
    if (error.status === 404) {
      return false;
    }
    throw error;
  }
}

async function findBranchesContainingCommit(octokit, owner, repo, commitSha) {
  try {
    const { data: branches } = await octokit.repos.listBranches({
      owner,
      repo,
      per_page: 100
    });

    const matchingBranches = [];

    for (const branch of branches) {
      const branchName = branch.name;
      let commitFound = false;
      let page = 1;

      while (!commitFound) {
        try {
          // Get the commit history for the branch, paginated to avoid large responses
          const { data: commits } = await octokit.repos.listCommits({
            owner,
            repo,
            sha: branchName,
            per_page: 100,
            page: page
          });

          // Check if the commitSha is in the current page of commits
          if (commits.some(commit => commit.sha === commitSha)) {
            console.log(`${repo}: Found commit ${commitSha} in branch ${branchName}`);
            matchingBranches.push(branchName);
            commitFound = true;
          }

          // If there are no more commits, break the loop
          if (commits.length === 0) {
            break;
          }

          page++;
        } catch (error) {
          console.log(`Error checking branch ${branchName} on page ${page}: ${error.message}`);
          break; // Exit the loop if an error occurs
        }
      }
    }

    return matchingBranches;
  } catch (error) {
    console.error(`Error finding branches containing commit: ${error.message}`);
    throw error;
  }
}

async function getSubmoduleBranch(octokit, logRepo, owner, repo, submodulePath, submoduleCommit, currentGitmodules) {
  console.log(`${logRepo}: ${repo}: Getting branches containing commit ${submoduleCommit}...`);

  try {
    const matchingBranches = await findBranchesContainingCommit(octokit, owner, repo, submoduleCommit);

    if (matchingBranches.length > 1) {
      console.log(`${logRepo}: ${repo}: Multiple matching branches found!`);
      console.log(`${logRepo}: ${repo}: Current .gitmodules content:`);
      console.log(currentGitmodules);

      console.log(`${logRepo}: ${repo}: Matching branches:`);
      console.log(`0) Skip this submodule`);
      matchingBranches.forEach((branch, index) => {
        console.log(`${index + 1}) ${branch}`);
      });

      let choice = await prompt('Enter your choice: ');
      let choiceIndex = parseInt(choice);

      while (isNaN(choiceIndex) || choiceIndex < 0 || choiceIndex > matchingBranches.length) {
        console.log('Invalid choice. Please enter a number.');
        choice = await prompt('Enter your choice: ');
        choiceIndex = parseInt(choice);
      }

      switch (choiceIndex) {
        case 0:
          return null;
        default:
          return matchingBranches[choiceIndex-1];
      }

    } else if (matchingBranches.length === 1) {
      return matchingBranches[0];
    }

    console.log(`${repo}: ${submodulePath}: Unable to determine branch!`);
    return null;

  } catch (error) {
    console.error(`Error getting branches for ${submodulePath}: ${error.message}`);
    return null;
  }
}

async function createOrResetBranch(octokit, owner, repo, branchName, baseBranch = 'master') {
  console.log(`${repo}: Creating or resetting branch ${branchName}...`);

  try {
    // Get the SHA of the latest commit on the base branch
    const { data: baseRef } = await octokit.git.getRef({
      owner,
      repo,
      ref: `heads/${baseBranch}`,
    });

    const baseSha = baseRef.object.sha;

    try {
      // Try to get the current branch (to check if it exists)
      await octokit.git.getRef({
        owner,
        repo,
        ref: `heads/${branchName}`,
      });

      // If the branch exists, update it to point to the latest commit of the base branch
      await octokit.git.updateRef({
        owner,
        repo,
        ref: `heads/${branchName}`,
        sha: baseSha,
        force: true,
      });

      console.log(`${repo}: Branch ${branchName} has been reset to ${baseBranch}`);
    } catch (error) {
      if (error.status === 404) {
        // If the branch doesn't exist, create it
        await octokit.git.createRef({
          owner,
          repo,
          ref: `refs/heads/${branchName}`,
          sha: baseSha,
        });

        console.log(`${repo}: Branch ${branchName} has been created`);
      } else {
        throw error;
      }
    }
  } catch (error) {
    console.error(`${repo}: Error creating or resetting branch ${branchName}:`, error.message);
    throw error;
  }
}

async function updateGitmodules(octokit, owner, repo) {
  console.log(`${repo}: Checking .gitmodules for updates...`);

  try {
    const { data: gitmodulesFile } = await octokit.repos.getContent({
      owner,
      repo,
      path: '.gitmodules',
    });

    console.log(`${repo}: Original .gitmodules SHA: ${gitmodulesFile.sha}`);

    const gitmodulesContent = Buffer.from(gitmodulesFile.content, 'base64').toString('utf-8');
    const gitmodules = parse(gitmodulesContent);

    let updatedContent = gitmodulesContent;
    let changes = false;

    for (const section in gitmodules) {
      if (section.startsWith('submodule ')) {
        const submodule = gitmodules[section];
        if (!submodule.branch) {
          const submodulePath = submodule.path;
          const submoduleUrl = submodule.url;
          const submoduleOwner = submoduleUrl.split('/').slice(-2)[0];
          const submoduleRepo = submoduleUrl.split('/').slice(-1)[0].replace('.git', '');

          const { data: submoduleCommit } = await octokit.repos.getContent({
            owner,
            repo,
            path: submodulePath,
          });

          const branch = await getSubmoduleBranch(octokit, repo, submoduleOwner, submoduleRepo, submodulePath, submoduleCommit.sha, gitmodulesContent);
          if (branch) {
            console.log(`${repo}: ${submodulePath}: Setting branch to ${branch}`);
            const newBranchLine = `\tbranch = ${branch}`;
            const sectionRegex = new RegExp(`\\[submodule "${submodulePath}"\\][^\\[]*`, 's');
            const sectionMatch = updatedContent.match(sectionRegex);
            if (sectionMatch) {
              const updatedSection = sectionMatch[0] + newBranchLine + '\n';
              updatedContent = updatedContent.replace(sectionRegex, updatedSection);
              changes = true;
            }
          }
        }
      }
    }

    if (!changes) {
      console.log(`${repo}: No changes needed in .gitmodules`);
      return false;
    }

    console.log(`${repo}: Updated .gitmodules with new content:`);
    console.log(updatedContent);

    const createPRChoice = await prompt(`${repo}: Push changes to branch ${prBranch}? (y/n): `);
    if (createPRChoice.toLowerCase() !== 'y') {
      console.log(`${repo}: Skipping branch/commit creation...`);
      return false;
    }

    console.log(`${repo}: Creating/resetting branch ${prBranch}...`);
    await createOrResetBranch(octokit, owner, repo, prBranch);

    const result = await octokit.repos.createOrUpdateFileContents({
      owner,
      repo,
      path: '.gitmodules',
      message: 'Update .gitmodules with submodule branch information\n\nChangelog-entry: Update .gitmodules with submodule branch information',
      content: Buffer.from(updatedContent).toString('base64'),
      sha: gitmodulesFile.sha,
      branch: prBranch,
    });

    console.log(`${repo}: File update result:`, result.status, result.data.commit.sha);

    return true;
  } catch (error) {
    console.error(`${repo}: Error updating .gitmodules:`, error.message);
    if (error.response) {
      console.error(`${repo}: Error response:`, error.response.data);
    }
    throw error;
  }
}

async function createPR(octokit, owner, repo) {
  const createPRChoice = await prompt(`Create PR for ${repo}? (y/n): `);
  if (createPRChoice.toLowerCase() !== 'y') {
    console.log(`Skipping PR creation for ${repo}`);
    return;
  }

  const { data: pr } = await octokit.pulls.create({
    owner,
    repo,
    title: 'Update .gitmodules with submodule branch information',
    head: prBranch,
    base: 'master',
    body: 'Changelog-entry: Update .gitmodules with submodule branch information',
  });

  console.log(`Created PR: ${pr.html_url}`);
}

async function processRepository(repo, octokit) {
  const repoName = repo.name;

  if (repo.archived) {
    console.log(`${repoName}: Skipping archived repository...`);
    return;
  }

  const isYoctoRepo = await checkRepoYaml(octokit, githubOrg, repoName);
  if (!isYoctoRepo) {
    console.log(`${repoName}: Skipping non-yocto repository...`);
    return;
  }

  const hasUnsetModuleBranch = await checkGitModules(octokit, githubOrg, repoName);
  if (!hasUnsetModuleBranch) {
    console.log(`${repoName}: Skipping repository with all module branches set...`);
    return;
  }

  console.log(`${repoName}: Repository qualifies for processing.`);

  const changes = await updateGitmodules(octokit, githubOrg, repoName);
  if (changes) {
    await createPR(octokit, githubOrg, repoName);
  }
}

async function main() {
  const { Octokit } = await import('@octokit/rest');

  const octokit = new Octokit({ 
    auth: process.env.GITHUB_TOKEN,
    retry: { enabled: false }
  });

  const maxRepos = process.env.MAX_REPOS ? parseInt(process.env.MAX_REPOS) : Infinity;
  let processedRepos = 0;

  try {
    for await (const response of octokit.paginate.iterator(octokit.repos.listForOrg, {
      org: githubOrg,
      per_page: 100
    })) {
      for (const repo of response.data) {
        try {
          await processRepository(repo, octokit);
        } catch (error) {
          console.error(`Error processing repository ${repo.name}:`, error);
          throw error; // Re-throw the error to stop the script
        }
        await delay(1000);
        
        processedRepos++;
        if (processedRepos >= maxRepos) {
          console.log(`Reached the maximum number of repositories to process (${maxRepos})`);
          return;
        }
      }
    }
  } finally {
    rl.close();
  }
}

main().catch(error => {
  console.error("Script stopped due to an error:", error);
  process.exit(1);
});