Skip to content

Instantly share code, notes, and snippets.

@davidroth
Last active February 1, 2020 16:06
Show Gist options
  • Save davidroth/5243901 to your computer and use it in GitHub Desktop.
Save davidroth/5243901 to your computer and use it in GitHub Desktop.
Fast-Import / Fast-Export Rewriter to migrate bazaar bugtracking metdata properties to git. More infos: http://www.fusonic.net/en/blog/migrating-from-bazaar-to-git/ Licence: Mit X11 / BSD
// Fast-Import / Fast-Export Rewriter to migrate bazaar bugtracking metdata properties to git.
// More infos: http://www.fusonic.net/en/blog/migrating-from-bazaar-to-git/
// Licence: Mit X11 / BSD
using System;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Text;
using System.Diagnostics;
using System.Collections.Generic;
namespace migrate
{
class FastImportRewriter
{
static string inputFile = "/PATH_TO_YOUR_EXPORT_FILE.bzr";
static readonly Regex dataLengthRegex = new Regex("data ([0-9]+)?", RegexOptions.Compiled);
static readonly Regex bugLengthRegex = new Regex("property bugs ([0-9]+)", RegexOptions.Compiled);
static int blockSize = 1024;
static CommitPart commitPart;
static FileStream outputStream;
public static void Main(string[] args)
{
Stopwatch watch = Stopwatch.StartNew();
string outputFileName = inputFile + "_rewrite";
long fileSize = new FileInfo(inputFile).Length;
long currentPos = 0;
File.Delete(outputFileName);
using (outputStream = new FileStream(outputFileName, FileMode.CreateNew))
{
using (LineReader inputStream = new LineReader(File.OpenRead(inputFile)))
{
string s = null;
while ((s = inputStream.ReadLine()) != null)
{
currentPos = inputStream.BaseStream.Position;
if (currentPos % 1000 == 0)
{
double percentage = (100.0 / fileSize * currentPos);
LogStatus(percentage);
}
if (ShouldSkipLine(s))
continue;
else if (s.StartsWith("reset "))
{
if (commitPart != null)
{
commitPart.Flush();
}
commitPart = new CommitPart();
commitPart.Reset = s;
} else if (s.StartsWith("commit "))
{
if (commitPart != null)
{
commitPart.Flush();
}
commitPart = new CommitPart();
commitPart.Commit = s;
} else if (s.StartsWith("mark "))
{
commitPart.Mark = s;
} else if (s.StartsWith("committer "))
{
commitPart.Committer = s;
} else if (s.StartsWith("data ") && (commitPart != null && commitPart.MessageOrig == null))
{
int dataLength = ParseDataLength(s);
commitPart.MessageOrig = ReadDataBlockAsString(inputStream, dataLength);
} else if (s.StartsWith("from "))
{
commitPart.From = s;
} else if (s.StartsWith("merge "))
{
commitPart.Merge = s;
} else if (s.StartsWith("property bugs "))
{
int bugLength = ParseBugLength(s);
int bugHeadLength = ("property bugs " + bugLength.ToString() + " ").Length;
commitPart.Bug = s.Substring(bugHeadLength);
if (s.Length - bugHeadLength < bugLength)
{
commitPart.Bug += "\n" + ReadDataBlockAsString(inputStream, bugLength - (s.Length - bugHeadLength));
}
} else if (s.StartsWith("R "))
{
if (!IsDirectory(s.Substring(2, s.IndexOf(" ", 2) - 2)))
{
commitPart.Renames.Add(s);
}
} else if (s.StartsWith("D "))
{
if (!IsDirectory(s.Substring(2)))
{
commitPart.Deletes.Add(s);
}
} else if (s.StartsWith("M "))
{
if (s.StartsWith("M 040000"))
{
// track directories
dirs.Add(s.Substring(11));
// Skip Directory
continue;
}
//Reached end of commit block, now rewrite the commit and output the data
if (commitPart != null)
{
commitPart.Flush();
commitPart = null;
}
WriteLine(s);
} else if (s.StartsWith("data ") && commitPart == null)
{
int dataLength = ParseDataLength(s);
WriteLine(s);
int current = dataLength;
while (current - blockSize > 0)
{
var data = ReadDataBlock(inputStream, 1024);
Write(data);
current = current - blockSize;
}
if (current > 0)
{
var data = ReadDataBlock(inputStream, current);
Write(data);
}
WriteLine(string.Empty);
} else
{
Console.WriteLine("Skipping: " + s);
}
}
}
if (commitPart != null)
{
commitPart.Flush();
}
LogStatus(100);
Console.WriteLine("Rewrite took " + watch.Elapsed);
}
}
private static bool IsDirectory(string s)
{
bool isDir = dirs.Contains(s);
return isDir;
}
private static readonly HashSet<string> dirs = new HashSet<string>();
static void LogStatus(double percentage)
{
Console.Write("====== " + Math.Round(percentage, 2) + " % ====== \r");
}
static void Write(byte[] buffer)
{
outputStream.Write(buffer, 0, buffer.Length);
}
static readonly byte[] newline = Encoding.ASCII.GetBytes(Environment.NewLine);
static void WriteLine(string s)
{
byte[] bytes = System.Text.UTF8Encoding.Default.GetBytes(s);
outputStream.Write(bytes, 0, bytes.Length);
outputStream.Write(newline, 0, newline.Length);
}
static string ReadDataBlockAsString(LineReader inputStream, int dataLength)
{
return System.Text.UTF8Encoding.Default.GetString(inputStream.ReadBytes(dataLength));
}
static byte[] ReadDataBlock(LineReader inputStream, int dataLength)
{
byte[] buffer = inputStream.ReadBytes(dataLength);
return buffer;
}
static int ParseDataLength(string s)
{
var match = dataLengthRegex.Match(s);
if (match.Success)
{
return int.Parse(match.Groups[1].Value);
}
return 0;
}
static int ParseBugLength(string s)
{
var match = bugLengthRegex.Match(s);
if (match.Success)
{
return int.Parse(match.Groups[1].Value);
}
return 0;
}
static bool ShouldSkipLine(string s)
{
bool skip = s == string.Empty || s.StartsWith("feature") || s.StartsWith("property branch-nick");
return skip;
}
class CommitPart
{
static readonly Regex origBugIdRegex = new Regex(@"(\* [0-9]+\:)", RegexOptions.Compiled);
static readonly Regex bugsRegex = new Regex("http://mantis.yourcompany.net/view.php\\?id=([0-9]+)", RegexOptions.Compiled);
public CommitPart()
{
Deletes = new List<string>();
Renames = new List<string>();
}
public string Reset { get; set; }
public string Commit { get; set; }
public string Mark { get; set; }
public string Committer { get; set; }
public string MessageOrig { get; set; }
public string MessageNew { get; private set; }
public string From { get; set; }
public string Merge { get; set; }
public string Bug { get; set; }
public List<string> Deletes
{
get;
private set;
}
public List<string> Renames
{
get;
private set;
}
public void RewriteMesageWithBug()
{
if (MessageOrig == null)
{
return;
}
string message = MessageOrig;
if (Bug == null)
{
message = message.Replace("* ", string.Empty);
MessageNew = "data " + System.Text.UTF8Encoding.Default.GetByteCount(message).ToString() + "\n" + message;
return;
}
string[] bugIdStrings = ExtractBugId(Bug);
string newMessageString;
if (bugIdStrings.Length > 0)
{
message = origBugIdRegex.Replace(MessageOrig, "");
string strings = string.Join(", ", bugIdStrings);
newMessageString = "bug " + strings + ": " + message;
} else
{
newMessageString = message;
}
newMessageString = newMessageString.Replace("* ", string.Empty);
int byteLength = System.Text.UTF8Encoding.Default.GetByteCount(newMessageString);
MessageNew = "data " + byteLength.ToString() + "\n" + newMessageString;
}
static string[] ExtractBugId(string s)
{
var matches = bugsRegex.Matches(s);
if (matches.Count > 0)
{
var result = matches.Cast<Match>().Select(x => "#" + x.Groups[1].Value).ToArray();
return result;
}
return new string[0];
}
public void Flush()
{
commitPart.RewriteMesageWithBug();
if (Reset != null)
{
var l = "reset refs/tags/".Length;
string reset = Reset.Substring(0, l) + Reset.Substring(l, Reset.Length - l).Replace(' ', '_');
WriteLine(reset);
WriteLine(From);
WriteLine(string.Empty);
return;
}
WriteLine(Commit);
WriteLine(Mark);
WriteLine(Committer);
WriteLine(MessageNew);
if (From != null)
WriteLine(From);
if (Merge != null)
WriteLine(Merge);
foreach (string renamed in Renames)
{
WriteLine(renamed);
}
foreach (string deleted in Deletes)
{
WriteLine(deleted);
}
}
}
}
public class LineReader : BinaryReader
{
public LineReader(Stream stream)
: base(stream, Encoding.UTF8)
{
}
static readonly byte[] newline = Encoding.ASCII.GetBytes(Environment.NewLine);
public string ReadLine()
{
List<byte> buffer = new List<byte>(64);
try
{
while (true)
{
byte lastByte = base.ReadByte();
if (lastByte == newline[0])
{
return System.Text.UTF8Encoding.Default.GetString(buffer.ToArray());
}
buffer.Add(lastByte);
}
} catch (EndOfStreamException)
{
return null;
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment