morkdump

morkdump

#!/usr/bin/env ruby
#
# Mork Dumper, based on the CPAN module "Mozilla::Mork"
#   http://search.cpan.org/~kript/Mozilla-Mork-0.01/lib/Mozilla/Mork.pm
#
# description:
#   "Mork" is a format of Mozilla's internal data files, like
#   summary files (*.msf) of Thunderbird. This command reports
#   all contents of the specified Mork file as a JSON string.
#
# usage:
#   First, you have to install the Mozilla::Mork.
#
#     % curl -L http://cpanmin.us | perl - --sudo App::cpanminus
#     % sudo cpan Mozilla::Mork
#     % sudo cpan JSON
#
#   or
#
#     % curl -L http://cpanmin.us | perl - App::cpanminus
#     % cpan Mozilla::Mork
#     % cpan JSON
#
#   Then you can run this command.
#
#     % morkdump /path/to/morkfile
#
#   If you specify "--decode" option, subjects in msf files are decoded.
#
#     % morkdump --decode /path/to/morkfile
#
#   If you specify "--parse-flags" option, flags in msf files are shown
#   with human readable flag names.
#
#     % morkdump --parse-flags /path/to/morkfile

require "open3"
require "json"
require "optparse"
require "kconv"

def parse_mork(file)
  morkdump = <<-"PERL"
use Mozilla::Mork;
use JSON;

my $file = "#{file}";

my $MorkDetails = Mozilla::Mork->new($file);
my $results = $MorkDetails->ReturnReferenceStructure();

my $json = JSON->new->allow_nonref;
print $json->encode($results);

exit 0;
  PERL

  stdout, error, status = Open3.capture3("perl", :stdin_data => morkdump)
  parsed = stdout.encode("UTF-16BE", :invalid => :replace,
                                     :undef => :replace,
                                     :replace => '?')
  parsed = parsed.encode("UTF-8")
  JSON.parse(parsed)
end

$nsMsgMessageFlags = {
  :Read            => 0x00000001,
  :Replied         => 0x00000002,
  :Marked          => 0x00000004,
  :Expunged        => 0x00000008,
  :HasRe           => 0x00000010,
  :Elided          => 0x00000020,
  :FeedMsg         => 0x00000040,
  :Offline         => 0x00000080,
  :Watched         => 0x00000100,
  :SenderAuthed    => 0x00000200,
  :Partial         => 0x00000400,
  :Queued          => 0x00000800,
  :Forwarded       => 0x00001000,
  :Priorities      => 0x0000E000,
  :New             => 0x00010000,
  :Ignored         => 0x00040000,
  :IMAPDeleted     => 0x00200000,
  :MDNReportNeeded => 0x00400000,
  :MDNReportSent   => 0x00800000,
  :Template        => 0x01000000,
  :Attachment      => 0x10000000,
  :Labels          => 0x0E000000,
}

$nsMsgFolderFlags = {
  :Newsgroup       => 0x00000001,
  :Unused3         => 0x00000002,
  :Mail            => 0x00000004,
  :Directory       => 0x00000008,
  :Elided          => 0x00000010,
  :Virtual         => 0x00000020,
  :Unused5         => 0x00000040,
  :Unused2         => 0x00000080,
  :Trash           => 0x00000100,
  :SentMail        => 0x00000200,
  :Drafts          => 0x00000400,
  :Queue           => 0x00000800,
  :Inbox           => 0x00001000,
  :ImapBox         => 0x00002000,
  :Archive         => 0x00004000,
  :Unused1         => 0x00008000,
  :Unused4         => 0x00010000,
  :GotNew          => 0x00020000,
  :Unused6         => 0x00040000,
  :ImapPersonal    => 0x00080000,
  :ImapPublic      => 0x00100000,
  :ImapOtherUser   => 0x00200000,
  :Templates       => 0x00400000,
  :PersonalShared  => 0x00800000,
  :ImapNoselect    => 0x01000000,
  :CreatedOffline  => 0x02000000,
  :ImapNoinferiors => 0x04000000,
  :Offline         => 0x08000000,
  :OfflineEvents   => 0x10000000,
  :CheckNew        => 0x20000000,
  :Junk            => 0x40000000,
  :Favorite        => 0x80000000,
}
$nsMsgFolderFlags[:SpecialUse] = [
  :Inbox,
  :Drafts,
  :Trash,
  :SentMail,
  :Templates,
  :Junk,
  :Archive,
  :Queue,
].collect do |flag_name|
  $nsMsgFolderFlags[flag_name]
end.reduce do |a, b|
  a | b
end

def parse_record_flags(record)
  if folder_record?(record)
    parse_folder_flags(record["flags"])
  else
    parse_message_flags(record["flags"])
  end
end

def folder_record?(record)
  record["folderName"] or record["columnStates"] or record["mailboxName"]
end

def thread_record?(record)
  record["threadId"] and not record["message-id"]
end

def parse_message_flags(flags)
  flags = flags.to_i(16)
  $nsMsgMessageFlags.keys.select do |flag_name|
    flag_value = $nsMsgMessageFlags[flag_name]
    flags & flag_value != 0
  end.join(",")
end

def parse_folder_flags(flags)
  flags = flags.to_i(16)
  $nsMsgFolderFlags.keys.select do |flag_name|
    flag_value = $nsMsgFolderFlags[flag_name]
    flags & flag_value != 0
  end.join(",")
end

def normalize_records(records, options={})
  records = records.collect do |record|
    sorted_record = {}
    record.keys.sort.each do |key|
      value = record[key]
      value = value.toutf8 if options[:decode]
      if key == "flags" and options[:parse_flags]
        value = "#{value} (#{parse_record_flags(record)})"
      end
      sorted_record[key] = value
    end
    sorted_record
  end

  records.sort do |a, b|
    sort_records(a, b)
  end
end

def sort_records(a, b)
  case
  when (folder_record?(a) and not folder_record?(b))
    -1
  when (not folder_record?(a) and folder_record?(b))
    1
  when (thread_record?(a) and not thread_record?(b))
    -1
  when (not thread_record?(a) and thread_record?(b))
    1
  when (a["message-id"] and b["message-id"])
    a["message-id"] <=> b["message-id"]
  when (a["threadId"] and b["threadId"])
    a["threadId"] <=> b["threadId"]
  else
    0
  end
end

def main
  options = {
    :decode => false,
    :parse_flags => nil,
  }
  parser = OptionParser.new
  parser.on("--[no-]decode",
            "Decode non-ASCII strings in the subject field.") do |decode|
    options[:decode] = decode
  end
  parser.on("--[no-]parse-flags",
            "Parse flags of messages.") do |parse_flags|
    options[:parse_flags] = parse_flags
  end
  parser.parse!(ARGV)

  file = ARGV[0]
  unless file
    print "usage: morkdump <file>\n"
    return 1
  end

  if file.end_with?(".msf") and options[:parse_flags].nil?
    options[:parse_flags] = true
  end

  records = parse_mork(file)
  records = normalize_records(records, options)
  print JSON.pretty_generate(records)
  print "\n"
  0
end

exit main