Mailman 2.1 to 3.0 conversion

README.md

Migrating from mailman 2.1 to 3

• Copy mailman2 to shared storage
• set up nix shell env

cat > fix-subject.patch << EOF
diff --git a/hyperkitty/management/commands/hyperkitty_import.py b/hyperkitty/management/commands/hyperkitty_import.py
index d2e8b34..393e339 100644
--- a/hyperkitty/management/commands/hyperkitty_import.py
+++ b/hyperkitty/management/commands/hyperkitty_import.py
@@ -174,7 +174,7 @@ class DbImporter(object):
             # Un-wrap the subject line if necessary
             if message["subject"]:
                 message.replace_header(
-                    "subject", TEXTWRAP_RE.sub(" ", message["subject"]))
+                    "subject", TEXTWRAP_RE.sub(" ", message["subject"]).strip().replace("\n", " ").replace("\r", ""))
             if unixfrom:
                 message.set_unixfrom(unixfrom)
             if message['message-id'] is None:
EOF

cat > handle-dates.patch << EOF
diff --git a/hyperkitty/management/commands/hyperkitty_import.py b/hyperkitty/management/commands/hyperkitty_import.py
index 393e339..6a1a898 100644
--- a/hyperkitty/management/commands/hyperkitty_import.py
+++ b/hyperkitty/management/commands/hyperkitty_import.py
@@ -129,7 +129,7 @@ class DbImporter(object):
     def _get_date(self, message, header):
         try:
             date = message.get(header)
-        except TypeError as e:
+        except (ValueError, TypeError) as e:
             if self.verbose:
                 self.stderr.write(
                     "Can't get {} header in message {}: {}.".format(

EOF

cat > fix-error.patch << EOF
diff --git a/hyperkitty/management/commands/hyperkitty_import.py b/hyperkitty/management/commands/hyperkitty_import.py
index 917c8fc..bb9e8fe 100644
--- a/hyperkitty/management/commands/hyperkitty_import.py
+++ b/hyperkitty/management/commands/hyperkitty_import.py
@@ -216,7 +216,7 @@ class DbImporter(object):
                 self.stderr.write(
                     "Message {} failed to import, skipping".format(
                         unquote(message["Message-ID"])))
-                self.stderr.write(e)
+                self.stderr.write(str(e))
                 continue
             email = Email.objects.get(
                 mailinglist__name=self.list_address,

EOF

cat > handle-bad-emails.patch << EOF
diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 8c55946..43b1c75 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -616,8 +616,12 @@ def _import_roster(mlist, config_dict, members, role, action=None):
                 address = usermanager.create_address(original_email)
                 address.verified_on = datetime.datetime.now()
             user.link(address)
-        member = mlist.subscribe(address, role)
-        assert member is not None
+        try:
+            member = mlist.subscribe(address, role)
+            assert member is not None
+        except Exception as err:
+            print("Skipping broken member:", email, err)
+            continue
         prefs = config_dict.get('user_options', {}).get(email)
         if email in config_dict.get('members', {}):
             member.preferences.delivery_mode = DeliveryMode.regular

EOF

cat > mailman_migrate.nix << EOF
with import <nixpkgs> {};

let
  py3 = python3.override {
    packageOverrides = self: super: {
      django = self.django_1_11;
      hyperkitty = super.hyperkitty.overrideAttrs (old: {
        patches = [
          ./fix-subject.patch
          ./handle-dates.patch
          ./fix-error.patch
        ];
      });
      mailman = super.mailman.overrideAttrs (old: {
        # Overriden in package, so patches doesn't work
        patchPhase = old.patchPhase + ''
          cat \${./handle-bad-emails.patch} | patch ''\${patchFlags:--p1}
        '';
      });
    };
  };
  myenv = (py3.withPackages (ps: [ps.mailman ps.mailman-web])).env;
in myenv
EOF

• Begin import

nix-shell maiman_migrate.nix
cd /var/lib/mailman-web

cat > list_changes.py << EOF
from datetime import timedelta
m.admin_immed_notify = False
m.advertised = False
m.archive_policy = ArchivePolicy.private
m.autorespond_owner = ResponseAction.respond_and_discard
m.autorespond_postings = ResponseAction.respond_and_discard
m.autorespond_requests = ResponseAction.respond_and_discard
m.autoresponse_grace_period = timedelta(days=90)
m.autoresponse_owner_text = "This list is archived"
m.autoresponse_postings_text = "This list is archived"
m.autoresponse_request_text = "This list is archived"
m.default_member_action = Action.discard
m.default_nonmember_action = Action.discard
m.description = "Legacy List, archive only"
m.digest_send_periodic = False
m.digests_enabled = False
m.info = "Imported from mailman 2.1 on paphos"
m.respond_to_post_requests = False
m.send_welcome_message = False
m.subscription_policy = SubscriptionPolicy.moderate
EOF

cat > createlists.sh << EOF
#!/usr/bin/env bash
pck="$1"
name="$(echo $pck | cut -d/ -f5)"
echo $name
mailman create $name@redbricktest.ml
mailman import21 $name@redbricktest.ml $pck
echo $name DONE
EOF

cat > migratemail.sh << EOF
#!/usr/bin/env bash
mbox="$1"
name="$(basename $mbox | cut -d. -f1)"
echo $name
if mailman create $name@redbricktest.ml; then
    cat list_changes.py | mailman shell -l $name@redbricktest.ml
fi
chmod +r $mbox
sudo -u wwwrun mailman-web hyperkitty_import -l $name@redbricktest.ml $mbox
echo $name DONE
EOF

chmod +x createlists.sh migratemail.sh
for pck in /storage/mailman2/lists/**/config.pck; do ./createlists.sh $pck; done
(for f in /storage/mailman2/archives/**/*.mbox/*.mbox; do echo $f; done) | parallel -u -j8 ./migratemail.sh
chown -R mailman /var/lib/mailman

• Cleaning up broken archives or deleting archives

You can get a python console in the mailman env

sudo -u wwwrun mailman-web shell
from hyperkitty.models import MailingList
ml = MailingList.objets.get(name="whatever@redbricktest.ml")
ml.delete()

• List lists you just imported

mailman-web shell
from hyperkitty.models import MailingList
from datetime import datetime, timezone
from json import dumps
lists = sorted([l for l in MailingList.objects.all() if l.created_at > datetime(2020, 3, 26, tzinfo=timezone.utc)], key=lambda l: l.created_at)
print(dumps([vars(l) for l in lists], default=str))