Remove duplicates from your 1Password vault

1password_remove_duplicates.py

# use https://github.com/Ben-Hampson/1Password-Deduplicator instead for v2 support

import dateutil.parser
import subprocess
import tldextract
import argparse
import datetime
import shlex
import json

dry_run = False
prompt = True
vaults = {}

def run(cmd):
    try:
        return subprocess.run(shlex.split(cmd), check=True, capture_output=True, text=True).stdout
    except subprocess.CalledProcessError as e:
        print(e.stderr.strip() or f'Error running command `{cmd}`')
        exit()
def domain_parts(item):
    if not 'domain_parts' in item:
        item['domain_parts'] = [tldextract.extract(url=node['u']) for node in item.get('overview', {}).get('URLs', [])]
    return item['domain_parts']
def domains(item):
    s = set()
    for d in domain_parts(item):
        if d.subdomain == 'www':
            d = (d.domain, d.suffix)
        s.add('.'.join(p for p in d if p))
    return s
def root_domains(item):
    return set('.'.join(p for p in (d.domain, d.suffix) if p) for d in domain_parts(item))
def username(item):
    return item['overview']['ainfo']
def password(item):
    return details(item)['password']
def otp(item):
    return details(item)['one-time password']
def details(item):
    if not 'details' in item:
        item['details'] = json.loads(run(f'op get item {item["uuid"]} --fields "username,password,one-time password"'))
    return item['details']
def delete(item):
    if dry_run:
        print(f'To delete duplicate item {username(item)} with password {password(item)} in vault {vaults[item["vaultUuid"]]} for site{"s" if len(domains(item)) > 1 else ""} {", ".join(domains(item))}, run again without the dry run flag')
        item['trashed'] = 'Y'
        return
    if prompt:
        confirm = input(f'Are you sure you want to delete duplicate item {username(item)} with password {password(item)} in vault {vaults[item["vaultUuid"]]} for site{"s" if len(domains(item)) > 1 else ""} {", ".join(domains(item))}? (Y/n): ')
        if confirm != 'Y':
            return
    run(f'op delete item {item["uuid"]}')
    print(f'Deleted duplicate item {username(item)} for site{"s" if len(domains(item)) > 1 else ""} {", ".join(domains(item))}')
    item['trashed'] = 'Y'

parser = argparse.ArgumentParser(description='Remove duplicate logins from your 1Password vault')
parser.add_argument('-d', '--dry-run', action='store_true', help='Output the items to be removed without actually removing them')
parser.add_argument('-y', '--yes', action='store_true', help="Don't prompt for delete confirmation")
parser.add_argument('--vault', metavar='[vault id]', help='Only search for duplicates in the specified vault')
parser.add_argument('--tag', metavar='[tag id]', action='append', help='Only search for duplicates with the specified tags')
args = parser.parse_args()
dry_run = args.dry_run
prompt = not args.yes

vaults = {v['uuid']: v['name'] for v in json.loads(run('op list vaults'))}

cmd = 'op list items --categories Login'
if args.vault:
    cmd += f' --vault {args.vault}'
if args.tag:
    cmd += f' --tags {",".join(args.tag)}'
items = json.loads(run(cmd))
logins = (item for item in items if int(item['templateUuid']) == 1)
uniq = {}
for new in logins:
    if new['trashed'] == 'Y':
        continue
    for d in root_domains(new):
        try:
            ex = uniq[(d, username(new))]
        except KeyError:
            uniq[(d, username(new))] = new
            continue
        if ex['trashed'] == 'Y':
            continue
        if domains(new) != domains(ex) and password(new) != password(ex):
            continue
        if (otp(new) and not otp(ex)) or (len(password(new)) > len(password(ex))):
            uniq[(d, username(ex))] = new
            delete(ex)
        else:
            delete(new)

• Install packages

pip install python-dateutil tldextract

• Run script

python 1password_remove_duplicates.py -y

@pauladams8 Thanks for sharing this!

I made some changes to make it run with the latest version of op, added the ability to archive instead of delete, and added the ability to ignore favourites. Would you be ok if I shared it here?

Of course - I'd been meaning to update it myself to work with the new CLI but never got round it. Let me know if you have any issues.

Thanks! Feel free to try it out and see if it works for you. There are a few things I haven't tested yet.

1Password-Deduplicator

Hello everyone. I am new to using python scripting and I'm sure I'm making some kind of mistake, but when I run the program I get this response:

Traceback (most recent call last):
File "", line 198, in _run_module_as_main
File "", line 88, in _run_code
File "/Users/samurphis/Downloads/1Password-Deduplicator-master/1password_deduplicator.py", line 2, in
import tldextract
ModuleNotFoundError: No module named 'tldextract'

Any ideas on what mistake I'm making here?

Update: poking around on the web led me to using this pip3 install tldextract and now it runs :)

This no longer works with 1Password for Mac 8.10.22 (81022007)

This script only works with the CLI v1. You'll need to install it manually by downloading the old CLI from https://app-updates.agilebits.com/product_history/CLI and following the manual install instructions here - https://developer.1password.com/docs/cli/get-started/

You'll probably need to uninstall the new CLI first if you have that installed.

I'll update it when I get a chance, but feel free to submit a PR to @ben-hampson 's repo if you want v2 support.

Sorry just seen @ben-hampson has already updated it to work with the new version. Check out the repo here - https://github.com/Ben-Hampson/1Password-Deduplicator