Last active
November 15, 2023 19:29
-
-
Save pauladams8/1df2783103ee1594e7e82b3d9d182785 to your computer and use it in GitHub Desktop.
Remove duplicates from your 1Password vault
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# use https://github.com/Ben-Hampson/1Password-Deduplicator instead for v2 support | |
import dateutil.parser | |
import subprocess | |
import tldextract | |
import argparse | |
import datetime | |
import shlex | |
import json | |
dry_run = False | |
prompt = True | |
vaults = {} | |
def run(cmd): | |
try: | |
return subprocess.run(shlex.split(cmd), check=True, capture_output=True, text=True).stdout | |
except subprocess.CalledProcessError as e: | |
print(e.stderr.strip() or f'Error running command `{cmd}`') | |
exit() | |
def domain_parts(item): | |
if not 'domain_parts' in item: | |
item['domain_parts'] = [tldextract.extract(url=node['u']) for node in item.get('overview', {}).get('URLs', [])] | |
return item['domain_parts'] | |
def domains(item): | |
s = set() | |
for d in domain_parts(item): | |
if d.subdomain == 'www': | |
d = (d.domain, d.suffix) | |
s.add('.'.join(p for p in d if p)) | |
return s | |
def root_domains(item): | |
return set('.'.join(p for p in (d.domain, d.suffix) if p) for d in domain_parts(item)) | |
def username(item): | |
return item['overview']['ainfo'] | |
def password(item): | |
return details(item)['password'] | |
def otp(item): | |
return details(item)['one-time password'] | |
def details(item): | |
if not 'details' in item: | |
item['details'] = json.loads(run(f'op get item {item["uuid"]} --fields "username,password,one-time password"')) | |
return item['details'] | |
def delete(item): | |
if dry_run: | |
print(f'To delete duplicate item {username(item)} with password {password(item)} in vault {vaults[item["vaultUuid"]]} for site{"s" if len(domains(item)) > 1 else ""} {", ".join(domains(item))}, run again without the dry run flag') | |
item['trashed'] = 'Y' | |
return | |
if prompt: | |
confirm = input(f'Are you sure you want to delete duplicate item {username(item)} with password {password(item)} in vault {vaults[item["vaultUuid"]]} for site{"s" if len(domains(item)) > 1 else ""} {", ".join(domains(item))}? (Y/n): ') | |
if confirm != 'Y': | |
return | |
run(f'op delete item {item["uuid"]}') | |
print(f'Deleted duplicate item {username(item)} for site{"s" if len(domains(item)) > 1 else ""} {", ".join(domains(item))}') | |
item['trashed'] = 'Y' | |
parser = argparse.ArgumentParser(description='Remove duplicate logins from your 1Password vault') | |
parser.add_argument('-d', '--dry-run', action='store_true', help='Output the items to be removed without actually removing them') | |
parser.add_argument('-y', '--yes', action='store_true', help="Don't prompt for delete confirmation") | |
parser.add_argument('--vault', metavar='[vault id]', help='Only search for duplicates in the specified vault') | |
parser.add_argument('--tag', metavar='[tag id]', action='append', help='Only search for duplicates with the specified tags') | |
args = parser.parse_args() | |
dry_run = args.dry_run | |
prompt = not args.yes | |
vaults = {v['uuid']: v['name'] for v in json.loads(run('op list vaults'))} | |
cmd = 'op list items --categories Login' | |
if args.vault: | |
cmd += f' --vault {args.vault}' | |
if args.tag: | |
cmd += f' --tags {",".join(args.tag)}' | |
items = json.loads(run(cmd)) | |
logins = (item for item in items if int(item['templateUuid']) == 1) | |
uniq = {} | |
for new in logins: | |
if new['trashed'] == 'Y': | |
continue | |
for d in root_domains(new): | |
try: | |
ex = uniq[(d, username(new))] | |
except KeyError: | |
uniq[(d, username(new))] = new | |
continue | |
if ex['trashed'] == 'Y': | |
continue | |
if domains(new) != domains(ex) and password(new) != password(ex): | |
continue | |
if (otp(new) and not otp(ex)) or (len(password(new)) > len(password(ex))): | |
uniq[(d, username(ex))] = new | |
delete(ex) | |
else: | |
delete(new) |
This script only works with the CLI v1. You'll need to install it manually by downloading the old CLI from https://app-updates.agilebits.com/product_history/CLI and following the manual install instructions here - https://developer.1password.com/docs/cli/get-started/
You'll probably need to uninstall the new CLI first if you have that installed.
I'll update it when I get a chance, but feel free to submit a PR to @ben-hampson 's repo if you want v2 support.
Sorry just seen @ben-hampson has already updated it to work with the new version. Check out the repo here - https://github.com/Ben-Hampson/1Password-Deduplicator
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This no longer works with
1Password for Mac 8.10.22 (81022007)