-
-
Save pauladams8/1df2783103ee1594e7e82b3d9d182785 to your computer and use it in GitHub Desktop.
# use https://github.com/Ben-Hampson/1Password-Deduplicator instead for v2 support | |
import dateutil.parser | |
import subprocess | |
import tldextract | |
import argparse | |
import datetime | |
import shlex | |
import json | |
dry_run = False | |
prompt = True | |
vaults = {} | |
def run(cmd): | |
try: | |
return subprocess.run(shlex.split(cmd), check=True, capture_output=True, text=True).stdout | |
except subprocess.CalledProcessError as e: | |
print(e.stderr.strip() or f'Error running command `{cmd}`') | |
exit() | |
def domain_parts(item): | |
if not 'domain_parts' in item: | |
item['domain_parts'] = [tldextract.extract(url=node['u']) for node in item.get('overview', {}).get('URLs', [])] | |
return item['domain_parts'] | |
def domains(item): | |
s = set() | |
for d in domain_parts(item): | |
if d.subdomain == 'www': | |
d = (d.domain, d.suffix) | |
s.add('.'.join(p for p in d if p)) | |
return s | |
def root_domains(item): | |
return set('.'.join(p for p in (d.domain, d.suffix) if p) for d in domain_parts(item)) | |
def username(item): | |
return item['overview']['ainfo'] | |
def password(item): | |
return details(item)['password'] | |
def otp(item): | |
return details(item)['one-time password'] | |
def details(item): | |
if not 'details' in item: | |
item['details'] = json.loads(run(f'op get item {item["uuid"]} --fields "username,password,one-time password"')) | |
return item['details'] | |
def delete(item): | |
if dry_run: | |
print(f'To delete duplicate item {username(item)} with password {password(item)} in vault {vaults[item["vaultUuid"]]} for site{"s" if len(domains(item)) > 1 else ""} {", ".join(domains(item))}, run again without the dry run flag') | |
item['trashed'] = 'Y' | |
return | |
if prompt: | |
confirm = input(f'Are you sure you want to delete duplicate item {username(item)} with password {password(item)} in vault {vaults[item["vaultUuid"]]} for site{"s" if len(domains(item)) > 1 else ""} {", ".join(domains(item))}? (Y/n): ') | |
if confirm != 'Y': | |
return | |
run(f'op delete item {item["uuid"]}') | |
print(f'Deleted duplicate item {username(item)} for site{"s" if len(domains(item)) > 1 else ""} {", ".join(domains(item))}') | |
item['trashed'] = 'Y' | |
parser = argparse.ArgumentParser(description='Remove duplicate logins from your 1Password vault') | |
parser.add_argument('-d', '--dry-run', action='store_true', help='Output the items to be removed without actually removing them') | |
parser.add_argument('-y', '--yes', action='store_true', help="Don't prompt for delete confirmation") | |
parser.add_argument('--vault', metavar='[vault id]', help='Only search for duplicates in the specified vault') | |
parser.add_argument('--tag', metavar='[tag id]', action='append', help='Only search for duplicates with the specified tags') | |
args = parser.parse_args() | |
dry_run = args.dry_run | |
prompt = not args.yes | |
vaults = {v['uuid']: v['name'] for v in json.loads(run('op list vaults'))} | |
cmd = 'op list items --categories Login' | |
if args.vault: | |
cmd += f' --vault {args.vault}' | |
if args.tag: | |
cmd += f' --tags {",".join(args.tag)}' | |
items = json.loads(run(cmd)) | |
logins = (item for item in items if int(item['templateUuid']) == 1) | |
uniq = {} | |
for new in logins: | |
if new['trashed'] == 'Y': | |
continue | |
for d in root_domains(new): | |
try: | |
ex = uniq[(d, username(new))] | |
except KeyError: | |
uniq[(d, username(new))] = new | |
continue | |
if ex['trashed'] == 'Y': | |
continue | |
if domains(new) != domains(ex) and password(new) != password(ex): | |
continue | |
if (otp(new) and not otp(ex)) or (len(password(new)) > len(password(ex))): | |
uniq[(d, username(ex))] = new | |
delete(ex) | |
else: | |
delete(new) |
pauladams8
commented
Aug 2, 2021
- Install packages
- Run script
@pauladams8 Thanks for sharing this!
I made some changes to make it run with the latest version of op
, added the ability to archive instead of delete, and added the ability to ignore favourites. Would you be ok if I shared it here?
Of course - I'd been meaning to update it myself to work with the new CLI but never got round it. Let me know if you have any issues.
Thanks! Feel free to try it out and see if it works for you. There are a few things I haven't tested yet.
Hello everyone. I am new to using python scripting and I'm sure I'm making some kind of mistake, but when I run the program I get this response:
Traceback (most recent call last):
File "", line 198, in _run_module_as_main
File "", line 88, in _run_code
File "/Users/samurphis/Downloads/1Password-Deduplicator-master/1password_deduplicator.py", line 2, in
import tldextract
ModuleNotFoundError: No module named 'tldextract'
Any ideas on what mistake I'm making here?
Update: poking around on the web led me to using this pip3 install tldextract and now it runs :)
This no longer works with 1Password for Mac 8.10.22 (81022007)
This script only works with the CLI v1. You'll need to install it manually by downloading the old CLI from https://app-updates.agilebits.com/product_history/CLI and following the manual install instructions here - https://developer.1password.com/docs/cli/get-started/
You'll probably need to uninstall the new CLI first if you have that installed.
I'll update it when I get a chance, but feel free to submit a PR to @ben-hampson 's repo if you want v2 support.
Sorry just seen @ben-hampson has already updated it to work with the new version. Check out the repo here - https://github.com/Ben-Hampson/1Password-Deduplicator