jwmcgettigan/bitwarden_duplicate_cleaner.py

## delete_duplicate_bitwarden_items.py
# Copyright © 2023 Justin McGettigan
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

# This script will pull all of your vault 'items' using the 'bw list items' command and then it will compare
# 	all properties that are not inherently unique from the returned JSON to determine if they are duplicates.
# Note: It removes older duplicates first - the newest copy of the 'item' will be the only one to remain.
# 	You can simply flip the '>' sign to '<' if you want to preserve the oldest 'item' instead.
#
# Setup Steps
# 1. You must install Bitwarden CLI first: https://bitwarden.com/help/cli/#download-and-install
# 2. Login to the CLI with the 'bw login' command.  You need your session key setup before continuing: https://bitwarden.com/help/cli/#using-a-session-key
# 3. Make sure to backup your 'items'.  You can use the 'bw export' command to do so: https://bitwarden.com/help/cli/#export
# 4. Run this python script and your duplicate 'items' will start being deleted. https://bitwarden.com/help/cli/#delete
# 	Note: I am NOT using the '--permanent' flag.  This means you can restore anything this script deletes within 30 days.
# 	Note2: The deletion process is pretty slow (1-2/items per second) so you'll likely need to let it run for a while.

import json
import hashlib
import subprocess

item_dict = {}

# Get the JSON data for each item in the vault
output = subprocess.check_output(['bw', 'list', 'items'])
items = json.loads(output)

for item in items:
    # Remove unique fields from the item data
    item_data = item.copy()
    del item_data['id']
    del item_data['folderId']
    del item_data['revisionDate']
    del item_data['creationDate']
    del item_data['deletedDate']

    # Calculate a hash of the item data
    item_hash = hashlib.sha256(str(item_data).encode('utf-8')).hexdigest()

    # Check if we've seen this item before
    if item_hash in item_dict:
        # Compare the revisionDate to see which item is newer
        if item['revisionDate'] > item_dict[item_hash]['revisionDate']:
            print(f'Duplicate item found: {item["name"]}')
            subprocess.run(['bw', 'delete', 'item', item_dict[item_hash]['id']])
            print(f'Deleted older item "{item_dict[item_hash]["name"]}".')
            item_dict[item_hash] = item
        else:
            print(f'Duplicate item found: {item["name"]}')
            subprocess.run(['bw', 'delete', 'item', item['id']])
            print(f'Deleted older item "{item["name"]}".')
    else:
        item_dict[item_hash] = item
	# Copyright © 2023 Justin McGettigan
	# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
	# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
	# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

	# This script will pull all of your vault 'items' using the 'bw list items' command and then it will compare
	# all properties that are not inherently unique from the returned JSON to determine if they are duplicates.
	# Note: It removes older duplicates first - the newest copy of the 'item' will be the only one to remain.
	# You can simply flip the '>' sign to '<' if you want to preserve the oldest 'item' instead.
	#
	# Setup Steps
	# 1. You must install Bitwarden CLI first: https://bitwarden.com/help/cli/#download-and-install
	# 2. Login to the CLI with the 'bw login' command. You need your session key setup before continuing: https://bitwarden.com/help/cli/#using-a-session-key
	# 3. Make sure to backup your 'items'. You can use the 'bw export' command to do so: https://bitwarden.com/help/cli/#export
	# 4. Run this python script and your duplicate 'items' will start being deleted. https://bitwarden.com/help/cli/#delete
	# Note: I am NOT using the '--permanent' flag. This means you can restore anything this script deletes within 30 days.
	# Note2: The deletion process is pretty slow (1-2/items per second) so you'll likely need to let it run for a while.

	import json
	import hashlib
	import subprocess

	item_dict = {}

	# Get the JSON data for each item in the vault
	output = subprocess.check_output(['bw', 'list', 'items'])
	items = json.loads(output)

	for item in items:
	# Remove unique fields from the item data
	item_data = item.copy()
	del item_data['id']
	del item_data['folderId']
	del item_data['revisionDate']
	del item_data['creationDate']
	del item_data['deletedDate']

	# Calculate a hash of the item data
	item_hash = hashlib.sha256(str(item_data).encode('utf-8')).hexdigest()

	# Check if we've seen this item before
	if item_hash in item_dict:
	# Compare the revisionDate to see which item is newer
	if item['revisionDate'] > item_dict[item_hash]['revisionDate']:
	print(f'Duplicate item found: {item["name"]}')
	subprocess.run(['bw', 'delete', 'item', item_dict[item_hash]['id']])
	print(f'Deleted older item "{item_dict[item_hash]["name"]}".')
	item_dict[item_hash] = item
	else:
	print(f'Duplicate item found: {item["name"]}')
	subprocess.run(['bw', 'delete', 'item', item['id']])
	print(f'Deleted older item "{item["name"]}".')
	else:
	item_dict[item_hash] = item