Display relevant terraform policy diffs, omitting redundant items

tf_big_plans.py

#!/usr/bin/env python3

"""
This python script improves the usability of terraform 0.12 by
eliminating the display of redundant changes, typically found in 
resource attributes like policy/policy_data for AWS/GCP providers.

This is a known limitation with the legacy terraform SDK, as described here:
https://github.com/hashicorp/terraform/issues/21901

This script requires:
- python3
- terraform 0.12.24 or newer
- a github repo with terraform in a folder named tf in the root


## HOW DO:

1. Add this file to your github repo in a folder named scripts
2. Add an alias for easily using this script with tf plan. 
(replace .zshrc where appropriate)

```
cat <<EOF >> ~/.zshrc
tf_big_plans () {
    terraform plan -out=terraform.tfplan
    terraform show -no-color -json terraform.tfplan > terraform.tfplan.json
    python3 ./../scripts/tf_big_plans.py
}
EOF
source ~/.zshrc
```

3. Add the following to your .gitignore:
```
# tfplan files
*.tfplan*
```

4. Instead of running `tf plan`, use `tf_big_plans`

5. The plan will display normally, followed by a prettyprint display of
    any resources that were CHANGED. Review the regular terraform plan first,
    then use the additional output to review resources showing too many redundant
    changes.


## EXAMPLE USE CASE

Create a bigquery_dataset resource and assign 30 or more users view access to
    the dataset using terraform. Apply this, then add another user and run `tf plan`. 
    The output will show 30 users being removed and 31 being added. If you run 
    `tf_big_plans`, the additional output will only show the single user you added.


## FUTURE IMPROVEMENTS
- allow flags like -target to be used
- Add color to outputs and improve structure of displaying changes
- highlight things like email addresses, since that's primarily what this is used for
- don't run the script if terraform plan shows no changes

---------------------------

adapted from nitrocode's gist: 
https://gist.github.com/nitrocode/4820ba57f3bf5ef4d4254738496fc79c#file-find-tf-changes-py
"""

import json
import pprint
import os


def updates_only(resource):
    if resource['change']['actions'][0] != 'update':
        return False
    if 'before' not in resource['change'].keys():
        print('no before for ', resource['address'])
        return False
    if 'after' not in resource['change'].keys():
        print('no after for ', resource['address'])
        return False
    return True


def all_keys(before, after):
    return list(set().union(before.keys(), after.keys()))


def list_measuring_contest(bef, aft):
    output = dict()
    gaining = [item for item in aft if item not in bef]
    if gaining:
        output['GAINING'] = gaining
    removing = [item for item in bef if item not in aft]
    if removing:
        output['LOSING'] = removing
    return output


def dict_measuring_contest(before, after):
    changes = dict()
    for key in all_keys(before, after):
        if key not in before.keys():
            changes[key] = {'GAINING': after[key]}
            continue
        if key not in after.keys():
            changes[key] = {'LOSING': before[key]}
            continue
        if before[key] == after[key]:
            continue
        if isinstance(before[key], dict) and isinstance(after[key], dict):
            nested_changes = dict_measuring_contest(before[key], after[key])
            if nested_changes:
                changes[key] = nested_changes
        elif isinstance(before[key], list) and isinstance(after[key], list):
            sub_changes = list_measuring_contest(before[key], after[key])
            if sub_changes:
                changes[key] = sub_changes
    if not changes:
        return
    return changes


def recursive_replace(obj, replace="", replace_value=None):
    if isinstance(obj, dict):
        for k, v in obj.items():
            obj[k] = recursive_replace(v)
    elif isinstance(obj, list):
        obj = [recursive_replace(x) for x in obj]
    elif isinstance(obj, str):
        if obj == replace:
            return replace_value
    return obj


def recursive_json_loads(obj):
    if isinstance(obj, dict):
        for k, v in obj.items():
            obj[k] = recursive_json_loads(v)
    elif isinstance(obj, list):
        obj = [recursive_json_loads(x) for x in obj]
    elif isinstance(obj, str):
        try:
            return json.loads(obj, sort_keys=True)
        except json.JSONDecodeError:
            return obj
        except TypeError:
            return obj
    return obj


def clean_up_obj(obj):
    obj = recursive_json_loads(obj)
    obj = recursive_replace(obj)
    return obj


def return_resource_diff(resource):
    # resource is of type: update, and has content for before & after

    before = clean_up_obj(resource['change']['before'])
    after = clean_up_obj(resource['change']['after'])

    changes = dict_measuring_contest(before, after)

    if not changes:
        return
    return changes


def run():

    with open(f'{os.getcwd()}/terraform.tfplan.json', "rb") as f:
        data = json.loads(f.read())

    changes = dict()

    updates = [x for x in data['resource_changes'] if updates_only(x)]

    for resource in updates:

        out = return_resource_diff(resource)
        if out:
            changes[resource['address']] = out

    return changes


if __name__ == '__main__':
    changes = run()
    pprint.pprint(changes)