Skip to content

Instantly share code, notes, and snippets.

@apollosadmin
Created May 22, 2025 03:53
Show Gist options
  • Save apollosadmin/fc2fa3d32447c3157ac26f1337c7506a to your computer and use it in GitHub Desktop.
Save apollosadmin/fc2fa3d32447c3157ac26f1337c7506a to your computer and use it in GitHub Desktop.

Diff Summary.

diff File

The diff file describes the net changes across the total DB. You should expect changes to this file to represent intended changes to the shovel output. If you are performing a refactor, with no intended code changes, you should expect no changes to this file.

csv File

The csv file represents all operations that were performed that resulted in a data update. This doesn't including touching a content item (only updating a timestamp), but does include wiping a field out and then restoring that field to it's original contents. It will also track id changes.

Optimally, a dag that is not producing changes should not be wiping/restoring data on a run (should produce an empty csv). Practically, we have a long way to go in this area. Some operations (such as keeping track of content_tags) is easier to do by wiping the table and restoring, which produces operations, but doesn't result in a net diff.

854a855
> 56efc77d-04e7-45fd-98ad-e31a156d5081 \N \N \N \N \N MALE \N f Person 150 rock \N \N apollos_demo \N people
993d993
< 68f4aa91-0c93-4b9b-86de-41ec474b2acf \N \N \N \N \N MALE \N f Person 150 rock \N \N apollos_demo \N people
table_name changed_fields client_query action row_data
people "last_name"=>"Wilson","first_name"=>"Vincent" INSERT INTO people (apollos_type, birth_date, campus_id, email, first_name, gender, last_name, origin_id, origin_type, profile_image_url, updated_at) VALUES ('Person',NULL,NULL,NULL,'Vincent','MALE','Wilson',150,'rock',NULL,'2025-05-22T03:52:42.837850+00:00'::timestamptz) ON CONFLICT (origin_id, church_slug, origin_type) DO UPDATE SET apollos_type = excluded.apollos_type, birth_date = excluded.birth_date, campus_id = excluded.campus_id, email = excluded.email, first_name = excluded.first_name, gender = excluded.gender, last_name = excluded.last_name, profile_image_url = excluded.profile_image_url, updated_at = excluded.updated_at U "id"=>"68f4aa91-0c93-4b9b-86de-41ec474b2acf","email"=>NULL,"phone"=>NULL,"gender"=>"MALE","campus_id"=>NULL,"last_name"=>NULL,"origin_id"=>"150","apollos_id"=>"Person:68f4aa91-0c93-4b9b-86de-41ec474b2acf","birth_date"=>NULL,"first_name"=>NULL,"church_slug"=>"apollos_demo","origin_type"=>"rock","apollos_type"=>"Person","apollos_user"=>"f","profile_image_url"=>NULL
people INSERT INTO people (apollos_type, birth_date, campus_id, email, first_name, gender, last_name, origin_id, origin_type, profile_image_url, updated_at) VALUES ('Person',NULL,NULL,NULL,NULL,'MALE',NULL,163,'rock',NULL,'2025-05-22T03:52:42.837850+00:00'::timestamptz) ON CONFLICT (origin_id, church_slug, origin_type) DO UPDATE SET apollos_type = excluded.apollos_type, birth_date = excluded.birth_date, campus_id = excluded.campus_id, email = excluded.email, first_name = excluded.first_name, gender = excluded.gender, last_name = excluded.last_name, profile_image_url = excluded.profile_image_url, updated_at = excluded.updated_at I "id"=>"56efc77d-04e7-45fd-98ad-e31a156d5081","email"=>NULL,"phone"=>NULL,"gender"=>"MALE","campus_id"=>NULL,"last_name"=>NULL,"origin_id"=>"163","apollos_id"=>"Person:56efc77d-04e7-45fd-98ad-e31a156d5081","birth_date"=>NULL,"first_name"=>NULL,"church_slug"=>"apollos_demo","origin_type"=>"rock","apollos_type"=>"Person","apollos_user"=>"f","profile_image_url"=>NULL
media "node_id"=>"d5e387a1-3579-4fa1-9b44-6dd89d30d3ee" INSERT INTO media (apollos_type, created_at, updated_at, node_id, node_type, type, url, origin_id, origin_type) VALUES ('IMAGE','2025-05-22T03:52:42.837850+00:00'::timestamptz,'2025-05-22T03:52:42.837850+00:00'::timestamptz,'d5e387a1-3579-4fa1-9b44-6dd89d30d3ee','Campus','IMAGE','https://rock.apollos.app/GetImage.ashx?guid=4e181881-b156-48b0-be40-54af2b14b3fb','campusimage-71','rock') ON CONFLICT (origin_id, church_slug, origin_type) DO UPDATE SET apollos_type = excluded.apollos_type, created_at = excluded.created_at, updated_at = excluded.updated_at, node_id = excluded.node_id, node_type = excluded.node_type, type = excluded.type, url = excluded.url U "id"=>"061d25ff-a1b9-4c85-b2b3-0073f51a2c0e","url"=>"https://rock.apollos.app/GetImage.ashx?guid=4e181881-b156-48b0-be40-54af2b14b3fb","type"=>"IMAGE","node_id"=>"bccb32bd-53e7-4f71-9e2c-5ce79e4b8419","metadata"=>"{}","node_type"=>"Campus","origin_id"=>"campusimage-71","apollos_id"=>"IMAGE:061d25ff-a1b9-4c85-b2b3-0073f51a2c0e","church_slug"=>"apollos_demo","origin_type"=>"rock","apollos_type"=>"IMAGE"
media "node_id"=>"30403309-08ec-4dde-9182-4a9c8d8477cf" INSERT INTO media (apollos_type, created_at, updated_at, node_id, node_type, type, url, origin_id, origin_type) VALUES ('IMAGE','2025-05-22T03:52:42.837850+00:00'::timestamptz,'2025-05-22T03:52:42.837850+00:00'::timestamptz,'30403309-08ec-4dde-9182-4a9c8d8477cf','Campus','IMAGE','https://rock.apollos.app/GetImage.ashx?guid=4e181881-b156-48b0-be40-54af2b14b3fb','campusimage-71','rock') ON CONFLICT (origin_id, church_slug, origin_type) DO UPDATE SET apollos_type = excluded.apollos_type, created_at = excluded.created_at, updated_at = excluded.updated_at, node_id = excluded.node_id, node_type = excluded.node_type, type = excluded.type, url = excluded.url U "id"=>"061d25ff-a1b9-4c85-b2b3-0073f51a2c0e","url"=>"https://rock.apollos.app/GetImage.ashx?guid=4e181881-b156-48b0-be40-54af2b14b3fb","type"=>"IMAGE","node_id"=>"d5e387a1-3579-4fa1-9b44-6dd89d30d3ee","metadata"=>"{}","node_type"=>"Campus","origin_id"=>"campusimage-71","apollos_id"=>"IMAGE:061d25ff-a1b9-4c85-b2b3-0073f51a2c0e","church_slug"=>"apollos_demo","origin_type"=>"rock","apollos_type"=>"IMAGE"
media "node_id"=>"2990ca37-4397-4fec-a6c2-382b6f5b22c0" INSERT INTO media (apollos_type, created_at, updated_at, node_id, node_type, type, url, origin_id, origin_type) VALUES ('IMAGE','2025-05-22T03:52:42.837850+00:00'::timestamptz,'2025-05-22T03:52:42.837850+00:00'::timestamptz,'2990ca37-4397-4fec-a6c2-382b6f5b22c0','Campus','IMAGE','https://rock.apollos.app/GetImage.ashx?guid=4e181881-b156-48b0-be40-54af2b14b3fb','campusimage-71','rock') ON CONFLICT (origin_id, church_slug, origin_type) DO UPDATE SET apollos_type = excluded.apollos_type, created_at = excluded.created_at, updated_at = excluded.updated_at, node_id = excluded.node_id, node_type = excluded.node_type, type = excluded.type, url = excluded.url U "id"=>"061d25ff-a1b9-4c85-b2b3-0073f51a2c0e","url"=>"https://rock.apollos.app/GetImage.ashx?guid=4e181881-b156-48b0-be40-54af2b14b3fb","type"=>"IMAGE","node_id"=>"30403309-08ec-4dde-9182-4a9c8d8477cf","metadata"=>"{}","node_type"=>"Campus","origin_id"=>"campusimage-71","apollos_id"=>"IMAGE:061d25ff-a1b9-4c85-b2b3-0073f51a2c0e","church_slug"=>"apollos_demo","origin_type"=>"rock","apollos_type"=>"IMAGE"
media "node_id"=>"bccb32bd-53e7-4f71-9e2c-5ce79e4b8419" INSERT INTO media (apollos_type, created_at, updated_at, node_id, node_type, type, url, origin_id, origin_type) VALUES ('IMAGE','2025-05-22T03:52:42.837850+00:00'::timestamptz,'2025-05-22T03:52:42.837850+00:00'::timestamptz,'bccb32bd-53e7-4f71-9e2c-5ce79e4b8419','Campus','IMAGE','https://rock.apollos.app/GetImage.ashx?guid=4e181881-b156-48b0-be40-54af2b14b3fb','campusimage-71','rock') ON CONFLICT (origin_id, church_slug, origin_type) DO UPDATE SET apollos_type = excluded.apollos_type, created_at = excluded.created_at, updated_at = excluded.updated_at, node_id = excluded.node_id, node_type = excluded.node_type, type = excluded.type, url = excluded.url U "id"=>"061d25ff-a1b9-4c85-b2b3-0073f51a2c0e","url"=>"https://rock.apollos.app/GetImage.ashx?guid=4e181881-b156-48b0-be40-54af2b14b3fb","type"=>"IMAGE","node_id"=>"2990ca37-4397-4fec-a6c2-382b6f5b22c0","metadata"=>"{}","node_type"=>"Campus","origin_id"=>"campusimage-71","apollos_id"=>"IMAGE:061d25ff-a1b9-4c85-b2b3-0073f51a2c0e","church_slug"=>"apollos_demo","origin_type"=>"rock","apollos_type"=>"IMAGE"
people BEGIN TRANSACTION; -- We need to pull the people tags over to the correct person -- Sometimes, someone already has a tag, so we can skip it UPDATE people_tag SET person_id = '56efc77d-04e7-45fd-98ad-e31a156d5081' WHERE person_id = '68f4aa91-0c93-4b9b-86de-41ec474b2acf' and NOT EXISTS (SELECT 1 FROM people_tag WHERE person_id = '56efc77d-04e7-45fd-98ad-e31a156d5081' and tag_id = people_tag.tag_id); -- We need to update the charges to point to the merged person id. UPDATE charges SET person_id = '56efc77d-04e7-45fd-98ad-e31a156d5081' WHERE person_id = '68f4aa91-0c93-4b9b-86de-41ec474b2acf'; UPDATE recurring_charges SET person_id = '56efc77d-04e7-45fd-98ad-e31a156d5081' WHERE person_id = '68f4aa91-0c93-4b9b-86de-41ec474b2acf'; -- Now we can delete the old person DELETE FROM people WHERE id = '68f4aa91-0c93-4b9b-86de-41ec474b2acf' AND apollos_user is false; -- And update our person to have the correct origin_id UPDATE people SET origin_id = '150' WHERE id = '56efc77d-04e7-45fd-98ad-e31a156d5081'; COMMIT; D "id"=>"68f4aa91-0c93-4b9b-86de-41ec474b2acf","email"=>NULL,"phone"=>NULL,"gender"=>"MALE","campus_id"=>NULL,"last_name"=>"Wilson","origin_id"=>"150","apollos_id"=>"Person:68f4aa91-0c93-4b9b-86de-41ec474b2acf","birth_date"=>NULL,"first_name"=>"Vincent","church_slug"=>"apollos_demo","origin_type"=>"rock","apollos_type"=>"Person","apollos_user"=>"f","profile_image_url"=>NULL
people "origin_id"=>"150" BEGIN TRANSACTION; -- We need to pull the people tags over to the correct person -- Sometimes, someone already has a tag, so we can skip it UPDATE people_tag SET person_id = '56efc77d-04e7-45fd-98ad-e31a156d5081' WHERE person_id = '68f4aa91-0c93-4b9b-86de-41ec474b2acf' and NOT EXISTS (SELECT 1 FROM people_tag WHERE person_id = '56efc77d-04e7-45fd-98ad-e31a156d5081' and tag_id = people_tag.tag_id); -- We need to update the charges to point to the merged person id. UPDATE charges SET person_id = '56efc77d-04e7-45fd-98ad-e31a156d5081' WHERE person_id = '68f4aa91-0c93-4b9b-86de-41ec474b2acf'; UPDATE recurring_charges SET person_id = '56efc77d-04e7-45fd-98ad-e31a156d5081' WHERE person_id = '68f4aa91-0c93-4b9b-86de-41ec474b2acf'; -- Now we can delete the old person DELETE FROM people WHERE id = '68f4aa91-0c93-4b9b-86de-41ec474b2acf' AND apollos_user is false; -- And update our person to have the correct origin_id UPDATE people SET origin_id = '150' WHERE id = '56efc77d-04e7-45fd-98ad-e31a156d5081'; COMMIT; U "id"=>"56efc77d-04e7-45fd-98ad-e31a156d5081","email"=>NULL,"phone"=>NULL,"gender"=>"MALE","campus_id"=>NULL,"last_name"=>NULL,"origin_id"=>"163","apollos_id"=>"Person:56efc77d-04e7-45fd-98ad-e31a156d5081","birth_date"=>NULL,"first_name"=>NULL,"church_slug"=>"apollos_demo","origin_type"=>"rock","apollos_type"=>"Person","apollos_user"=>"f","profile_image_url"=>NULL
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment