Created
April 7, 2020 21:42
-
-
Save amarjandu/11d0e72e60b1b1dbc49abacb2ab3fe78 to your computer and use it in GitHub Desktop.
tombstone (#1679)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| """ | |
| This script will tombstone the bundles in `bundles_to_tombstone` | |
| Usage: | |
| - tag HCA/data-ops team in the comments under the Data-Store issue for the tombstone reqeust with @HumanCellAtlas/data-ops | |
| - edit `stage` to target deployment. | |
| - edit `bundles_to_tombstone` to contain the bundles to tombstone. | |
| - edit `tombstone_reason` with explanation for tombstoning | |
| - Place the edited script in a GitHub gist | |
| - Post gist link to Slack in #dcp-ops asking for review and verification that these bundles should be tonbstoned | |
| - Tag @dataops in the #dcp-ops channel in the comments to the tread posted above. | |
| - If given the go-ahead, execute script | |
| - Monitor output. Announce errors in #dcp-ops, ask for help in #data-store-eng | |
| """ | |
| import sys | |
| import time | |
| import json | |
| from random import randint | |
| from traceback import format_exc | |
| from concurrent.futures import ThreadPoolExecutor, as_completed | |
| from hca.dss import DSSClient | |
| from hca.util.exceptions import SwaggerAPIException | |
| stage = "staging" | |
| tombstone_reason = "Errors within Azul, see Azul(#1679)" | |
| assert stage | |
| assert tombstone_reason | |
| BATCH_SIZE = 1000 | |
| def _get_dss_client(): | |
| if stage == "prod": | |
| dss_client = DSSClient(swagger_url="https://dss.data.humancellatlas.org/v1/swagger.json") | |
| else: | |
| dss_client = DSSClient(swagger_url=f"https://dss.{stage}.data.humancellatlas.org/v1/swagger.json") | |
| return dss_client | |
| dss_client = _get_dss_client() | |
| def tombstone_bundle(fqid): | |
| uuid, version = fqid.split(".", 1) | |
| bundle_found = bundle_is_found(uuid, version) | |
| bundle_already_tombstoned = bundle_is_tombstoned(uuid, version) | |
| if bundle_found and not bundle_already_tombstoned: | |
| resp = dss_client.delete_bundle( | |
| replica="aws", | |
| uuid=uuid, | |
| version=version, | |
| reason=tombstone_reason, | |
| ) | |
| for _ in range(20): | |
| if bundle_is_tombstoned(uuid, version): | |
| break | |
| else: | |
| time.sleep(randint(1,3)) | |
| else: | |
| raise Exception(f"Unable to verity tombstone {uuid}") | |
| return bundle_found, bundle_already_tombstoned, fqid | |
| def bundle_is_found(uuid, version): | |
| try: | |
| resp = dss_client.get_bundle(replica="aws", uuid=uuid, version=version) | |
| return True | |
| except SwaggerAPIException as e: | |
| if 404 == e.code: | |
| return False | |
| else: | |
| raise | |
| def bundle_is_tombstoned(uuid, version): | |
| bundle_query_with_version = {"query":{"bool":{"must":[{"match":{"uuid":uuid}},{"match":{"version":version}},]}}} | |
| resp = dss_client.post_search(replica="aws", es_query=bundle_query_with_version, output_format="raw") | |
| if 0 == resp['total_hits']: | |
| return False | |
| elif 'admin_deleted' in resp['results'][0]['metadata']: | |
| return True | |
| else: | |
| return False | |
| def bundle_status_string(bundle_found, bundle_already_tombstoned): | |
| if bundle_already_tombstoned: | |
| return "ALLREADY_TOMBSTONED" | |
| elif not bundle_found: | |
| return "NOT_FOUND" | |
| else: | |
| return "TOMBSTONED" | |
| if __name__ == "__main__": | |
| bundles_to_tombstone = [ | |
| "72573e5e-bc88-43f6-8b36-eec962644ad7.2020-02-06T183307.743045Z", | |
| "4882ec6f-fe81-43d8-9446-595fbf6319da.2019-08-12T144035.510469Z", | |
| "869c5cd3-98fd-4947-85eb-29d1a9cbd496.2018-08-17T202940.855330Z", | |
| "81fbc171-984e-4ea2-beb9-d58607b93df3.2019-08-05T185113.345359Z", | |
| "deb7e2b6-101d-40ad-93d7-a4e922807b22.2019-07-10T190307.877273Z", | |
| "04cdef35-5c0c-4f75-ab1b-ab66c66a6e58.2019-07-23T175423.984602Z", | |
| "1076c830-8986-4fee-9e7a-847fe0b227c7.2019-08-12T144035.510424Z", | |
| "3aa3510d-b363-4283-97b5-354bf339d297.2020-02-06T205353.641296Z", | |
| "716b3dd5-4104-4ebf-b85b-7bce2d6ff892.2019-07-23T175914.819925Z", | |
| "bef1bde1-788f-4c8b-a97b-6de6209aa7f5.2020-02-06T205253.351515Z", | |
| "328a2bf0-3f3c-4441-96a8-ddf41852e650.2019-08-12T144035.510389Z", | |
| "4717eb27-350f-41c1-adc2-3589e17048ee.2019-08-12T144035.510642Z", | |
| "1bd9bdbe-33a2-4eb5-90e2-1df5fa00d504.2018-08-17T202728.071749Z", | |
| "d55ce053-95e3-48a0-bec9-2d22718b6628.2019-08-05T185529.676531Z", | |
| "71a582dc-6a03-4bb4-a4f7-29bbf289d8b9.2019-08-12T144035.510561Z", | |
| "d32d815d-6e0d-4fc4-80be-938891fd5b18.2020-02-06T183236.357965Z", | |
| "9118e5b4-e45c-4581-bec7-c8f20bdb5151.2020-02-06T191710.090378Z", | |
| "f961b3fe-6e6f-4bed-9dc4-f042d3b3d91b.2019-08-05T224620.278567Z", | |
| "b635b2e7-d72f-406d-9ac1-4181df36341f.2019-08-05T185024.610340Z", | |
| "85a199f1-6820-4d08-9787-6694d357b3d4.2019-07-10T183934.593670Z", | |
| "7ef3700c-d2d3-4e6e-b73e-c22f3380e7c3.2019-08-12T144035.510487Z", | |
| "39629ab2-fc94-4e97-902f-de4ffa23f79a.2019-08-05T184644.217873Z", | |
| "3ffccaa0-56cb-463c-a2b3-6808ac1f0e9e.2019-07-10T184036.829039Z", | |
| "3c3061a7-1dc5-422e-bc14-4760b47f6ad2.2019-08-12T144035.510345Z", | |
| "8f585ec2-54c2-4d7b-8cea-0a9ceada6c8f.2019-08-21T221540.862704Z", | |
| "aaa47f1a-204e-4be5-9832-a95a8dbf8dcc.2020-02-06T184140.714789Z", | |
| "ebb4680d-d7c4-46dd-9941-b0bbcfd1c3a7.2020-02-06T200713.724776Z", | |
| "15aa0398-0533-4787-bd06-701f9f7ac59f.2020-02-06T215435.836074Z", | |
| "28d9d019-4e33-449f-8627-db52391ffd3c.2020-02-06T185423.712098Z", | |
| "a416ef51-3c77-49b7-913a-b62080aa3786.2019-07-30T192056.427560Z", | |
| "9c85a682-8a29-4062-8503-c45e37be5bf6.2019-07-10T190123.278032Z", | |
| "cb97a8e3-8593-45df-bc63-caaa9200d3c4.2019-10-03T105524.916954Z", | |
| "d4303853-529f-4617-a765-76bb9c5723fa.2019-07-10T184405.638125Z", | |
| "00005cce-dfbe-40a9-922a-e1da11a11ada.2019-08-12T144035.510398Z", | |
| "170933df-4405-4e28-9611-3b0a7d165ae5.2019-07-23T175724.419005Z", | |
| "1f671207-2e4a-452e-94f0-547d696317a3.2018-08-17T202546.197851Z", | |
| "7b4fd692-c38d-43c6-bb5b-bf3dcbc837eb.2019-08-12T144035.510371Z", | |
| "84bf575e-a5be-4c31-8716-604157ab5acf.2020-02-06T183738.897996Z", | |
| "9eb1a782-4c15-42b7-8d4a-e6dfca346fe3.2019-08-22T163824.964078Z", | |
| "d4da4f71-02b4-437c-91ae-5219f479a175.2019-07-30T192020.496371Z", | |
| "ad88924f-248f-4c6e-bad9-d64495b03de1.2019-08-12T144035.510652Z", | |
| "a0a78a4e-7891-4ef2-a22d-1bfab43ffe47.2019-08-12T144035.510354Z", | |
| "ea25386a-dc17-453d-9312-b61995f2c3f6.2019-08-21T110748.639681Z", | |
| "5bf86ddb-a57b-49be-b734-8b09d7259e60.2019-08-12T144035.510505Z", | |
| "a44bfb8e-37c3-44d5-a8e8-f0fd916d22f9.2020-02-06T200220.171143Z", | |
| "499b690e-feee-4675-8531-882a62b6020c.2019-08-21T030312.315201Z", | |
| "81c5a0ed-50d3-4c60-b1b4-57df9862b636.2019-08-21T181455.323568Z", | |
| "6715eed2-f9a1-49dc-8e98-239446211e87.2020-02-06T215916.696303Z", | |
| "f99ef36d-0b73-414c-b004-9ca3a23ab2c5.2019-11-05T171907.123671Z", | |
| "fe118edd-0ffb-4134-be08-1a46a2bbcb95.2020-02-06T201301.877775Z", | |
| "79ec8f41-fccc-46ed-af43-e63892238d77.2019-08-21T153540.733071Z", | |
| "41af0283-946e-4a10-be95-46aaaa9d1bc8.2020-02-06T185736.341234Z", | |
| "88aed2c9-6537-40fe-a04c-cd58900e9abf.2019-08-21T224410.548780Z", | |
| "7e330852-f693-4303-bc83-09e6d145d737.2019-08-21T233444.728103Z", | |
| "05fc240e-cd04-40e0-b6c2-059ba982f1de.2019-07-30T192059.629519Z", | |
| "d1ed619d-f950-4f55-8e98-ae058bee5ad8.2020-02-06T205314.173862Z", | |
| "8586d38e-b743-4ec5-8e74-38ab4b6c7e02.2019-09-09T221828.818745Z", | |
| "8b5c0729-37b1-4377-ab1c-5b932de10826.2019-09-05T182154.052280Z", | |
| "51e56940-706e-4adf-8f1b-db31308b3d8d.2019-07-30T192029.999854Z", | |
| "8388fed2-1af2-4102-a578-82d3ef5ace03.2019-08-12T144035.510281Z", | |
| "edca3333-66c5-401f-964c-52de1f575d1e.2020-02-06T185357.016348Z", | |
| "f89b2ea9-6698-4d3b-a104-97cda08cff27.2019-08-05T225838.273291Z", | |
| "fff16441-5212-4319-bf52-9e23d0eff7eb.2019-08-12T144035.510478Z", | |
| "f840f06a-58b1-4699-a672-ef0b08fb3304.2019-08-12T144035.510263Z", | |
| "1276a46f-1202-445b-ab7d-7b5b7451f223.2020-02-06T201718.145544Z", | |
| "27a43436-6e63-44ce-a2b3-80618f00b7f6.2019-08-21T025721.178682Z", | |
| "3637ab79-f66f-44e6-bafd-cea54af2531a.2020-02-06T215845.406268Z", | |
| "2e9b493d-e7ff-473e-84e1-82c9dc6d98e0.2020-02-06T211042.578220Z", | |
| "cb4f6254-a123-420a-9d00-fdf1e30067ec.2020-02-06T211109.366273Z", | |
| "0cc25d1d-6218-4dad-ba5f-91ac4144a24b.2019-08-21T025515.596836Z", | |
| "0f0c3650-d063-481d-b4ef-fe228be23caa.2020-02-06T211114.028992Z", | |
| "192ccf23-5847-458d-9527-c730677e7093.2019-08-22T013132.633966Z", | |
| "a44844d6-2728-4595-9fa7-bb6f2a34b629.2019-08-12T144035.510680Z", | |
| "a68d08a7-49bc-4ac3-b2cf-6b644a79b0d1.2020-02-06T235713.380658Z", | |
| "0d3477d7-e190-4e9c-a163-f8819db1f227.2019-09-09T221824.848034Z", | |
| "e72edef7-b3de-4895-bcc2-9d1c3678d6e4.2019-08-01T223656.070705Z", | |
| "25ea2837-8e81-44be-b5af-79f82f90f90f.2020-02-06T235229.291861Z", | |
| "5c1c20ea-823e-4476-b81f-e62337a4b0fb.2020-02-06T204326.141860Z", | |
| "5a37f51b-3f66-4b22-95c1-d82842265f19.2019-08-12T144035.510589Z", | |
| "8b8aa932-3d02-4696-b434-b0bd79d7654d.2020-02-06T211019.412105Z", | |
| "a36a76dd-3036-42e6-beb2-dd41e1e312b6.2019-08-12T144035.510552Z", | |
| "e22e7f03-4917-4ab9-ba0c-31784fa62de9.2019-08-12T144035.510623Z", | |
| "0d6d1122-acf0-466d-ab0b-a131edfb663e.2019-08-12T144035.510670Z", | |
| "45957a9c-0532-4481-a5d5-9d685ed353b7.2020-02-06T201341.295214Z", | |
| "002fe390-ba3f-4aa7-ae43-aa7949b09bb1.2019-08-12T144035.510309Z", | |
| "67bd4c99-a5f8-4607-8571-9c97fed0832e.2019-08-22T194039.502693Z", | |
| "64bc8ed8-0ceb-49c0-9b36-3f8acfb343ba.2019-07-08T231711.917523Z", | |
| "1b6d8348-d6e9-406a-aa6a-7ee886e52bf9.2019-10-03T105524.911627Z", | |
| "8b7635a4-88a4-4c76-9946-6e3bebd51bfa.2019-08-12T144035.510318Z", | |
| "5c1f1f24-64d6-4a37-9a23-e0e811cd0a8e.2020-02-06T185708.514497Z", | |
| "6a0de4df-9962-4a6a-a29a-9ab522fe6b09.2019-09-05T185111.818094Z", | |
| "3013e0db-e471-4134-9e0c-b3bed2464a8e.2019-08-22T020408.008157Z", | |
| "36a39838-28de-4e78-859e-d0778620c778.2019-08-05T185045.489448Z", | |
| "9bd51e8f-abb2-40e6-8ee2-e0afa8b6caa6.2019-08-21T174204.877310Z", | |
| "10ae73ef-d2e1-4a1d-a192-cf332d850c4f.2019-08-12T144035.510570Z", | |
| "b6037eec-60da-48da-85a4-b7438539a6ae.2019-08-12T144035.510633Z", | |
| "7c6c2d07-1f32-4886-94b9-bfc37a2b2d20.2020-02-06T183743.288566Z", | |
| "f50fb0fd-5c56-4c86-a9bc-b9bed63eb9b0.2020-02-06T220144.584510Z", | |
| "9f0bafe1-e93e-47af-b220-7851130e372a.2020-02-06T205349.135028Z", | |
| "14273b8d-bca6-4b72-be72-ca4f19623ee4.2019-09-09T221739.861168Z", | |
| "967ae26c-d015-4e5e-bc37-8ab625c7db15.2020-02-06T220111.840042Z", | |
| "4accc284-186c-4b81-9810-29314a4a6948.2019-08-22T181959.633839Z", | |
| "65f2d9bb-4145-44d4-b0b7-d8490b9162a0.2020-02-06T204640.008839Z", | |
| "91b291ea-40eb-4e0b-b289-4482ce207f29.2018-08-17T202425.293015Z", | |
| "ed47b69d-4e7d-4956-b5a7-7cfa7583a3cd.2020-02-06T183200.345202Z", | |
| "caf3a1b6-60d3-42fd-a56c-48fe0e6de571.2019-08-21T174233.336465Z", | |
| "af4392b2-2387-47db-8b08-6223c8b5718c.2020-02-06T215510.415147Z", | |
| "8ee609f9-0fd8-4d3f-90a7-dc39e24ced16.2020-02-06T182429.028219Z", | |
| "516b4765-7a60-46c5-9880-ec597a3eddba.2020-02-06T204321.438422Z", | |
| "6f01e3ae-9743-4424-983b-18b18f0ffeed.2019-08-12T144035.510290Z", | |
| "3c1b06ed-c573-4431-8ddf-b38652950cee.2020-02-06T185802.746897Z", | |
| "cb88115c-75ef-4e8e-bbd7-a48e18960344.2019-10-03T105524.915852Z", | |
| "d9cdeb51-67f2-4e4a-9321-b4125f721900.2020-02-06T191323.779205Z", | |
| "cc3a6b51-ae84-4ad3-a4a5-f1845afff174.2020-02-06T211506.493433Z", | |
| "daaf63cf-04ae-4335-868c-32e63af0c37a.2019-08-21T155534.803383Z", | |
| "227f2010-a6df-4c1b-9fb7-fc6cedeff8ce.2019-08-12T144035.510416Z", | |
| "6086e0e1-c679-44ad-bc62-57875b216fc4.2019-08-12T144035.510380Z", | |
| "00770ec4-3e00-47a6-be44-c9eeaa8a5d45.2020-02-06T235253.043943Z", | |
| "08ca411d-c58a-487a-b7da-4d5e83732252.2019-08-21T132355.516037Z", | |
| "1b81f5a4-9223-43d4-9f61-0c63eaf14ba5.2020-02-06T185446.956074Z", | |
| "286bfd51-a873-415b-b14e-bb6b629c9c2f.2019-08-05T225834.495519Z", | |
| "69db98a2-5d9c-4779-a2a9-49a271116d88.2019-09-05T182349.691081Z", | |
| "363aeb9a-010d-49ec-b8e0-9062552b9062.2019-08-21T061635.627527Z", | |
| "d4f85d6c-ed8a-4b22-a12b-a7b605865ced.2020-02-06T200313.596638Z", | |
| "9ca7449a-4b91-4cdd-b226-a7fbb7b1ffd0.2019-09-11T130901.205981Z", | |
| "fa09d968-3d16-4c6d-80f4-dd9a4505f4fa.2019-08-05T184718.657513Z", | |
| "d94c5530-a389-42d0-96a0-d17072e152fb.2020-02-06T204225.471880Z", | |
| "26dadf9e-4690-48df-b7f1-eac2b50c1462.2019-08-21T031539.262341Z", | |
| "17cc5a74-ac16-441f-ba26-666411eef921.2020-02-06T204246.280969Z", | |
| "3acc603d-b617-49ae-aa0d-353db0aa9e7e.2019-08-21T044630.263721Z", | |
| "0d7aee1d-a3d7-4532-903a-c531ac07998d.2019-08-12T144035.510451Z", | |
| "53efec69-fede-47e2-9155-192c2fa60e0c.2020-02-06T220049.198405Z", | |
| "b7e88f9e-45e4-4472-aa77-fc46886614a4.2019-08-12T144035.510542Z", | |
| "89a909ff-43e2-4ef9-832a-11ea6ab7689a.2019-08-22T221254.167287Z", | |
| "6d8a35b1-f652-4335-89a1-5936b9afe7fb.2020-02-06T204715.554317Z", | |
| "a551b36b-b74d-4205-b1bf-31b9f846d205.2020-02-06T183312.891066Z", | |
| "845933aa-b238-4c55-a6a0-4c622c67af53.2019-08-05T184721.920593Z", | |
| "ec1701e3-7b92-427e-895d-d95fe273837d.2020-02-06T211434.961672Z", | |
| "f269045b-1b3b-40d6-86ae-398a5b699e1d.2019-08-21T200208.078086Z", | |
| "fd74a009-b59b-4a5e-98f9-b6b987aa387d.2020-02-06T190140.649375Z", | |
| "97735720-4c1f-4b69-9463-e6c9febc2764.2019-08-21T204634.481290Z", | |
| "8d00a98f-9319-4afa-9c5f-c9b13f82bc28.2020-02-06T220612.318931Z", | |
| "c99ea041-0daf-4b25-bee9-d9691af11a5e.2019-08-12T144035.510496Z", | |
| "bf100561-a19c-4464-8904-bc765533fc3e.2020-02-06T235331.935298Z", | |
| "8642baea-d7c6-4d14-a6ec-a46fee5871f4.2019-08-05T200407.656720Z", | |
| "5d0b19de-3512-4192-a3e0-e2785b7112fc.2019-07-08T231648.329089Z", | |
| "2b0e8d36-9e0a-413a-9dd3-f0b8d95adbf7.2020-02-06T183651.484339Z", | |
| "2817d919-7ef1-42ec-b338-03674dd02feb.2019-08-05T224610.349229Z", | |
| "5b8fc043-b00e-4699-b75f-19be0c7d9dc0.2019-08-21T030950.661595Z", | |
| "76ccc6b1-51af-450d-84c6-d53c483cdc4d.2020-02-06T184110.735067Z", | |
| "905b7332-568f-4bee-9b7a-4bad5665e68e.2019-07-10T190546.990820Z", | |
| "e2223cbe-a550-46d1-9a70-5ee0fc439ca4.2020-02-06T201642.426315Z", | |
| "a182995c-83c9-4da9-8029-f73b1d742d1b.2019-08-21T203351.082780Z", | |
| "2baf256c-2c15-4587-81bd-852f8a3932b4.2019-07-10T190507.501929Z", | |
| "53d23c1c-7cab-4b09-ad07-c560c1e22f54.2020-02-06T190106.116016Z", | |
| "3b17e3c9-5f1c-49c6-b4fc-e3d658e7b731.2019-08-22T011145.980970Z", | |
| "70372f87-4cd4-40d7-8c09-1ebc641e1384.2019-08-21T041145.674161Z", | |
| "ab37c0a0-7617-42b3-8e4d-eebd201b702e.2019-08-02T190448.268469Z", | |
| "9e91edba-447e-47d3-8095-4406520e2fe9.2019-08-21T183339.122360Z", | |
| "d8f7f78d-1c77-49cc-b934-28d0da11c6d0.2019-08-12T144035.510251Z", | |
| "9cd161ce-4f1c-49c2-953e-0a5f80bf422f.2020-02-06T191356.115863Z", | |
| "f7081630-0d97-4550-8b16-77008ea868a3.2019-08-12T144035.510524Z", | |
| "5131a83e-abe6-4c0b-80c4-88491a6b133c.2019-07-10T185248.245209Z", | |
| "24cda1ca-d588-402e-880b-b560e6167527.2019-08-01T215522.012761Z", | |
| "284679d1-6b69-4d52-8774-c13943610299.2019-08-22T040300.065850Z", | |
| "67cdc6a8-9f47-4dd9-908d-9654e489361d.2019-08-12T144035.510362Z", | |
| "9d993f96-67d2-4640-8217-89b1ecc60028.2019-08-23T180850.874897Z", | |
| "7becab21-1efa-406b-8bc7-4d350febe688.2020-02-06T201239.846908Z", | |
| "b079e99c-42e7-4d9f-ae83-19f747b4adcb.2019-08-02T190138.629254Z", | |
| "6963e95a-f81d-4bfc-9639-46da2a958d7a.2019-08-12T144035.510223Z", | |
| "32b6880e-6c5c-41e4-b5b1-96b097cc52da.2019-08-12T144035.510515Z", | |
| "a870a1dd-4279-4ebc-bdc8-c9e7755435ba.2020-02-06T183712.624565Z", | |
| "86d19d63-aa71-4376-88b8-81c0a13d9b1c.2019-08-05T185443.996062Z", | |
| "2e373a6e-f531-449b-a989-eb163322ce83.2019-08-21T124432.677723Z", | |
| "3bab143b-606a-4890-afa1-7dfa92370cf7.2019-08-12T144035.510661Z", | |
| "6529d0e9-22ff-4276-99ca-884e45a5ceb3.2019-08-12T144035.510580Z", | |
| "95d5b3b3-705e-41a8-ba5d-a42120bd9e4b.2020-02-06T182527.793101Z", | |
| "3562d45c-57e4-4c25-8f02-ac52b3bd78b0.2020-02-06T220148.873960Z", | |
| "6a6bd6e3-af01-4e0c-81ad-97eb0bc15db5.2019-08-21T121907.088922Z", | |
| "88220b7f-0cdd-429c-9007-16121d151575.2019-11-05T171918.693612Z", | |
| "b70af688-8518-4361-9f43-510c575e9cbf.2020-02-06T185452.453569Z", | |
| "8cd7c4db-675b-4412-bb60-df0340245828.2019-08-12T144035.510533Z", | |
| "6faf2454-76ad-45d1-bfbe-ed07fc2ec812.2019-09-05T182214.717461Z", | |
| "4977a59e-26fc-4d7c-9ba6-61292e9619a7.2020-02-06T200241.549270Z", | |
| "62893d1d-91ea-4842-8d76-d6d4e67a795e.2019-08-12T144035.510327Z", | |
| "8e77dc03-6d1b-49e9-8913-d31bcb1a85c1.2020-02-06T220519.902349Z", | |
| "d487d37b-32e7-4ef9-954e-94013f9ce624.2020-02-06T215515.956018Z", | |
| "c062b332-ca7a-4081-b908-c73b4651b484.2020-02-06T201336.809023Z", | |
| "e618db92-72f7-4d00-9b94-dcd78aee29c3.2020-02-06T200308.668879Z", | |
| "f5ca6a1e-a870-4474-a50f-0dbeac647b40.2019-08-01T223624.412220Z", | |
| "1a26883e-d034-491c-8d0a-0e93e854eacb.2019-08-12T144035.510300Z", | |
| "868aa09f-e2ba-4350-a01b-3e31eba86bf3.2020-02-06T182532.813126Z", | |
| "48cd08f9-6adc-4deb-8ead-0d1fd7d73d84.2020-02-06T185807.729062Z", | |
| "8fe80644-0fc2-45c3-9f9a-cfdf62e227d6.2020-02-06T191746.017062Z", | |
| "49f72d9c-1e29-47d4-8c39-c2705fa54710.2020-02-06T182450.005436Z", | |
| "94b4d6a7-881f-42d8-ac42-fc61153b26ac.2018-08-17T202025.771186Z", | |
| "5c4b214d-ffd7-426b-88a2-cf611e5803f7.2019-08-12T144035.510613Z", | |
| "1e72e930-1ee8-4490-b367-3c6aa4d61cdd.2020-02-06T200632.025047Z", | |
| "a2703158-3f33-4a4a-bde6-b1ffde84e68e.2019-07-10T185853.163903Z", | |
| "c0e82caf-6fed-4524-9267-d97ac0a4d2f2.2019-08-21T201115.268364Z", | |
| "c02ec92e-1aa0-49ae-a2fd-ceeb4b0cbdb3.2019-08-21T142506.105155Z", | |
| "f838b9fe-3e03-41cf-8e1f-1f17db456975.2019-08-21T042049.677180Z", | |
| "4420213f-8c68-4559-8d76-b583820a1ff3.2019-08-22T020546.734540Z", | |
| "84709ab3-965d-4157-bdc9-f7a5c135ad14.2020-02-06T215407.724055Z", | |
| "30c1b159-8596-46d6-9095-f11aafc2857e.2020-02-06T235752.282023Z", | |
| "52245757-73ba-43a2-a78d-0eba01cb4ce1.2019-10-03T105524.918205Z", | |
| "21f0bde4-3a17-4f4d-9ba6-6673ecf6b309.2020-02-06T235336.697308Z", | |
| "54941366-0935-4941-9548-e86e5edc1285.2019-07-08T230621.912106Z", | |
| "2165e389-4760-4e89-98a8-49dbc0af00d0.2019-08-12T144035.510433Z", | |
| "820d48c7-a150-4a3f-81a6-1ca1d2665bad.2019-07-10T192013.756332Z", | |
| "cab1806b-c238-41aa-ad83-5408656640fd.2019-09-05T182345.933762Z", | |
| "f0d40e54-7ccd-4bdb-8d4b-e40256942760.2019-07-08T231627.366031Z", | |
| "98472b92-6aa4-4c27-97ad-52b9c2b531b4.2019-08-05T185110.383443Z", | |
| "b5236f06-6f11-494f-b5b3-41f3fafeb88e.2019-07-23T180000.212292Z", | |
| "ee7c7be6-979f-42e2-a189-608a06cdec22.2020-02-06T191254.950379Z", | |
| "e78fb1d9-e6f0-4c9e-8e66-d5d64d4fa970.2018-08-17T202915.922549Z", | |
| "eb4d5222-e363-4d91-888b-6c7b138fc059.2019-08-05T184632.417327Z", | |
| "f6d920a5-63ab-4d4b-8fb7-0fe2fdbf2fcd.2019-08-21T211132.456507Z", | |
| "a1e2d3d2-3b47-4d20-a331-548818f3604b.2019-08-21T153444.034252Z", | |
| "0dce2a49-1d81-45a2-98c3-75c1e154e96e.2019-08-12T144035.510336Z", | |
| "6d9faabd-5da1-4643-8f02-bce67d9fc9bb.2019-08-12T144035.510442Z", | |
| "6ff1a6f0-f214-45bc-878a-f8919cb3e581.2019-08-22T060015.593244Z", | |
| "76490c8a-62a5-4889-827a-b637a91d7202.2020-02-06T191351.040496Z", | |
| "f0f5d1f7-b777-431e-8618-c6f43ac7136a.2019-09-09T221719.097039Z" | |
| ] | |
| for i in range(0, len(bundles_to_tombstone), BATCH_SIZE): | |
| dss_client = _get_dss_client() | |
| fqids_batch = bundles_to_tombstone[i:i+BATCH_SIZE] | |
| with ThreadPoolExecutor(max_workers=20) as e: | |
| futures = [e.submit(tombstone_bundle, fqid) for fqid in fqids_batch] | |
| for f in as_completed(futures): | |
| try: | |
| bundle_found, bundle_already_tombstoned, fqid = f.result() | |
| except SwaggerAPIException as e: | |
| if 403 == e.code: # bail out on the first credentials denial | |
| sys.stderr.write(format_exc()) | |
| sys.exit(1) | |
| except Exception: | |
| sys.stderr.write(format_exc()) | |
| else: | |
| status_string = bundle_status_string(bundle_found, bundle_already_tombstoned) | |
| print(status_string, fqid) | |
| sys.stdout.flush() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment