Forked from radiaph/gist:d931eb3ff9f23f1fd336db3122fcd335
Created
April 16, 2021 16:21
-
-
Save Madakkak/699c9958d762e16e89730c14eafbbe60 to your computer and use it in GitHub Desktop.
Ingest Status data shape working proposal
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { // begin Collection Session | |
| "id": 12345, | |
| "parentFlowId": 123, // all other Flows in this Collection Session are children | |
| "flows": [ | |
| { | |
| "id": 123, | |
| "meta": { | |
| "sequenceNumber": 12345, // logical timestamp used to impose ordering on asynchronous updates | |
| "type": "WHITE_GLOVE" | "NATIVE_LITE" | "SELF_SERVE_NATIVE" | "SELF_SERVE_LOADFILE" | "SELF_SERVE_OVERLAY" | "SERVER_ADMIN_NATIVE" | "SERVER_ADMIN_LOADFILE" | "SERVER_ADMIN_OVERLAY", | |
| "name": "", | |
| "description": "", | |
| "createdBy": "user@domain", | |
| "createdByFullName": "FIRST LAST" | null, | |
| "createdAt": 0.0, | |
| "completedAt": 0.0 | null, | |
| "options": { | |
| "handleExceptions": true, | |
| "instructions": "" | |
| // possibly more in the future | |
| }, | |
| "state": "QUEUED" | "ACTIVE" | "PAUSED" | "TERMINATED", | |
| "canPause": true, // Can only be true during transfer ("upload" stage). | |
| "canDelete": true, // Always false once ingest ("process" stage) completes. | |
| "canRollback": true, // Always false until ingest ("process" stage) completes. | |
| "timeLeft": 0.0 | |
| }, | |
| "progress": { | |
| // UPLOAD phase | |
| "location": { | |
| "id": "123:location", // first part is the transfer ID | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "files": 789, // for Native Lite, this will be exactly 1 | |
| "bytes": 123456789 | |
| }, | |
| "detection": { // this maps to the "Validating Files" step of native flows | |
| "id": "123:detection", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "files": 456, | |
| }, | |
| "transfer": { | |
| "id": "123:transfer", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "files": 123, | |
| "bytes": 123456, | |
| "rate": 0.0 | null | |
| }, | |
| "transferReporting": { | |
| "id": "123:transferReporting", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "report": "https://somewhere-on-s3" | null, | |
| "agentLog": "https://somewhere-on-s3" | null, | |
| }, | |
| "validation": { // this maps to the "Validating Files" step of loadfile/overlay flows | |
| "id": "456:validation", // first part is the validation ID | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "filesProcessed": 123, | |
| "filesTotal": 456, | |
| }, | |
| "validationReporting": { | |
| "id": "123:validationReporting", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "report": "https://somewhere-on-s3" | null | |
| }, | |
| // PROCESS phase | |
| "expansion": { | |
| "id": "789:expansion", // first part is the ingest ID | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "filesProcessed": 123, | |
| "filesTotal": 456 | |
| }, | |
| "filtering": { | |
| "id": "789:filtering", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "filesProcessed": 123, | |
| "filesTotal": 456 | |
| }, | |
| "extraction": { | |
| "id": "789:extraction", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "filesProcessed": 123, | |
| "filesTotal": 456 | |
| }, | |
| "deduplication": { | |
| "id": "789:deduplication", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "filesProcessed": 123, | |
| "filesTotal": 456 | |
| }, | |
| "documentCreation": { | |
| "id": "789:documentCreation", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "filesProcessed": 123, | |
| "filesTotal": 456 | |
| }, | |
| "taggingFoldering": { | |
| "id": "789:taggingFoldering", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "filesProcessed": 123, | |
| "filesTotal": 456 | |
| }, | |
| "ingestReporting": { | |
| "id": "789:ingestReporting", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "newDocuments": 123456 | null, | |
| "billingSize": 1234567890 | null, | |
| "ingestReport": "https://somewhere-on-s3" | null, | |
| "exceptions": 12 | null, | |
| "exceptionReport": "https://somewhere-on-s3" | null | |
| }, | |
| // ENRICH phase | |
| "simDocs": { | |
| "id": "789:simDocs", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "filesProcessed": 123, | |
| "filesTotal": 456 | |
| }, | |
| "threading": { | |
| "id": "789:threading", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED", | |
| "filesProcessed": 123, | |
| "filesTotal": 456 | |
| }, | |
| // QC phase | |
| "remediation": { | |
| "id": "789:remediation", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED" | |
| }, | |
| // ROLLBACK phase | |
| "rollback": { | |
| "id": "789:rollback", | |
| "sequenceNumber": 12345, | |
| "state": "NOT_REQUIRED" | "PENDING" | "ACTIVE" | "DONE" | "FAILED | |
| } | |
| } | |
| } | |
| ] | |
| } | |
| ] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment