Scope: This file governs all development workflows.
Act as a senior, very experienced, software engineer who:
- Loathes complexity
- Doesn't trust user reports, validates all claims locally
| """Processes a CSV file using AutoML's object detection format into tfrecords. | |
| This script will accept a CSV file path or URL and write tfrecord files at the | |
| provided output path. Example usage: | |
| ``` | |
| mkdir -p /tmp/salad_dataset | |
| python tfrecords_from_dataset_descriptor.py \ | |
| --csv_path=gs://cloud-ml-data/img/openimage/csv/salads_ml_use.csv \ | |
| --output_path=/tmp/salad_dataset | |
| ``` |
| { | |
| "embeddings": [ | |
| { | |
| "tensorName": "My tensor", | |
| "tensorShape": [ | |
| 1000, | |
| 50 | |
| ], | |
| "tensorPath": "https://gist.githubusercontent.com/owahltinez/eab8d4c2afc6966857fea7cbb5add900/raw/8bf1fd142644b1f1a7b139c9754e6ffbc130e33c/tensors.tsv", | |
| "metadataPath": "https://gist.githubusercontent.com/owahltinez/3781ae090f317371103a72022916c173/raw/0bafaf1ff3e1934417b2d5a0704db077ce9d39d2/metadata.tsv" |
| Asru70 | |
| Asru56 | |
| Asru76 | |
| Asru50 | |
| Asru58 | |
| Asru56 | |
| Asru55 | |
| Asru70 | |
| Asru58 | |
| Asru76 |
| -2.247926779091358185e-02 4.126220196485519409e-02 2.201553061604499817e-02 4.435282200574874878e-02 4.219446703791618347e-02 7.351647363975644112e-04 -1.420196741819381714e-01 3.519088402390480042e-02 7.742006331682205200e-02 1.293895300477743149e-02 7.603470981121063232e-02 -1.221615076065063477e-01 -2.842330746352672577e-02 -6.303963717073202133e-03 -3.483119606971740723e-02 2.308848127722740173e-02 8.119910210371017456e-02 1.293514519929885864e-01 1.212856918573379517e-01 -7.862001657485961914e-02 -1.989656686782836914e-01 -1.172609031200408936e-01 1.417047232389450073e-01 5.320393666625022888e-02 -4.523742944002151489e-02 -1.421208977699279785e-01 -1.612507738173007965e-02 -8.112639933824539185e-02 -1.910826005041599274e-02 1.615460962057113647e-01 -4.172783717513084412e-02 1.076077520847320557e-01 1.423008143901824951e-01 -9.967331588268280029e-02 -1.324560940265655518e-01 7.485345005989074707e-02 -1.712135504931211472e-03 -7.529520243406295776e-02 4.165450483560562134e-02 -3.049586899578571320e-02 -7.7 |
| import json | |
| import os | |
| import sys | |
| import time | |
| def list_files(opts: str, marker: str = None) -> list: | |
| fpath = f"/tmp/{int(1E9 * time.monotonic())}.json" | |
| marker_opt = f'--marker "{marker}"' if marker else "" | |
| os.system(f'az storage fs file list {opts} --show-next-marker {marker_opt} > "{fpath}"') |
| key | city_wikidata | city_name | population | |
|---|---|---|---|---|
| AD_02 | Q24413 | El Tarter | 747 | |
| AE_AJ | Q530171 | Ajman | 238119 | |
| AE_AZ | Q1519 | Abu Dhabi | 1000000 | |
| AE_AZ | Q234600 | Al Ain | 631005 | |
| AE_AZ | Q234600 | Al Ain | 766936 | |
| AE_AZ | Q12241857 | Madinat Zayed | 29095 | |
| AE_DU | Q612 | Dubai | 2502715 | |
| AE_FU | Q4045 | Fujairah | 93673 | |
| AE_RK | Q2126436 | Ras al-Khaimah | 115949 |
| key | country_code | country_name | subregion1_code | subregion1_name | subregion2_code | subregion2_name | locality_code | locality_name | match_string | aggregate_report_offset | |
|---|---|---|---|---|---|---|---|---|---|---|---|
| BR_AC_1200013 | BR | Brazil | AC | Acre | 1200013 | Acrelândia | |||||
| BR_AC_1200054 | BR | Brazil | AC | Acre | 1200054 | Assis Brasil | |||||
| BR_AC_1200104 | BR | Brazil | AC | Acre | 1200104 | Brasiléia | |||||
| BR_AC_1200138 | BR | Brazil | AC | Acre | 1200138 | Bujari | |||||
| BR_AC_1200179 | BR | Brazil | AC | Acre | 1200179 | Capixaba | |||||
| BR_AC_1200203 | BR | Brazil | AC | Acre | 1200203 | Cruzeiro do Sul | |||||
| BR_AC_1200252 | BR | Brazil | AC | Acre | 1200252 | Epitaciolândia | |||||
| BR_AC_1200302 | BR | Brazil | AC | Acre | 1200302 | Feijó | |||||
| BR_AC_1200328 | BR | Brazil | AC | Acre | 1200328 | Jordão |
| // RECOMMENDED: Bind use cases to a lifecycle in a single call: | |
| CameraX.bindToLifecycle( | |
| this as LifecycleOwner, preview, imageCapture, imageAnalyzer) | |
| // Do *NOT* bind your use cases like this | |
| CameraX.bindToLifecycle(this as LifecycleOwner, preview) | |
| CameraX.bindToLifecycle(this as LifecycleOwner, imageCapture) | |
| CameraX.bindToLifecycle(this as LifecycleOwner, imageAnalyzer) |