| import os | |
| import pytest | |
| import s3fs | |
| import fsspec | |
| import pickle | |
| from multiprocessing import Process, Queue | |
| fs = fsspec.filesystem('local') | |
| Thread 9160 (idle): "MainThread" | |
| wait (threading.py:331) | |
| _wait_once (grpc/_common.py:116) | |
| wait (grpc/_common.py:156) | |
| _next (grpc/_channel.py:959) | |
| __next__ (grpc/_channel.py:542) | |
| run (apache_beam/runners/worker/sdk_worker.py:262) | |
| main (apache_beam/runners/worker/sdk_worker_main.py:212) | |
| <module> (apache_beam/runners/worker/sdk_worker_main.py:367) | |
| _run_code (<frozen runpy>:88) |
LocalDirectRunner"disabled" we mean you never set c.InputCacheStorage.fsspec_class in your confighttps://github.com/ranchodeluxe/staged-recipes/tree/gpm_zarr_failcase
| Source | s3://gesdisc-cumulus-prod-protected | s3://gesdisc-cumulus-prod-protected | s3://gesdisc-cumulus-prod-protected |
|---|
| Worker: severity: ERROR timestamp { seconds: 1706984149 nanos: 336367845 } message: "OSError: [Errno 22] Bad Request [while running \'Create|OpenWithKerchunk|FilterVars|WriteCombinedReference|ValidateDatasetDimensions/OpenWithKerchunk/Open with Kerchunk\']\nTraceback (most recent call last):\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 113, in _error_wrapper\n return await func(*args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/aiobotocore/client.py\", line 371, in _make_api_call\n raise error_class(parsed_response, operation_name)\nbotocore.exceptions.ClientError: An error occurred (400) when calling the HeadObject operation: Bad Request\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"apache_beam/runners/common.py\", line 1435, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 851, in apache_beam.runners.common.P |
| Estimate summary | |
| Upfront cost,Monthly cost,Total 12 months cost,Currency | |
| 0,1649.33,19791.97,USD | |
| ,,* Includes upfront cost | |
| Detailed Estimate | |
| Group hierarchy,Region,Description,Service,Upfront,Monthly,First 12 months total,Currency,Configuration summary | |
| My Estimate,US West (Oregon),,AWS Secrets Manager,0,2.01,24.12,USD,"Number of secrets (5), Average duration of each secret (30 days), Number of API calls (2000 per month)" | |
| My Estimate,US West (Oregon),,Amazon Elastic Container Registry,0,97.16,1165.92,USD,"DT Inbound: Internet (0 TB per month), DT Outbound: Internet (1 TB per month), Amount of data stored (50 GB per month), Data transfer cost (92.16)" |
| Inter-application communication/networking options in ECS versus K8s with an emphasis on avoiding setting up load balancers: | |
| +--------------+----------------------------------------------+-----------------+-----------------+-------------+ | |
| | FEATURE | DESCRIPTION | AWS ECS | KUBERNETES | CONCLUSION | | |
| +--------------+----------------------------------------------+-----------------+-----------------+-------------+ | |
| | | Multiple containers per TaskDef/Pod. | | | | | |
| | | Supports inter-container communication | supports | supports | | | |
| | **Sidecar | via loopback on `localhost:<port>`. This | inter-container | inter-container | | | |
| | Networking** | also means port collisions have to be | comms | comms | | |
| 3/3/2023, 5:52:48 AM f25f239053e947c88640866bb6691653 | |
| Installing collected packages: opentelemetry-instrumentation-urllib3 tf-veda-wfs3-service-dev | |
| 3/3/2023, 5:52:47 AM f25f239053e947c88640866bb6691653 | |
| Requirement already satisfied: deprecated>=1.2.6 in /opt/bitnami/python/lib/python3.10/site-packages (from opentelemetry-api~=1.12->opentelemetry-instrumentation-urllib3==0.37b0) (1.2.13) tf-veda-wfs3-service-dev | |
| 3/3/2023, 5:52:47 AM f25f239053e947c88640866bb6691653 | |
| Requirement already satisfied: setuptools>=16.0 in /opt/bitnami/python/lib/python3.10/site-packages/setuptools-63.4.3-py3.10.egg (from opentelemetry-instrumentation==0.37b0->opentelemetry-instrumentation-urllib3==0.37b0) (63.4.3) tf-veda-wfs3-service-dev | |
| 3/3/2023, 5:52:47 AM f25f239053e947c88640866bb6691653 | |
| Requirement already satisfied: opentelemetry-instrumentation==0.37b0 in /opt/bitnami/python/lib/python3.10/site-packages (from opentelemetry-instrumentation-urllib3==0.37b0) (0.37b0) tf-veda-wfs3-service-dev | |
| 3/3/2023, 5:52:47 AM f25f239053e94 |
| addons: | |
| - name: vpc-cni | |
| version: 1.7.5 # not sure what version we are currently at b/c I don't have any k8 RBAC access | |
| attachPolicyARNs: | |
| - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy | |
| serviceAccountRoleARN: arn:aws:iam::aws:policy/<whatever> | |
| tags: | |
| team: eks | |
| attachPolicy: | |
| Statement: # just showing what should currenlty exist for this role |
| { | |
| "cluster_name":"elasticsearch", | |
| "nodes":{ | |
| "EYc9thF9ROy-pSMMrX4UoQ":{ | |
| "timestamp":1466627259673, | |
| "name":"MN-E (Ultraverse)", | |
| "transport_address":"127.0.0.1:9300", | |
| "host":"127.0.0.1", | |
| "ip":[ | |
| "127.0.0.1:9300", |
| { | |
| "cluster_name":"elasticsearch", | |
| "nodes":{ | |
| "Rg-xqnDESdmQnqkuTgHbhw":{ | |
| "timestamp":1466461514864, | |
| "name":"Snowfall", | |
| "transport_address":"127.0.0.1:9300", | |
| "host":"127.0.0.1", | |
| "ip":[ | |
| "127.0.0.1:9300", |