NikosAlexandris

import dace
import numpy as np
import time as timer
from rich import print
REPETITIONS_DEFAULT = 10


@dace
def myprogram_daced(a):
    for i in range(a.shape[0]):

NikosAlexandris

 - Attribute name _nc3_strict with value 1
 - Attribute name _NCProperties with value b'version=2,netcdf=4.7.3,hdf5=1.10.6,'
 - Attribute name Conventions with value b'CF-1.7,ACDD-1.3'
  + Attribute value = 'CF-1.7,ACDD-1.3' will be fed to n = 'Conventions' in 'zobj.attrs[Conventions]'

 - Attribute name id with value b'DOI:10.5676/EUM_SAF_CM/SARAH/V003'
  + Attribute value = 'DOI:10.5676/EUM_SAF_CM/SARAH/V003' will be fed to n = 'id' in 'zobj.attrs'

 - Attribute name product_version with value b'3.0'
  + Attribute value = '3.0' will be fed to n = 'product_version' in 'zobj.attrs'

NikosAlexandris

Input references : ['SISin202001020000004231000101MA.parquet', 'SISin202001010000004231000101MA.parquet']
Out is : {'.zgroup': b'{"zarr_format":2}'}
Out is : {'time/.zarray': b'{"chunks":[512],"compressor":null,"dtype":"<f8","fill_value":null,"filters":[{"id":"zlib","level":4}],"order":"C","shape":[48],"zarr_format":2}'}
Out is : {'time/0': FileNotFoundError('/SISin202001020000004231000101MA.nc')}
Out is : {'.zgroup': b'{"zarr_format":2}'}
Out is : {'time/.zarray': b'{"chunks":[512],"compressor":null,"dtype":"<f8","fill_value":null,"filters":[{"id":"zlib","level":4}],"order":"C","shape":[48],"zarr_format":2}'}
Out is : {'time/0': FileNotFoundError('/SISin202001010000004231000101MA.nc')}
Out is : {'.zgroup': b'{"zarr_format":2}'}
Out is : {'time/.zarray': b'{"chunks":[512],"compressor":null,"dtype":"<f8","fill_value":null,"filters":[{"id":"zlib","level":4}],"order":"C","shape":[48],"zarr_format":2}'}
Out is : {'time/.zattrs': b'{"_ARRAY_DIMENSIONS":["time"],"axis":"T"}'}

NikosAlexandris

from pathlib import Path
from functools import partial
import typer
from typing import Optional
import xarray as xr
from rich import print
import fsspec
from fsspec.implementations.reference import LazyReferenceMapper
from kerchunk.hdf import SingleHdf5ToZarr
from kerchunk.combine import MultiZarrToZarr

NikosAlexandris

import typer
import hashlib
from pathlib import Path
from functools import partial
import multiprocessing
from fsspec.implementations.reference import LazyReferenceMapper
import kerchunk
import os
import fsspec
from kerchunk.hdf import SingleHdf5ToZarr

NikosAlexandris

import math
import operator
import numpy as np
from itertools import product
from functools import reduce
import typer
from pvgisprototype.cli.typer_parameters import OrderCommands
from pydantic import BaseModel, ValidationError, conlist
from pydantic import ConfigDict
from pydantic import field_validator

NikosAlexandris

@app.command(
    'check-chunks',
    no_args_is_help=True,
    help='Check for chunk size consistency along series of files in a format supported by Xarray',
)
def check_chunk_consistency(
    source_directory: Annotated[Path, typer_argument_source_directory],
    pattern: Annotated[str, typer_option_filename_pattern] = "*.json",
    verbose: Annotated[int, typer.Option(..., "--verbose", "-v", count=True, help="Increase verbosity level.")] = 0,
):

NikosAlexandris

### Input configuration
#
# Example configuration:
# You can get the names of your inputs by running: swaymsg -t get_inputs
# Put your touchpad's ID to replace "Touchpad-ID" (keep the quotation marks)

input type:touchpad {
    dwt enabled
    tap enabled
    natural_scroll enabled

NikosAlexandris

Examples using for

Simple

• Print

for (k, v in $*) {
  print "KEY IS ". k . " VALUE IS ". v;
}

NikosAlexandris

Overview of Miller's streaming processing, and memory usage

Miller is streaming when possible (exceptions noted below) -- most verbs:

• operate and store in memory a single and independent record at a time
• don't have a state to retain from one record to the next
• don't wait for complete ingestion of input before producing any output
• (likewise, "statements" (i.e. $z = $x + $y) in the Miller programming language are implicit callbacks executed once per record)
• operate on files which are larger than the system's memory
• consume other programms output via pipe, e.g. tail -f some-file | mlr --some-flags