make_sub_tarfiles.py

import tarfile
from pathlib import Path

import numpy as np

from sunpy.net import Fido, attrs as a

import dkist
import dkist.net

# Set this to be the parent directory of both datasets
WORKING_DIR = Path("/data/dkist/prod/pid_2_114")

# Uncomment this to run a globus transfer which will download the two datasets to the above dir
# (Assumes your Globus paths and filesystem paths are the same)
# dkist.net.transfer_complete_datasets(["BKPLX", "AJQWW"], path=WORKING_DIR/"{dataset_id}")

def make_tar(dataset, archive_dir=None, tarfilename=None, base_path=None, tile_slice=np.s_[...]):
    ds = dataset
    if isinstance(dataset, dkist.TiledDataset):
        filenames = []
        for tile in dataset.flat:
            filenames += tile[tile_slice].files.filenames
        # Pick one tile to use to get the base path
        ds = tile
    else:
        filenames = dataset.files.filenames

    archive_dir = archive_dir or Path(".")
    tarfilename = tarfilename.format(**dataset.inventory) or f"{dataset.inventory['datasetId']}.tar"
    base_path = base_path or ds.files.basepath

    with tarfile.open(archive_dir / tarfilename, "w") as tf:
        for fname in filenames:
            tf.add(base_path / fname, arcname=fname)
        if asdf_file := list(base_path.glob("*.asdf")):
            tf.add(asdf_file[0], arcname=asdf_file[0].name)


# VISP
bkplx = dkist.load_dataset(WORKING_DIR / "BKPLX")

make_tar(bkplx[0], tarfilename="{datasetId}_stokesI.tar", archive_dir=Path("~").expanduser())

# VBI
ajqww = dkist.load_dataset(WORKING_DIR / "AJQWW")

make_tar(ajqww, tile_slice=np.s_[0], tarfilename="{datasetId}_single_mosaic.tar", archive_dir=Path("~").expanduser())