shcheklein

#!/usr/bin/env python
"""
Retry Processing Example with DataChain

This script demonstrates DataChain's retry functionality by:
1. Creating an initial in-memory dataset with sample data
2. Processing records with simulated errors
3. Using retry flags to reprocess failed records in a second pass
"""

shcheklein

{
    "data": {
        "dvclive/dvc.yaml::dvclive/plots/metrics/lr/pg0.tsv": [
            {
                "type": "vega",
                "revisions": [
                    "eef82dc"
                ],
                "content": {
                    "$schema": "https://vega.github.io/schema/vega-lite/v5.json",

shcheklein

{
  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
  "data": {
    "values": [
      {"step": 0, "accuracy_train": "0.0", "rev": "workspace"},
      {"step": 1, "accuracy_train": "0.0", "rev": "workspace"},
      {"step": 2, "accuracy_train": "0.0", "rev": "workspace"},
      {"step": 3, "accuracy_train": "0.0", "rev": "workspace"},
      {"step": 4, "accuracy_train": "0.0", "rev": "workspace"},
      {"step": 5, "accuracy_train": "0.0", "rev": "workspace"},

shcheklein

from img2dataset import download
import shutil
import os
from pyspark.sql import SparkSession  # pylint: disable=import-outside-toplevel

from pyspark import SparkConf, SparkContext


# Preparations:
#

shcheklein

#!/bin/bash

# A script to download LAION metada in parallel
#
# Based on https://github.com/rom1504/img2dataset/blob/main/dataset_examples/laion5B.md
#
# - Use dedicated EC2 instance, with high network bandwith to make it faster
# - It should be taking an hour to download everything
#
# Usage example for normal dataset:

shcheklein

name: report

on: [push, pull_request]

jobs:
  run:
    runs-on: [ubuntu-latest]
    container: docker://dvcorg/cml-py3:latest
    steps:
      - uses: actions/checkout@v2

shcheklein

name: report-status

on: [push, pull_request]

jobs:
  run:
    runs-on: [ubuntu-latest]
    container: docker://dvcorg/cml-py3:latest
    steps:
      - uses: actions/checkout@v2

shcheklein

(.env) √ ~/Projects % dvc run -n a echo
(.env) √ ~/Projects % dvc run -n b echo
(.env) √ ~/Projects % cat dvc.yaml
stages:
  a:
    cmd: echo
  b:
    cmd: echo

shcheklein

  1325068    2.830    0.000   10.061    0.000 /mnt/ivan/dvc/dvc/utils/__init__.py:336(current_timestamp)
        7    0.001    0.000   10.499    1.500 /mnt/ivan/dvc/dvc/utils/__init__.py:112(dict_md5)
   443471    0.972    0.000   10.666    0.000 /mnt/ivan/dvc/dvc/state.py:307(_update_state_record_timestamp_for_inode)
  2650131    2.835    0.000   10.669    0.000 /mnt/ivan/dvc/.env/lib/python3.6/site-packages/nanotime/__init__.py:123(seconds)
   440796    1.904    0.000   10.949    0.000 /mnt/ivan/dvc/dvc/system.py:26(hardlink)
  1325069    4.983    0.000   11.126    0.000 /mnt/ivan/dvc/dvc/system.py:193(inode)
 18550649    6.868    0.000   11.130    0.000 /mnt/ivan/dvc/.env/lib/python3.6/posixpath.py:41(_get_sep)
  1325070    2.359    0.000   11.638    0.000 /mnt/ivan/dvc/dvc/state.py:145(_fetchall)
  6184578    3.550    0.000   11.854    0.000 /usr/lib/python3.6/logging/__init__.py:1286(debug)
        2    1.319    0.660   12.023    6.011 /usr/lib/python3.6/json/__init__.py:120(dump)

shcheklein

(.env) [ivan@ivan ~/Projects/test]$ ./test.sh
+ set -e
+ pip uninstall -y dvc
Uninstalling dvc-0.59.2:
  Successfully uninstalled dvc-0.59.2
+ pip install dvc==0.59.2
Collecting dvc==0.59.2
  Using cached https://files.pythonhosted.org/packages/78/fa/5d9fc54fbf99c75efd9cfbd81b479b2e3d0572b59694523d24c280cad018/dvc-0.59.2-py2.py3-none-any.whl
Requirement already satisfied: future>=0.16.0 in ./.env/lib/python3.7/site-packages (from dvc==0.59.2) (0.17.1)
Requirement already satisfied: asciimatics>=1.10.0 in ./.env/lib/python3.7/site-packages (from dvc==0.59.2) (1.11.0)