lebrice

"""A simple tool to add the name of downloaded paper pdf's in front of the id.
(Written by fabrice.normandin@gmail.com)


If there are multiple downloads of same paper, replaces the original with the
latest download. This can be useful in a downloads folder filled with copies.
For instance:
"""

import glob

lebrice

from __future__ import annotations

import inspect
from dataclasses import dataclass, field, Field, fields
from typing import Any, Callable, TypeVar, overload
from logging import getLogger as get_logger

logger = get_logger("conditional_fields")
T = TypeVar("T")

lebrice

## Flower follow-up Multiprocessing error, can't pickle _thread.Lock object

""" Follow-up to [last week's question](https://hackmd.io/OsKKGG3QSTawhaWMMMRaeA#Using-FLWR-How-do-I-run-LOTS-of-clients).

The solution from last week (using multiprocessing.Pool) doesn't quite work.
Solution: Use Processes instead:

Solution: Use the --array option of sbatch to run multiple jobs, and then use multiprocessing to run multiple processes within each job. However, Use Processes instead:
"""
import multiprocessing as mp

lebrice

from dataclasses import dataclass
import os
from simple_parsing import ArgumentParser
from itertools import product


@dataclass
class ProblemConfig:
    dataset: int = 0  # Which dataset ID to use.
    rank: int = 0  # The rank of some matrix

lebrice

from __future__ import annotations

# Context: Dataset is on GPU memory.
from typing import Iterable
import torch
from torch import Tensor
from torchvision.datasets import MNIST
from torch.utils.data import TensorDataset, DataLoader, Dataset, ConcatDataset

lebrice

"""Sets up a user cache directory for commonly used libraries, while reusing shared cache entries.

Use this to avoid having to download files to the $HOME directory, as well as to remove
duplicated downloads and free up space in your $HOME and $SCRATCH directories.

The user cache directory should be writeable, and doesn't need to be empty.
This command adds symlinks to (some of) the files contained in the *shared* cache directory to this
user cache directory.

The shared cache directory should be readable (e.g. a directory containing frequently-downloaded

lebrice

""" ImageNet datamodule adapted to the Mila cluster.

Can be used either with a PyTorch-Lightning Trainer, or by itself to easily get efficient
dataloaders for the ImageNet dataset.

Requirements (these are the versions I'm using, but this can probably be loosened a bit).
- pytorch-lightning==1.6.0
- lightning-bolts==0.5
"""

lebrice

from __future__ import annotations
from pytorch_lightning import Trainer
from typing_extensions import Unpack, ParamSpec
from typing import Callable, TypedDict

# Option A: TypedDict
# --> lots of code duplication!


class TrainerConfig(TypedDict, total=False):

lebrice

from __future__ import annotations
import itertools
from pytorch_lightning import Trainer
import torch
import numpy as np
from pytorch_lightning import LightningModule
from torch import nn, Tensor
import pytest
from .imagenet_ffcv import ImagenetFfcvDataModule

lebrice

""" ImageNet datamodule that uses FFCV. """

from __future__ import annotations

import typing
from collections.abc import Iterable, Sequence
from pathlib import Path
from typing import Any, Callable, TypeVar

import cv2  # noqa