cli99

import datetime
import gc
import pathlib
import torch

from composer.utils import get_device
from omegaconf import OmegaConf as om
from llmfoundry.models.mpt.modeling_mpt import ComposerMPTCausalLM
from composer.core import Precision
from composer import Trainer

cli99

import torch
import torch.nn as nn
from typing import Optional, Dict, Union, List
import math
from llmfoundry.models.layers.attention import GroupedQueryAttention
from contextlib import nullcontext
from omegaconf import OmegaConf as om
from einops import rearrange
import transformer_engine.pytorch as te
from transformer_engine.common import recipe

cli99

# Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# See LICENSE for license information.

import os
import argparse
from functools import partial
import contextlib
import torch
import torch.distributed as dist

cli99

import torch
from typing import List


def my_fn(x):
    x = x * 2
    x = x.tolist()
    x += [1, 2]
    # To torch tensor
    x = torch.tensor(x)

cli99

import os
from functools import partial

import torch
from torch.distributed.fsdp import FullyShardedDataParallel, MixedPrecision
from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy
from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import (
    apply_activation_checkpointing,
    checkpoint_wrapper
)

cli99

trap '/tmp/code tunnel unregister' EXIT
cd /tmp && curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz
tar -xf vscode_cli.tar.gz
/tmp/code tunnel --accept-server-license-terms --name mml-dev-01

cli99

import os
import torch
from composer.utils import get_device
from omegaconf import OmegaConf as om
from llmfoundry.models.mpt.modeling_mpt import ComposerMPTCausalLM
from composer.core import Precision
from composer import Trainer

import transformer_engine.pytorch as te
from transformer_engine.common import recipe

cli99

# Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# See LICENSE for license information.

import os
import argparse
import warnings

warnings.filterwarnings("ignore")

cli99

import os
from functools import partial

import torch
from torch.distributed.fsdp import FullyShardedDataParallel, MixedPrecision
from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy
from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import (
    apply_activation_checkpointing,
    checkpoint_wrapper
)

cli99

The repository for the assignment is public and Github does not allow the creation of private forks for public repositories.

The correct way of creating a private frok by duplicating the repo is documented here.

For this assignment the commands are:

1. Create a bare clone of the repository. (This is temporary and will be removed so just do it wherever.)

git clone --bare git@github.com:usi-systems/easytrace.git