wconstab

import torch
from torch import nn
class NestedWrap(torch.nn.Module):
    def __init__(self, sequential_model):
        super().__init__()
        for layer in sequential_model.children():
            def hook(module, input, output):
                print(f"module {id(module)} got input {input}")
            layer.register_forward_hook(hook)
        self.sequential_model = sequential_model

wconstab

nk0]:[2023-08-17 10:07:17,530] torch._dynamo.backends.distributed: [INFO]
[rank0]:[2023-08-17 10:07:17,530] torch._dynamo.backends.distributed: [INFO] DDPOptimizer used bucket cap 26214400 and created 2 buckets. Enable debug logs for detailed bucket info.
[rank0]:[2023-08-17 10:07:17,536] torch._dynamo.backends.distributed: [DEBUG]
[rank0]:[2023-08-17 10:07:17,536] torch._dynamo.backends.distributed: [DEBUG] DDPOptimizer produced the following bucket assignments:
[rank0]:[2023-08-17 10:07:17,536] torch._dynamo.backends.distributed: [DEBUG] ┌─────────┬────────────┬───────────────────────────────────┐
[rank0]:[2023-08-17 10:07:17,536] torch._dynamo.backends.distributed: [DEBUG] │   Index │   Size (b) │ Param Names                       │
[rank0]:[2023-08-17 10:07:17,536] torch._dynamo.backends.distributed: [DEBUG] ├─────────┼────────────┼───────────────────────────────────┤
[rank0]:[2023-08-17 10:07:17,536] torch._dynamo.backends.distributed: [DEBUG] │       0 │    1050624 │ L__self___level3_conv_weight      │

wconstab

aten.arange.default
aten.arange.start
aten.arange.start_step
aten.arange.start_out
aten.arange.out
aten.bitwise_and_.Tensor
aten.bitwise_and_.Scalar
aten.bitwise_or_.Tensor
aten.bitwise_or_.Scalar
aten.clamp_min_.default

wconstab

cuda eval  opacus_cifar10                      [2023-03-27 23:34:30,308] torch._dynamo.symbolic_convert: [INFO] Step 1: torchdynamo start tracing forward_pass
[2023-03-27 23:34:30,308] torch._dynamo.symbolic_convert: [DEBUG] TRACE starts_line benchmarks/dynamo/torchbench.py:362
[2023-03-27 23:34:30,308] torch._dynamo.symbolic_convert: [DEBUG] TRACE LOAD_FAST self []
[2023-03-27 23:34:30,308] torch._dynamo.symbolic_convert: [DEBUG] TRACE LOAD_ATTR autocast [UserDefinedObjectVariable(TorchBenchmarkRunner)]
[2023-03-27 23:34:30,309] torch._dynamo.symbolic_convert: [DEBUG] TRACE CALL_FUNCTION 0 [UserDefinedClassVariable()]
[2023-03-27 23:34:30,309] torch._dynamo.symbolic_convert: [DEBUG] TRACE SETUP_WITH 30 [UserDefinedObjectVariable(NullContext)]
[2023-03-27 23:34:30,309] torch._dynamo.symbolic_convert: [DEBUG] step triggered compile
Traceback (most recent call last):
  File "/scratch/whc/work/pytorch/torch/_dynamo/symbolic_convert.py", line 564, in step
    getattr(self, inst.opname)(inst)

wconstab

Issue: triton codegen is somehow sensitive to the 'default device'- it must be set to the per-rank GPU even before 'main' in order to not crash

wconstab

from ctypes import c_void_p, c_long
import torch
import random
from torch import empty_strided, as_strided, device
from torch._inductor.codecache import AsyncCompile

aten = torch.ops.aten
assert_size_stride = torch._C._dynamo.guards.assert_size_stride
async_compile = AsyncCompile()

wconstab

Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/wav2vec2-large-960h-lv60-self and are newly initialized: ['wav2vec2.masked_spec_embed']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
/scratch/whc/work/pytorch/torch/_dynamo/eval_frame.py:361: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled.Consider setting `torch.set_float32_matmul_precision('high')`
  warnings.warn(
Found cached dataset librispeech_asr_dummy (/data/home/whc/.cache/huggingface/datasets/patrickvonplaten___librispeech_asr_dummy/clean/2.1.0/f2c70a4d03ab4410954901bde48c54b85ca1b7f9bf7d616e7e2a72b5ee6ddbfc)
It is strongly recommended to pass the ``sampling_rate`` argument to this function. Failing to do so can result in silent errors that might be hard to debug.
/scratch/whc/work/pytorch/torch/storage.py:315: UserWarning: TypedStorage is deprecated. It will be removed in the future and Untyped

wconstab

import argparse
from typing import Dict
from ray.air import session

import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor

wconstab

import os

import pandas as pd
import torch
import torch._dynamo
from pytorch_lightning import LightningModule, Trainer
from pytorch_lightning.callbacks.progress import TQDMProgressBar
from pytorch_lightning.loggers import CSVLogger
from torch import nn
from torch.nn import functional as F

wconstab

# coding=utf-8
# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software