cccntu

Run huggingface transformers on M1 Macs, on GPU

• Requirement: macOS 12 Monterey

• First, install a conda env with M1 support

wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh
chmod +x ~/Downloads/Miniforge3-MacOSX-arm64.sh
sh ~/Downloads/Miniforge3-MacOSX-arm64.sh

cccntu

from transformers import AutoTokenizer, AutoModelWithLMHead
  
tokenizer = AutoTokenizer.from_pretrained("gpt2")

# model = AutoModelWithLMHead.from_pretrained("gpt2")

import re

nums = []

cccntu

cccntu

cccntu

from tqdm.auto import tqdm
import time

class LengthedGeneratorWrapper:
    """wraps an infinite generator with length for tqdm"""
    def __init__(self, infinite_generator, len):
        self.generator = infinite_generator
        self.len = len
    def __len__(self):
        return self.len

cccntu

import sys

# imports utils and imports
from src import *

from dataclasses import dataclass
from typing import Optional
from omegaconf import OmegaConf

cccntu

❯ rm out.csv
❯ cat 1.py
from glob import glob
import mmap

files = glob("data/*")

files.sort(key=lambda x: int(x.split("/")[-1].split(".")[0]))
write_f = open("out.csv", "w+b")

cccntu

import shelve
import time
# initialize
with shelve.open('shelvedb') as db:
    for i in range(10000):
        db[f'{i}'] = '{i*2}'
       
# no caching
with shelve.open('shelvedb') as db:
    tic = time.time()

cccntu

# Copyright The PyTorch Lightning team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,

cccntu

conda install -c conda-forge julia
conda activate julia
conda install notebook
julia # use julia REPL to install IJulia (necessary)

> using Pkg
> Pkg.add("IJulia")