pszemraj

#!/bin/bash

# Set environment variables
export WANDB_PROJECT="pileT5-summ"
export WANDB_WATCH="gradients"
export WANDB_ENTITY="pszemraj"

NUM_WORKERS=$(lscpu -p | egrep -v '^#' | sort -u -t, -k 2,4 | wc -l)
echo "Number of CPU cores: $NUM_WORKERS"

pszemraj

Messages Overview - 2024-03-30 04:20:45 - Total Messages: 6

User - Msg No. 1/6

Can you give me an up to date overview of Atlanta and the different areas of the city

Assistant - Msg No. 2/6

Sure, I can provide you with an overview of Atlanta and its different areas. Atlanta is the capital and most populous city in the state of Georgia, with a diverse population and a thriving economy. Here's a breakdown of some of the main areas:

pszemraj

"""
Creates an archive of a directory


pip install fire
"""

import os
import shutil
from pathlib import Path

pszemraj

import pkg_resources


def list_dependencies(package_name, level=0, explored=set()):
    # Define indent outside of try-except to ensure it's always assigned
    indent = "    " * level
    if package_name in explored:
        return
    explored.add(package_name)

pszemraj

from datasets import Dataset

# Your initial list of dictionaries
data = [
    {"id": 1, "text": "Hello world!", "label": 0},
    {"id": 2, "text": "How are you?", "label": 1},
    # Add more dictionaries as needed
]

# Convert list of dictionaries to a dictionary of lists

pszemraj

"""
anthropic_run_summarization.py - Generate summaries using langchain + LLMs

For usage details, run `python anthropic_run_summarization.py --help` and fire will print the usage details.

Notes:
- you need to have ANTHROPIC_API_KEY set as an environment variable (easiest way is export ANTHROPIC_API_KEY=memes123)
- install the dependencies using the requirements.txt file or below

    pip install fire langchain langchain-community langchain-anthropic clean-text tqdm tiktoken

pszemraj

from rapidfuzz import process, fuzz


def fuzzy_align(masterlist, list2, cutoff=70):
    # Dictionary to hold matches
    matches = {}

    # Track used indices to avoid duplicate matches in the masterlist
    used_indices = set()

pszemraj

import logging
from email.parser import BytesParser
from pathlib import Path

import fire
import html2text
import pandas as pd
from tqdm import tqdm

# Setup logging

pszemraj

import os

import numpy as np

from datasets import ClassLabel, Dataset, DatasetDict


def split_dataset(
    dataset: Dataset,
    test_size=0.025,

pszemraj

"""
this script will upload a folder to Hugging Face Hub

    python upload_folder.py --help

pip install fire huggingface-hub
"""

import logging