pszemraj

import json
import socket
import uuid

import yake
from flask import Flask, redirect, render_template_string, request, url_for
from markupsafe import escape

app = Flask(__name__)

pszemraj

import torch
import logging


def check_ampere_gpu():
    """
    Check if the GPU supports NVIDIA Ampere or later and enable FP32 in PyTorch if it does.
    """
    # Check if CUDA is available
    if not torch.cuda.is_available():

pszemraj

"""
cli.py - Command line interface for textsum.

this edition: fast CPU inference with intel IPEX https://archive.ph/oY5b1

Usage:
    textsum-dir --help
"""

import os

pszemraj

import json
import logging
import re
from datetime import datetime
from pathlib import Path

import datasets
import evaluate
import fire
import intel_extension_for_pytorch as ipex

pszemraj

#!/usr/bin/env python
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#

pszemraj

"""Summary
"""
import logging
from pathlib import Path

import fire
from datasets import Dataset, load_dataset
from tqdm.auto import tqdm
from transformers import AutoTokenizer

pszemraj

from pathlib import Path
import json


def update_tokenizer_post_processor(input_path):
    """
    Load a tokenizer configuration from the input path, update its post_processor
    with a custom TemplateProcessing configuration, and overwrite the original file.

    Args:

pszemraj

# pip install sentence-transformers
from sentence_transformers import SentenceTransformer, util, models

model_name = "nomic-ai/nomic-embed-text-v1"
pooling_mode = "mean"
word_embedding_model = models.Transformer(
    model_name,
    max_seq_length=8192,
    model_args={"trust_remote_code": True, "rotary_scaling_factor": 2},
    tokenizer_args={"trust_remote_code": True},

pszemraj

# pip install sentence-transformers -q
# source: https://www.sbert.net/docs/usage/semantic_textual_similarity.html

from sentence_transformers import SentenceTransformer, util

model = SentenceTransformer("all-MiniLM-L6-v2")

# Two lists of sentences
sentences1 = [
    "The cat sits outside",

pszemraj

from transformers import AutoTokenizer


def load_and_ensure_tokens(model_name):
    # Load the tokenizer
    tokenizer = AutoTokenizer.from_pretrained(model_name)

    # Essential special tokens with their default values
    essential_tokens = {
        "pad_token": "<pad>",