napsternxg

# Download the data from: https://gpc-browser.gs1.org/ using Download GPC as JSON

import json

data_file = "./data/GPC_May2024.json"
output_file = "./data/GPC_May2024.flattened.json"

with open(data_file) as fp:
    data = json.load(fp)

napsternxg

from collections import defaultdict
from typing import Callable


class LazyVal:
    _uncomputed_val = object()

    def __init__(self, fn, *args, **kwargs) -> None:
        self._val = self._uncomputed_val
        self.args = args

napsternxg

Brands: https://www.walmart.com/brand/branddirectory

let nodes = document.querySelectorAll("#maincontent > div > div > div:nth-child(3) > div > div > div > a")
let data = Array.from(nodes).map(d => Object.assign({link: d.href, name: d.text}))
console.log(JSON.stringify(data, null, 2))

Departments: https://www.walmart.com/all-departments

napsternxg

napsternxg

import onnx

model_path = "./model.onnx"
fixed_model_path = model_path.replace(".onnx", ".fixed.onnx")

# # Load the ONNX model which should have last layer as Sigmoid. 
# LGBM Models may sometime not add the Sigmoid op during export when using regression loss
onnx_model = onnx.load(model_path)
print(onnx_model)
onnx.checker.check_model(onnx_model)

napsternxg

napsternxg

from IPython.display import display, HTML

class DisplayEntities:
    @classmethod
    def display(cls, texts, grouped_entities):
        html = []
        html.append(cls.get_style())
        for text, entities in zip(texts, grouped_entities):
            html.append(cls.show_entities(text, entities))
        display(HTML("".join(html)))

napsternxg

import functools

import pandas as pd
import torch
import transformers
from accelerate import Accelerator
from datasets import Dataset
from torch.utils.data import DataLoader
from tqdm.auto import tqdm

napsternxg

import asyncio
import logging
import random
import time
from dataclasses import dataclass
from typing import Any

from tqdm.auto import tqdm

logger = logging.getLogger(__name__)

napsternxg

from datasets import load_dataset, Dataset, DatasetDict
from sentence_transformers.losses import CosineSimilarityLoss
from sentence_transformers import SentenceTransformer

from setfit import SetFitModel, SetFitTrainer, sample_dataset
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np
import json