mreso

$python repro.py
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
/home/ubuntu/miniconda3/envs/serve230/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2637: UserWarning: `max_length` is ignored when `padding`=`True` and there is no truncation strategy. To pad to max length, use `padding='max_length'`.
  warnings.warn(
Traceback (most recent call last):
  File "/home/ubuntu/miniconda3/envs/serve230/lib/python3.10/site-packages/torch/utils/_pytree.py", line 1264, in _treespec_to_json
    serialized_context = json.dumps(treespec.context)
  File "/home/ubuntu/miniconda3/envs/serve230/lib/python3.10/json/__init__.py", line 231, in dumps
    return _default_encoder.encode(obj)

mreso

#Saving snapshot
#Thu Feb 08 04:34:52 UTC 2024
python=/home/ubuntu/miniconda3/envs/serve0.9.0/bin/python
model_snapshot={\n  "name"\: "20240208043452822-shutdown.cfg",\n  "modelCount"\: 1,\n  "created"\: 1707366892822,\n  "models"\: {\n    "mnist"\: {\n      "1.0"\: {\n        "defaultVersion"\: true,\n        "marName"\: "mnist.mar",\n        "minWorkers"\: 16,\n        "maxWorkers"\: 16,\n        "batchSize"\: 1,\n        "maxBatchDelay"\: 100,\n        "responseTimeout"\: 120\n      }\n    }\n  }\n}
tsConfigFile=logs/config/20240208043104864-shutdown.cfg
version=0.9.0
workflow_store=model_store
load_models=mnist\=mnist.mar
model_store=model_store
number_of_gpu=0

mreso

# Copyright (c) Meta Platforms, Inc. and affiliates.
# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement.

# For dataset details visit: https://huggingface.co/datasets/samsum

import copy
import datasets
import itertools

from llama_recipes.datasets.utils import Concatenator

mreso

diff --git a/radish/bert/bert_tokenizer.cc b/radish/bert/bert_tokenizer.cc
index 75e10a1..f7267ba 100644
--- a/radish/bert/bert_tokenizer.cc
+++ b/radish/bert/bert_tokenizer.cc
@@ -16,7 +16,7 @@

 #include "radish/utils/basic_string_util.h"
 #include "radish/utils/logging.h"
-#include "source/utf8.h"
+#include "utf8.h"