I hereby claim:
- I am luistung on github.
- I am luistung (https://keybase.io/luistung) on keybase.
- I have a public key ASCcsP70NQH1pe_YHkb_VWNteyojKMnNa4gSEpNRxAF3_Qo
To claim this, I am signing this object:
#lang racket/base | |
(require quickscript | |
fmt ; needs to be installed first | |
racket/class | |
racket/set | |
racket/list) | |
;;; Author: Laurent Orseau | |
;;; License: [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) or | |
;;; [MIT license](http://opensource.org/licenses/MIT) at your option. |
from transformers import AutoTokenizer | |
from datasets import Dataset | |
import torch | |
import pandas as pd | |
import numpy as np | |
tokenizer = AutoTokenizer.from_pretrained("gpt2") | |
tokenizer.pad_token = tokenizer.eos_token # 设置填充符号 | |
qa_pairs = [ |
import torch | |
from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments | |
from datasets import Dataset | |
# 选择模型,这里可以替换为任何 transformers 支持的模型,如 "bert-base-uncased", "gpt2" 等 | |
model_name = "gpt2" | |
device = torch.device("cpu") | |
# 加载模型和分词器 | |
model = AutoModelForCausalLM.from_pretrained(model_name).to(device) |
I hereby claim:
To claim this, I am signing this object:
cmake_minimum_required(VERSION 3.0 FATAL_ERROR) | |
project(custom_ops) | |
find_package(Torch REQUIRED) | |
add_executable(example-app example-app.cpp) | |
target_link_libraries(example-app "${TORCH_LIBRARIES}") | |
set_property(TARGET example-app PROPERTY CXX_STANDARD 14) |
/* c++ version of tokenization for bert | |
Copyright (C) 2019 luistung | |
This program is free software: you can redistribute it and/or modify | |
it under the terms of the GNU General Public License as published by | |
the Free Software Foundation, either version 3 of the License, or | |
(at your option) any later version. | |
This program is distributed in the hope that it will be useful, | |
but WITHOUT ANY WARRANTY; without even the implied warranty of |
#include <iostream> | |
#include <fstream> | |
#include <string> | |
#include <vector> | |
#include <unordered_map> | |
#include <boost/algorithm/string.hpp> | |
#include <utf8proc.h> | |
//https://unicode.org/reports/tr15/#Norm_Forms | |
//https://ssl.icu-project.org/apiref/icu4c/uchar_8h.html |
syntax on | |
colorscheme desert | |
set number | |
set cindent | |
set tabstop=4 | |
set shiftwidth=4 | |
set expandtab | |
set termencoding=utf-8 | |
set fileencodings=utf-8 | |
set encoding=utf-8 |
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler | |
class TestHTTPHandler(BaseHTTPRequestHandler): | |
def do_GET(self): | |
self.protocal_version = 'HTTP/1.1' | |
self.send_response(200) | |
self.send_header("Welcome", "Contect") | |
self.end_headers() | |
self.wfile.write('hello world') | |
http_server = HTTPServer(('0.0.0.0', 7778), TestHTTPHandler) |
env = Environment(CPPFLAGS=["-std=gnu++0x", "-Wall", "-O0"]) | |
env.SharedLibrary('foo', ['f1.cpp']) | |
env.Program("test.cpp", LIBS=["boost_regex", "boost_locale", "foo"], LIBPATH='.') |