This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/env python | |
#usage argv[0] file_tobe_split target_dir | |
#按第一字段切分文件 | |
import sys | |
src_file = sys.argv[1] | |
tgt_dir = sys.argv[2] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
if [[ $# != 2 && $# != 3 ]] | |
then | |
echo '需要至少两个参数' >&2 | |
echo 'usage:' | |
echo "$(basename $0) file1 file2 tmpdir" >&2 | |
exit 1 | |
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def counter(interval): | |
if not hasattr(counter,'count'): | |
counter.count = 0 | |
counter.count += 1 | |
import sys | |
if counter.count % interval == 0: print >>sys.stderr, counter.count |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sys | |
lines = [] | |
for line in sys.stdin: | |
line = line.rstrip() | |
tokens = line.split('\t') | |
src, tgt = tokens[0], tokens[1] | |
lines.append(((src,tgt), line)) #TODO | |
lines.sort(key=lambda a:a[0]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import sys | |
import heapq | |
fileList = [open(sys.argv[1] + '/' + f) for f in os.listdir(sys.argv[1])] | |
def lineIterator(f): | |
for line in f: | |
line = line.rstrip() | |
tokens = line.split('\t') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
env = Environment(CPPFLAGS=["-std=gnu++0x", "-Wall", "-O0"]) | |
env.SharedLibrary('foo', ['f1.cpp']) | |
env.Program("test.cpp", LIBS=["boost_regex", "boost_locale", "foo"], LIBPATH='.') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler | |
class TestHTTPHandler(BaseHTTPRequestHandler): | |
def do_GET(self): | |
self.protocal_version = 'HTTP/1.1' | |
self.send_response(200) | |
self.send_header("Welcome", "Contect") | |
self.end_headers() | |
self.wfile.write('hello world') | |
http_server = HTTPServer(('0.0.0.0', 7778), TestHTTPHandler) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
syntax on | |
colorscheme desert | |
set number | |
set cindent | |
set tabstop=4 | |
set shiftwidth=4 | |
set expandtab | |
set termencoding=utf-8 | |
set fileencodings=utf-8 | |
set encoding=utf-8 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <iostream> | |
#include <fstream> | |
#include <string> | |
#include <vector> | |
#include <unordered_map> | |
#include <boost/algorithm/string.hpp> | |
#include <utf8proc.h> | |
//https://unicode.org/reports/tr15/#Norm_Forms | |
//https://ssl.icu-project.org/apiref/icu4c/uchar_8h.html |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* c++ version of tokenization for bert | |
Copyright (C) 2019 luistung | |
This program is free software: you can redistribute it and/or modify | |
it under the terms of the GNU General Public License as published by | |
the Free Software Foundation, either version 3 of the License, or | |
(at your option) any later version. | |
This program is distributed in the hope that it will be useful, | |
but WITHOUT ANY WARRANTY; without even the implied warranty of |
OlderNewer