Skip to content

Instantly share code, notes, and snippets.

View ecchochan's full-sized avatar

ecchochan

9 followers · 1 following
View GitHub Profile
@ecchochan
ecchochan / cantokenizer.py
Last active June 8, 2020 03:35
example use for https://github.com/ecchochan/tokenizers
from tokenizers import Tokenizer, AddedToken, pre_tokenizers, decoders, trainers
from tokenizers.models import WordPiece
from tokenizers.normalizers import NFKC, Lowercase, Sequence
from tokenizers.normalizers import BertNormalizer
from tokenizers.pre_tokenizers import BertPreTokenizer
from tokenizers.processors import BertProcessing
from typing import Optional, List, Union
@ecchochan
ecchochan / jieba_hk.py
Created June 24, 2020 03:14
lang_utils
chengyu = {'政由己出', '監守自盜', '蒙面喪心', '分星劈兩', '直諒多聞', '民安國泰', '孤掌難鳴', '坐以待旦', '紅紫亂硃', '如湯灌雪', '歡忻鼓舞', '三日新婦', '忐忑不安', '跨山壓海', '窮原竟委', '七縱七擒', '裁心鏤舌', '燕舞鶯歌', '播糠眯目', '違害就利', '泥牛入海', '瞠乎其後', '神譁鬼叫', '執經問難', '後會有期', '孔席不四', '提綱振領', '鬩牆誶帚', '敝蓋不棄', '人多嘴雜', '一匡天下', '爽然若失', '大名難居', '高城深池', '材朽行穢', '投筆從戎', '富可敵國', '販夫皁隸', '不過爾爾', '閒不容瞚', '別有心肝', '討流溯源', '地醜德齊', '五零四散', '同仇敵慨', '膽大心細', '頓腳捶胸', '從長商議', '居心叵測', '騎馬尋馬', '言三語四', '養生喪死', '長材茂學', '綿綿緡緡', '八方呼應', '頭疼腦熱', '捩手覆羹', '舞爪張牙', '遭劫在數', '不寒而慄', '傳爵襲紫', '少見多怪', '遂非文過', '砲鳳烹龍', '烽火連天', '酌古沿今', '坐而論道', '瀝血披肝', '扇枕溫衾', '任達不拘', '賊喊捉賊', '以水濟水', '驚惶無措', '嘔心鏤骨', '助天爲虐', '蜂屯烏合', '躡手躡足', '龍肝豹胎', '進退爲難', '惡貫滿盈', '顯親揚名', '心懷叵測', '恰如其分', '曲學阿世', '目注心凝', '做剛做柔', '文過遂非', '人莫予毒', '櫛風釃雨', '以義割恩', '樂往哀來', '情見力屈', '津津樂道', '銜石填海', '紙醉金迷', '奇才異能', '聲振寰宇', '挨山塞海', '宵衣旰食', '飲谷棲丘', '避君三舍', '迎風待月', '多錢善賈', '孤懸客寄', '怡然自若', '雞鳴候旦', '成羣結夥', '一表人材', '破鏡重合', '眼高手生', '蹈赴湯火', '利令志惛', '避溺山隅', '蹈機握杼', '無傷大雅', '論高寡合', '喪魂失魄', '趁人之危', '同牀異夢', '張王李趙', '月夕花朝', '拔羣出萃', '視死如飴', '相剋相濟', '魚貫雁行', '重足屏氣', '截長補短
@ecchochan
ecchochan / install.sh
Last active August 25, 2020 16:24
Install patched huggingface tokenizers
##########################################
##
## Install Rust
##
##########################################
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
source $HOME/.cargo/env
rustc
@ecchochan
ecchochan / cythonize.py
Created August 14, 2020 05:19
zh-en split paragraphs into sentences
#defining NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
from Cython.Build import cythonize
from distutils.core import setup, Extension
from distutils.extension import Extension
import sys
sys.argv = [sys.argv[0]] + ['build_ext','--inplace']
@ecchochan
ecchochan / show_console_on_screen.js
Created January 16, 2021 06:24
Show console message on screen :D
var ConsoleLogHTML=function(e,t,n,o,r,i,l){"use strict";for(var f=0;f<t.length;f++)r!==typeof n[t[f]]&&(e[t[f]]=n[t[f]]);var c=n.skipHtml,a=o.keys(e),s=r!==typeof n.clear&&n.clear,u=typeof jQuery!==r&&jQuery,p=function(){for(var e,t,n={},r=0;r<arguments.length;r++)for(t=o.keys(arguments[r]),e=0;e<t.length;e++)n[t[e]]=arguments[r][t[e]];return n},d=function(t,o,r,i,f,c){n.skipHtml[t]=function(){e[t].apply(n,arguments)},n[t]=function(){var e,a,s,u;for(e="",s=0;s<arguments.length;s++){if(a=arguments[s]+"",a===l)try{a="Object "+JSON.stringify(arguments[s])}catch(e){}e+=(s>0?" ":"")+a}e=(i?"["+(new Date).toLocaleTimeString()+"] ":"")+e,u=document.createElement("li"),u.setAttribute("data-level",t),u.innerText=e,r[t]&&u.setAttribute("class",r[t]),c?o.appendChild(u):o.insertBefore(u,o.firstChild),f&&n.skipHtml[t].apply(n,arguments)}};return{DEFAULTS:{error:"text-danger",warn:"text-warning",info:"text-success",debug:"text-info",log:""},disconnect:function(){n.skipHtml=c;for(var t=0;t<a.length;t++)n[a[t]]=e[a[t]];!1!==
@ecchochan
ecchochan / WSL.md
Last active September 9, 2022 16:03

1. Windows WSL

To run Linux in Windows 10, we have 2 options:

  • WSL 1
  • WSL 2

To enable WSL, follow instructions here.

1.1. WSL 1

@ecchochan
ecchochan / bash.py
Last active February 28, 2021 09:58
Run Bash In Colab with live output
# https://stackoverflow.com/a/59339154/9945605
# wget https://gist.githubusercontent.com/ecchochan/eff0b341bf1744ecb4c5498ff186126f/raw/324b262c1b3f99da6f7ef68526b46776e742f6fa/bash.py
import signal, sys
import subprocess as sp
class VerboseCalledProcessError(sp.CalledProcessError):
def __str__(self):
if self.returncode and self.returncode < 0:
@ecchochan
ecchochan / macos_teminal_shortcuts.md
Created May 4, 2022 02:18
MacOS Terminal Shortcuts

Highligh All Searches

  1. Go to "System Preferences" -> "Keyboard"
  2. Open "Shortcuts" tab
  3. Select "App Shortcuts"
  4. Add a new short cut by
    • choosing the app "Terminal"
    • Type "Edit->Find->Select All" in the Menu Title
    • In Keyboard Shortcut, press "Cmd+Shift+F"
  • Save it
@ecchochan
ecchochan / jwks.json
Created March 3, 2023 08:52
???
{"keys":[{"kty":"RSA","n":"pyWbYsdmTN1HuJtZH8AH_8AnhS6j_WQRE6QCmRfhV5tf-3HlX7PkqdHBdqD69ur3WwkOlZ_Yw-VCqs1OrJ_Ey5Cdh6MLUYGjLPkT06_GjXrDa86JlxVnRIBYl6rLQeW9X1RjsEhrKvrG_i4xH27jCRBTvhNgt91U4wg2FD9hLGREkk407Vhj3VhBOX3VgZR0swc73NDNV2JvDdFkQaeyZ3YJJUwffBnUbnkKGJjk6zLxz5HrpdLs0DHTiqV3n7p-VmJO2L7kLsVB50w0-FuTt0P7h59Nfxmj9-4IoZWHuem6yjSBkm4RpoYOWp9y-yAkdzdGyJUE_rhFc4fVaGRFtQ","e":"AQAB","alg":"RS256","use":"sig","kid":"pfp"}]}
@ecchochan
ecchochan / jwks.json
Created March 16, 2023 09:23
!!!
{"keys":[{"crv":"Ed25519","x":"EN2oH3nXGFm3e82lAwZ4LjF4uYiSdGDsfHjF76TXWEw","kty":"OKP","alg":"EdDSA","use":"sig","kid":"pfp"}]}