Skip to content

Instantly share code, notes, and snippets.

View tmanabe's full-sized avatar

Tomohiro Manabe tmanabe

View GitHub Profile
#!/usr/bin/env python
from argparse import ArgumentParser
from transformers import AutoModel, AutoTokenizer
import torch
"""
Usage: ./sort_vocab_by_norm.py \
line-corporation/line-distilbert-base-japanese --trust-remote-code
#!/usr/bin/env python
# Efficiently Evaluating Complex Boolean Expressions [Yahoo, SIGMOD 2010]
# https://theory.stanford.edu/~sergei/papers/sigmod10-index.pdf
class Node(object):
__slots__ = ["name", "_children", "_left_leaves", "begin", "end"]
def __init__(n, name, *_children):
#!/usr/bin/env python
import numpy
import sys
numpy.set_printoptions(edgeitems=4, linewidth=120, suppress=True)
ls = sys.stdin.read().split("\n")
from tensorflow_hub import load
#!/usr/bin/env python
class ZScorer(object):
__slots__ = ['means', 'stds']
def __init__(self):
self.means, self.stds = [], []
def learn(self, rank_lib):
#!/usr/bin/env python3
# coding: utf-8
from json import dumps
from numpy import array
from numpy import logical_and
from numpy import piecewise
from scipy import optimize
from sys import argv
#!/usr/bin/env python3
from collections import defaultdict
LABEL_CAP = 4
LETOR_DATASET = 'C:/letor/MSLR-WEB10K/Fold1/test.txt'
OLQ_CLICK_LOG = 'C:/OpenLiveQ_yahoo_data/OpenLiveQ-clickthrough.tsv'
def avg(l):
#!/usr/bin/env python
# coding: utf-8
from math import log
class BASE(object):
def __init__(self, id_to_relevance):
# Document ID -> Relevance
@tmanabe
tmanabe / trie_node.py
Created May 5, 2017 13:11
A very simple implementation of trie in Python. A node can represent a word as well as a letter.
#!/usr/bin/env python
# coding: utf-8
class TrieNode(object):
'''A node in a trie.'''
__slots__ = ['body', 'list_count', 'metakey']
def __init__(self, metakey='!'):
'''intializes a node in a trie.'''
#!/usr/bin/env python
# coding: utf-8
import matplotlib.pyplot as plt
import sys
from xgboost import Booster
from xgboost import plot_tree
from xgboost import XGBClassifier