This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import argparse | |
import hashlib | |
import os | |
import urllib | |
import warnings | |
from collections import OrderedDict | |
import torch | |
from torch import nn | |
from tqdm import tqdm |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import torch | |
import transformers | |
import argparse | |
from pytictoc import TicToc | |
def load_model_and_tokenizer(model_name): | |
the_tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) | |
tokenizer_model = transformers.AutoModel.from_pretrained(model_name) | |
return tokenizer_model, the_tokenizer |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
46c46 | |
< #define GETSERVBYPORT_R_ARGS 6 | |
--- | |
> #define GETSERVBYPORT_R_ARGS | |
131c131 | |
< #define HAVE_GETSERVBYPORT_R | |
--- | |
> /* #undef HAVE_GETSERVBYPORT_R */ | |
46c46 | |
< #define GETSERVBYPORT_R_ARGS 6 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
sudo add-apt-repository ppa:ubuntu-toolchain-r/test | |
sudo apt-get update | |
sudo apt-get install gcc-5 g++-5 | |
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 60 --slave /usr/bin/g++ g++ /usr/bin/g++-5 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/ruby | |
require 'json' | |
require 'json-schema' | |
schema = '{ | |
"$schema": "http://json-schema.org/draft-04/schema#", | |
"title": "demo", | |
"type": "object", | |
"properties": { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/ruby | |
require 'json' | |
require 'json-schema' | |
schema = '{ | |
"$schema": "http://json-schema.org/draft-04/schema#", | |
"title": "demo", | |
"type": "object", | |
"properties": { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"posts" : { | |
"post" : { | |
"properties" : { | |
"converted_text" : { | |
"type" : "string" | |
}, | |
"converted_text_size" : { | |
"type" : "long" | |
}, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Post < ActiveRecord::Base | |
include Tire::Model::Search | |
include Tire::Model::Callbacks | |
mapping do | |
indexes :id, :index => :not_analyzed | |
indexes :title, :analyzer => 'snowball', :boost => 3 | |
indexes :url, :analyzer => 'url' | |
indexes :created_at, :type => 'date', :include_in_all => false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Searches ElasticSearch fulltext engine via Tire | |
# | |
# @param term Search term to be used | |
# It has to start with a prefix specifying the field(s) to be searched: | |
# e.g. | |
# _all:Apple | |
# converted_text:Apple | |
# field_summary:Apple, etc. | |
# | |
# Each Lucene search term can be used |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Tests str for valid UTF-8 as described here: | |
# https://secure.wikimedia.org/wikipedia/en/wiki/Utf8 | |
# | |
# The following options are supported: | |
# | |
# :bmp_only Basic Multilingual Plane: only 1,2 byte characters are valid | |
# :debug prints debug info to stdout in case of an error | |
# | |
def valid_utf8?(str, options={}) | |
debug = options[:debug] || false |
NewerOlder