f0k

MIT License

Copyright (c) 2017 Jan Schlüter

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

noahtren

import tensorflow as tf
import tensorflow_text as text
from transformers import AutoTokenizer

def get_tf_tokenizer(hf_model_name, do_test=False):
  hf_tokenizer = AutoTokenizer.from_pretrained(hf_model_name)
  model_proto = hf_tokenizer.sp_model.serialized_model_proto()
  tf_tokenizer = text.SentencepieceTokenizer(model=model_proto, out_type=tf.int32)
  if do_test:
    test_string = "This is a testtt, hah! reaaly cool :)"

harupy

@pytest.hookimpl(hookwrapper=True)
def pytest_report_teststatus(report, config):
    outcome = yield
    res = outcome.get_result()

    attr_name = "___TIME___"
    if report.when == "setup":
        # HACK: store the start time in `config`
        setattr(config, attr_name, time.time())
    elif report.when == "call":