basujindal/gtp4o_tokenizer.py

## gtp4o_tokenizer.py
### based on https://twitter.com/tianle_cai/status/1790109646205890723

import tiktoken
import langdetect
from tqdm import trange, tqdm
import concurrent.futures
import time
import csv


lang = "ja"
num_workers = 40

T = tiktoken.get_encoding("o200k_base")

# # Check token language
# def process_item(item):
#     try:
#         if langdetect.detect(T.decode([item])) == lang:
#             return T.decode([item])
#     except:
#         pass

# Check token language
def process_item(item):
    try:
        return (langdetect.detect(T.decode([item])),T.decode([item]))
    except:
        pass


# Sort by length
length_dict = {}
for i in trange(T.n_vocab):
    try:
        length_dict[i] = len(T.decode([i]))
    except:
        pass


length_dict = dict(sorted(length_dict.items(), key=lambda item: -item[1]))

# Using ProcessPoolExecutor
def parallel_process_with_process_pool(items):
    results = []
    with concurrent.futures.ProcessPoolExecutor(max_workers=num_workers) as executor:
        future_to_item = {executor.submit(process_item, item): item for item in items}
        for future in tqdm(concurrent.futures.as_completed(future_to_item), total=len(future_to_item), desc="Processing items"):
        # for future in concurrent.futures.as_completed(future_to_item):
            item = future_to_item[future]
            try:
                result = future.result()
                if result is not None:
                    results.append(result)
            except Exception as e:
                # pass
                print(f'Item {item} generated an exception: {e}')
    return results

# def write_results_to_file(results, filename):
#     with open(filename, 'w') as f:
#         for result in results:
#             f.write(f"{result}\n")

def write_results_to_csv(results, filename):
    # Extract unique languages
    languages = sorted(set(result[0] for result in results))

    with open(filename, 'w', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(languages)

        language_dict = {lang: [] for lang in languages}
        for result in results:
            language_dict[result[0]].append(result[1])

        max_length = max(len(words) for words in language_dict.values())

        for i in range(max_length):
            row = []
            for lang in languages:
                if i < len(language_dict[lang]):
                    row.append(language_dict[lang][i])
                else:
                    row.append("")
            writer.writerow(row)


print("Parallel processing using ProcessPoolExecutor:")
results_process = parallel_process_with_process_pool(length_dict)

write_results_to_csv(results_process, "results_process.csv")
	### based on https://twitter.com/tianle_cai/status/1790109646205890723

	import tiktoken
	import langdetect
	from tqdm import trange, tqdm
	import concurrent.futures
	import time
	import csv


	lang = "ja"
	num_workers = 40

	T = tiktoken.get_encoding("o200k_base")

	# # Check token language
	# def process_item(item):
	# try:
	# if langdetect.detect(T.decode([item])) == lang:
	# return T.decode([item])
	# except:
	# pass

	# Check token language
	def process_item(item):
	try:
	return (langdetect.detect(T.decode([item])),T.decode([item]))
	except:
	pass


	# Sort by length
	length_dict = {}
	for i in trange(T.n_vocab):
	try:
	length_dict[i] = len(T.decode([i]))
	except:
	pass


	length_dict = dict(sorted(length_dict.items(), key=lambda item: -item[1]))

	# Using ProcessPoolExecutor
	def parallel_process_with_process_pool(items):
	results = []
	with concurrent.futures.ProcessPoolExecutor(max_workers=num_workers) as executor:
	future_to_item = {executor.submit(process_item, item): item for item in items}
	for future in tqdm(concurrent.futures.as_completed(future_to_item), total=len(future_to_item), desc="Processing items"):
	# for future in concurrent.futures.as_completed(future_to_item):
	item = future_to_item[future]
	try:
	result = future.result()
	if result is not None:
	results.append(result)
	except Exception as e:
	# pass
	print(f'Item {item} generated an exception: {e}')
	return results

	# def write_results_to_file(results, filename):
	# with open(filename, 'w') as f:
	# for result in results:
	# f.write(f"{result}\n")

	def write_results_to_csv(results, filename):
	# Extract unique languages
	languages = sorted(set(result[0] for result in results))

	with open(filename, 'w', newline='') as f:
	writer = csv.writer(f)
	writer.writerow(languages)

	language_dict = {lang: [] for lang in languages}
	for result in results:
	language_dict[result[0]].append(result[1])

	max_length = max(len(words) for words in language_dict.values())

	for i in range(max_length):
	row = []
	for lang in languages:
	if i < len(language_dict[lang]):
	row.append(language_dict[lang][i])
	else:
	row.append("")
	writer.writerow(row)


	print("Parallel processing using ProcessPoolExecutor:")
	results_process = parallel_process_with_process_pool(length_dict)

	write_results_to_csv(results_process, "results_process.csv")