This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import tensorflow as tf | |
from gensim.corpora import WikiCorpus | |
import os | |
import argparse | |
# lang = 'fa' farsi | |
def store(corpus, lang): | |
base_path = os.getcwd() | |
store_path = os.path.join(base_path, '{}_corpus'.format(lang)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from simpletransformers.language_modeling import LanguageModelingModel | |
import logging | |
logging.basicConfig(level=logging.INFO) | |
transformers_logger = logging.getLogger("transformers") | |
transformers_logger.setLevel(logging.WARNING) | |
train_args = { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import tensorflow | |
from tensorflow.keras.preprocessing.image import ImageDataGenerator | |
# Create training ImageDataGenerator object | |
train_data_gen = ImageDataGenerator(rotation_range=50, | |
width_shift_range=0.2, | |
height_shift_range=0.2, | |
zoom_range=0.3, | |
horizontal_flip=True, | |
vertical_flip=True, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import tensorflow | |
from PIL import Image | |
import glob | |
from tensorflow.keras.preprocessing.image import ImageDataGenerator | |
from tensorflow.keras.preprocessing.image import img_to_array | |
from tensorflow.keras.preprocessing.image import load_img | |
import numpy as np | |
aug = ImageDataGenerator( | |
rotation_range=30, |