FastText Language Detection - Training on macOS

fasttext_lid_training.sh

#!bin/bash

# grab labeled dataset
wget http://downloads.tatoeba.org/exports/sentences.tar.bz2
bunzip2 sentences.tar.bz2
tar xvf sentences.tar

# macos only for gshuf
brew install coreutils
awk -F"\t" '{print"__label__"$2" "$3}' < sentences.csv | gshuf > all.txt
head -n 10000 all.txt > valid.txt
tail -n +10001 all.txt > train.txt

# accuracy is 96.5%
./fasttext supervised -input train.txt -output langdetect -dim 16
./fasttext test langdetect.bin valid.txt

# accuracy is 98.5%
./fasttext supervised -input train.txt -output langdetect -dim 16 -minn 2 -maxn 4
./fasttext test langdetect.bin valid.txt