This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
# -*- coding: utf-8 -*- | |
# Authors: Chantal Amrhein and Mathias Müller | |
# University of Zurich | |
# imported modules | |
import argparse # for argument handling | |
import codecs # for opening files | |
from nltk import ngrams # for creating ngrams |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /bin/bash | |
# change base path before running this | |
base=/net/cephfs/scratch/mathmu/map-volatility/debug/complete | |
# install | |
virtualenv -p python3 $base/venvs/sockeye3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /bin/bash | |
chunk_input_dir=$1 | |
chunk_output_dir=$2 | |
chunk_prefix=$3 | |
chunk_index=$4 | |
model_paths=$5 | |
batch_size=$6 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /bin/env python3 | |
import numpy as np | |
import mxnet as mx | |
import argparse | |
import logging | |
def parse_args(): | |
parser = argparse.ArgumentParser() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /bin/bash | |
# change base path before running this | |
base=/net/cephfs/scratch/mathmu/map-volatility/debug/complete | |
# install | |
virtualenv -p python3 $base/venvs/sockeye3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /usr/bin/python3 | |
import argparse | |
import threading | |
import logging | |
import numpy | |
from subprocess import Popen, PIPE | |
from queue import Queue, Empty |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import mxnet as mx | |
import numpy as np | |
import random | |
from collections import namedtuple | |
def run(seed: int, init_method: str): | |
mx.random.seed(seed) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Author: Mathias Müller / mmueller@cl.uzh.ch | |
import mxnet as mx | |
import numpy as np | |
from typing import Optional, List | |
SOFTMAX_NAME = "softmax" | |
WEIGHTED_CROSS_ENTROPY_NAME = "weighted_cross_entropy" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
# author: Mathias Mueller / mathias.mueller@uzh.ch | |
# purpose: apply word alignment models trained with fast_align | |
# usage information | |
if [ $# -lt 6 ] | |
then | |
echo "[ERROR] Too few arguments. Expected 6 command line arguments." 1>&2 | |
echo "Usage: $0 <language 1> <language 2> <source txt file> <target txt file> <directory of trained model> <output file path>" 1>&2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
# author: Mathias Mueller / mathias.mueller@uzh.ch | |
# purpose: train word alignment models with fast_align | |
# usage information | |
if [ $# -lt 4 ] | |
then | |
echo "[ERROR] Too few arguments. Expected 4 command line arguments." 1>&2 | |
echo "Usage: $0 <language 1> <language 2> <path to training set without language suffix> <output directory for trained model>" 1>&2 |
NewerOlder