Skip to content

Instantly share code, notes, and snippets.

View ozancaglayan's full-sized avatar

Ozan Caglayan ozancaglayan

View GitHub Profile
from typing import List, Optional, Tuple, Dict
from torch.utils.data import Dataset
from torch.utils.data.sampler import RandomSampler, Sampler
class MultiTaskBatchSampler(Sampler):
def __init__(self, datasets: Dict[str, Dataset],
batch_size: int,
epoch_size: Optional[int] = None,
--- rawdata-v4-1882.txt 2018-08-13 15:23:10.065617891 +0300
+++ rawdata-v5.txt 2018-08-13 15:18:22.598301871 +0300
@@ -113,8 +113,7 @@
112;Halil Kuru;_halilkuru;23 Haz 2014;@06melihgokcek adlı kullanıcıya yanıt olarak @06melihgokcek başgan hackerlara terörist dedin. RTE'ye bir rica etde İSİD'e de terör örgütü deyiversin. Akhack ve redhack hangi kategoride?;0 yanıt 0 retweet 0 beğenme
113;B a t ı n.wav;batinmuzik;23 Haz 2014;Düne kadar Redhack taraftar olanlar,bugün Redhack'in terör örgütü olduğunu söyleyen ve img'i hackleyenlerle gurur duyuyor.Ne döneksiniz.;0 yanıt 0 retweet 0 beğenme
114;Kızıl;zapatistass;21 Haz 2014;Bir kuruluş kalbi sosyalizm düşmanlığıyla tutuşmuş insanlardan kuruluysa tabiki redhack'a terör örgütü der.;0 yanıt 0 retweet 0 beğenme
-115;Banu Barlas;banubarlass;17 Haz 2014;Flash flash flash Twitter geziye destek veren RedHack siber terör örgütünün adresini sildi http://www.akdenizpress.com/haber/65/twitter-redhacki-sildi ...
-2 retweet 0 beğenme
+115;Banu Barlas;banubarlass;17 Haz 2014;
@ozancaglayan
ozancaglayan / image_encoder.py
Created March 29, 2018 17:04
pytorch cnn image encoder
# -*- coding: utf-8 -*-
#
# author: Ozan Caglayan
from collections import OrderedDict
import torch
from torchvision import models
from torchvision.models.vgg import cfg as vgg_cfg
@ozancaglayan
ozancaglayan / slurm_aliases.rc
Last active June 8, 2022 09:47
Aliases for SLURM
###############
# SINFO aliases
###############
# Detailed sinfo
alias si="sinfo -o '%8P %10n %.11T %.4c %.8z %.6m %12G %10l %10L %10O %20E' -S '-P'"
# sinfo only on GPU partition
alias sig="si -p gpu"
# sinfo only on CPU partition
alias sic="si -p cpu"
@ozancaglayan
ozancaglayan / node_gpu_exporter.py
Created November 29, 2017 11:16
NVIDIA GPU textfile exporter
#!/usr/bin/env python3
import time
import atexit
from collections import OrderedDict
from py3nvml import py3nvml as nv
from prometheus_client import Gauge, CollectorRegistry
from prometheus_client import write_to_textfile
@ozancaglayan
ozancaglayan / weightedsampler.py
Created October 31, 2017 17:37
WeightedBatchSampler for PyTorch
class WeightedBatchSampler(Sampler):
def __init__(self, n_elems, batch_size,
initial_p=None, epoch_p_reset=False):
self.n_elems = n_elems
self.batch_size = batch_size
self.epoch_p_reset = epoch_p_reset
self.n_batches = math.ceil(self.n_elems / self.batch_size)
if initial_p is None:
@ozancaglayan
ozancaglayan / mtevalv13a.py
Created July 5, 2017 11:07
mteval-v13a.pl Python3 port
#!/usr/bin/env python3
import re
import math
import argparse
from collections import defaultdict
LOG_2 = math.log(2)
###################################################################################
adam-512emb-1000lstm-wdecay-feature-reshape.sampling Bleu_1: 0.46259 Bleu_2: 0.27595 Bleu_3: 0.16279 Bleu_4: 0.09660 CIDEr: 0.22156 METEOR: 0.22565 ROUGE_L: 0.33607
adam-512emb-1000lstm-feature-reshape.sampling Bleu_1: 0.45442 Bleu_2: 0.26771 Bleu_3: 0.15705 Bleu_4: 0.09146 CIDEr: 0.19169 METEOR: 0.21855 ROUGE_L: 0.32499
adam-512emb-1000lstm-wdecay-feature-reshape.beam12.1best Bleu_1: 0.45167 Bleu_2: 0.26816 Bleu_3: 0.15995 Bleu_4: 0.09787 CIDEr: 0.26480 METEOR: 0.24464 ROUGE_L: 0.34140
adam-512emb-1000lstm-feature-reshape-noprevctx.sampling Bleu_1: 0.44216 Bleu_2: 0.25580 Bleu_3: 0.14835 Bleu_4: 0.08502 CIDEr: 0.18997 METEOR: 0.22077 ROUGE_L: 0.32038
adam-512emb-1000lstm-feature-reshape.beam12.1best Bleu_1: 0.44151 Bleu_2: 0.25829 Bleu_3: 0.15098 Bleu_4: 0.09008 CIDEr: 0.24039 METEOR: 0.23844 ROUGE_L: 0.33360
adam-512emb-1000lstm-feature-reshape-noprevctx.beam12.1best Bleu_1: 0.43102 Bleu_2: 0.2487
Wemb_enc, -0.05387, 0.05577
Wemb_dec, -0.05280, 0.05521
encoder_W, -0.05016, 0.04749
encoder_b, 0.00000, 0.00000
encoder_U, -0.17179, 0.16453
encoder_Wx, -0.04784, 0.04707
encoder_bx, 0.00000, 0.00000
encoder_Ux, -0.15236, 0.15887
encoder_r_W, -0.04918, 0.04668
encoder_r_b, 0.00000, 0.00000
local argparse = require "argparse"
local moses = require "moses"
local parser = argparse("build_dictionary", "example")
parser:argument("input", "Input text file"):args('+')
parser:option('-o --output', 'Output directory', '.')
parser:option('-m --minfreq', 'Filter out words occuring < m times.', 0)
local args = parser:parse()