sgraaf

sgraaf

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from argparse import ArgumentParser

import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DataLoader, Dataset
from torch.utils.data.distributed import DistributedSampler
from transformers import BertForMaskedLM

sgraaf

🌞 Morning   131 commits  ███████▏░░░░░░░░░░░░░  34.3%
🌆 Daytime   179 commits  █████████▊░░░░░░░░░░░  46.9%
🌃 Evening    69 commits  ███▊░░░░░░░░░░░░░░░░░  18.1%
🌙 Night       3 commits  ▏░░░░░░░░░░░░░░░░░░░░   0.8%

sgraaf

from itertools import cycle

from lxml import html
from requests import Response, Session


class RotatingProxySession(Session):

    def __init__(self) -> None:
        super().__init__()

sgraaf

# coding: utf-8
from itertools import cycle
from pathlib import Path
from typing import Optional

from lxml import html
from requests import Response, Session


class RotatingUASession(Session):

sgraaf

import os
from concurrent.futures import Future, ThreadPoolExecutor
from functools import partial
from itertools import repeat
from typing import Any, Dict, List, Optional, Union

from requests import Response, Session


class ConcurrentSession(Session):

sgraaf

#!/bin/sh
set -e

LG=$1
WIKI_DUMP_NAME=${LG}wiki-latest-pages-articles.xml.bz2
WIKI_DUMP_DOWNLOAD_URL=https://dumps.wikimedia.org/${LG}wiki/latest/$WIKI_DUMP_NAME

# download latest Wikipedia dump in chosen language
echo "Downloading the latest $LG-language Wikipedia dump from $WIKI_DUMP_DOWNLOAD_URL..."
wget -c $WIKI_DUMP_DOWNLOAD_URL

sgraaf

sgraaf

import csv
from pathlib import Path
from typing import Optional, Union, Sequence, List, Literal

import pandas as pd
from tqdm import tqdm


def read_csv(
    file: Path,

sgraaf

#!/bin/sh
set -e

WIKI_DUMP_FILE_IN=$1
WIKI_DUMP_FILE_OUT=${WIKI_DUMP_FILE_IN%%.*}.txt

# clone the WikiExtractor repository
git clone https://github.com/attardi/wikiextractor.git

# extract and clean the chosen Wikipedia dump