dchaplinsky

# pip install pymorphy3
# pip install pymorphy3-dicts-uk

import pymorphy3
from collections import defaultdict
from itertools import product
from typing import List, List


morph = pymorphy3.MorphAnalyzer(lang="uk")

dchaplinsky

import os
import torch
import numpy as np
from sentence_transformers import SentenceTransformer

# Load pre-trained model for sentence embeddings
model = SentenceTransformer("paraphrase-multilingual-mpnet-base-v2")

# Set up LSTM model
input_size = 768  # Size of the sentence embeddings

dchaplinsky

import json
import argparse
from typing import Dict
from pathlib import Path

import smart_open
import ftfy
from tqdm import tqdm
import html2text
from datasets import load_dataset

dchaplinsky

import argparse
from flair.data import Sentence
from flair.embeddings import (
    DocumentEmbeddings,
    FlairEmbeddings,
    DocumentLMEmbeddings,
    DocumentPoolEmbeddings,
)
from torch import Tensor

dchaplinsky

#!/bin/bash

# You will need `apt get parallel pv` to make it run

# download file containing urls
curl http://webdatacommons.org/structureddata/2022-12/files/file.list > urls.txt

# create output file
touch output.txt

dchaplinsky

import bz2
import logging
import multiprocessing
import re
from pickle import PicklingError

# LXML isn't faster, so let's go with the built-in solution
from xml.etree.ElementTree import iterparse

dchaplinsky

import os.path
from flair.data import Dictionary
from flair.models import LanguageModel
from flair.trainers.language_model_trainer import LanguageModelTrainer, TextCorpus


def train_flair_embeddings(
    corpus_path="/data/ubertext/for_flair",
    dictionary_path="/home/dima/Projects/flair_embeddings/flair_dictionary.pkl",
    lm_file="./language_model_forward_no_amp_accum_grad_fixed",

dchaplinsky

from social_auth.backends.facebook import FacebookBackend
from social_auth.backends import google

def social_extra_values(sender, user, response, details, **kwargs):
    result = False
    
    if "id" in response:
        from apps.photo.models import Photo
        from urllib2 import urlopen, HTTPError
        from django.template.defaultfilters import slugify

dchaplinsky

import wn
import csv

from collections import Counter, defaultdict
from tqdm.notebook import tqdm


wn.download("pwn:3.1")
pwn = wn.Wordnet("pwn:3.1")

dchaplinsky

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
  <key>name</key>
	<string>Blackboard</string>
	<key>author</key>
	<string>Domenico Carbotta</string>
	<key>settings</key>
	<array>