icoxfog417

class EDINETGetDocumentsOperator(BaseOperator):

    @apply_defaults
    def __init__(self, filter_func=None, *args, **kwargs):
        self.filter_func = filter_func
        super().__init__(*args, **kwargs)

    def execute(self, context):
        self.log.info("Retreave list of documents from EDINET @ {}.".format(
            self.start_date.strftime("%Y/%m/%d")))

icoxfog417

def update(self, states, actions, rewards, values):
    # Calculate values (or advantage) at outside of update process.
    advantage = reward - values
    action_probs = self.actor(states)
    selected_action_probs = action_probs[self.to_one_hot(actions)]
    neg_logs = - log(selected_action_probs)
    policy_loss = reduce_mean(neg_logs * advantages)
 

icoxfog417

def update(self, states, actions, rewards):
    values = self.critic(states)
    advantage = reward - tf.stop_gradient(values)  # Prevent gradient flows to critic
    action_probs = self.actor(states)
    selected_action_probs = action_probs[self.to_one_hot(actions)]
    neg_logs = - log(selected_action_probs)
    policy_loss = reduce_mean(neg_logs * advantages)
 

icoxfog417

def update(self, states, actions, rewards):
    values = self.critic(states)
    advantage = reward - values
    action_probs = self.actor(states)
    selected_action_probs = action_probs[self.to_one_hot(actions)]
    neg_logs = - log(selected_action_probs)
    # If backprop executed, gradient of policy_loss will affect critic!
    policy_loss = reduce_mean(neg_logs * advantages)
 

icoxfog417

from datetime import datetime, timedelta
import time
import random
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from pprint import pprint


default_args = {
    "owner": "airflow",

icoxfog417

for batch in dp(train_data).preprocess().iterate(batch_size=32, epoch=10):
    model.train_on_batch(batch["review"], batch["polarity"])

icoxfog417

from chariot.dataset_preprocessor import DatasetPreprocessor
from chariot.transformer.formatter import Padding


dp = DatasetPreprocessor()
dp.process("review")\
    .by(ct.text.UnicodeNormalizer())\
    .by(ct.Tokenizer("en"))\
    .by(ct.token.StopwordFilter("en"))\
    .by(ct.Vocabulary(min_df=5, max_df=0.5))\

icoxfog417

import chariot.transformer as ct
from chariot.preprocessor import Preprocessor


preprocessor = Preprocessor()
preprocessor\
    .stack(ct.text.UnicodeNormalizer())\
    .stack(ct.Tokenizer("en"))\
    .stack(ct.token.StopwordFilter("en"))\
    .stack(ct.Vocabulary(min_df=5, max_df=0.5))\

icoxfog417

import os
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from chariot.storage import Storage


class SimilarityGraph():

    def __init__(self, vocabulary, nearest_neighbor=4, mode="connectivity",
                 representation="GloVe.6B.200d", root=""):

icoxfog417

import numpy as np
import spacy


class DependencyGraph():

    def __init__(self, lang, vocabulary):
        self.lang = lang
        self._parser = spacy.load(lang, disable=["ner", "textcat"])
        self.vocabulary = vocabulary