ceteri

import pathlib
import sys

from icecream import ic
import requests

term = "weather"
words = set([])

ceteri

$ ./bin/spark-shell
14/04/18 15:23:49 INFO spark.HttpServer: Starting HTTP Server
14/04/18 15:23:49 INFO server.Server: jetty-7.x.y-SNAPSHOT
14/04/18 15:23:49 INFO server.AbstractConnector: Started SocketConnector@0.0.0.0:49861
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 0.9.1
      /_/

ceteri

import spacy
import pytextrank

# example text
text = "Compatibility of systems of linear constraints over the set of natural numbers. Criteria of compatibility of a system of linear Diophantine equations, strict inequations, and nonstrict inequations are considered. Upper bounds for components of a minimal set of solutions and algorithms of construction of minimal generating sets of solutions for all types of systems are given. These criteria and the corresponding algorithms for constructing a minimal supporting set of solutions can be used in solving all the considered types systems and systems of mixed types."

# load a spaCy model, depending on language, scale, etc.
nlp = spacy.load("de_core_news_sm")

# add PyTextRank to the spaCy pipeline

ceteri

import org.apache.spark.graphx._
import org.apache.spark.rdd.RDD

case class Peep(name: String, age: Int)

val vertexArray = Array(
  (1L, Peep("Kim", 23)),
  (2L, Peep("Pat", 31)),
  (3L, Peep("Chris", 52)),
  (4L, Peep("Kelly", 39)),

ceteri

wget https://www2.census.gov/geo/tiger/GENZ2018/shp/cb_2018_us_state_20m.zip
mkdir maps
cd maps
unzip ../cb_2018_us_state_20m.zip

ceteri

  1 reward   0.00/  0.02/  1.00 len   7.83 saved tmp/ppo/froz/checkpoint_1/checkpoint-1
  2 reward   0.00/  0.02/  1.00 len   7.40 saved tmp/ppo/froz/checkpoint_2/checkpoint-2
  3 reward   0.00/  0.02/  1.00 len   7.21 saved tmp/ppo/froz/checkpoint_3/checkpoint-3
  4 reward   0.00/  0.03/  1.00 len   7.36 saved tmp/ppo/froz/checkpoint_4/checkpoint-4
  5 reward   0.00/  0.03/  1.00 len   7.26 saved tmp/ppo/froz/checkpoint_5/checkpoint-5
  6 reward   0.00/  0.05/  1.00 len   7.57 saved tmp/ppo/froz/checkpoint_6/checkpoint-6
  7 reward   0.00/  0.05/  1.00 len   7.82 saved tmp/ppo/froz/checkpoint_7/checkpoint-7
  8 reward   0.00/  0.07/  1.00 len   7.42 saved tmp/ppo/froz/checkpoint_8/checkpoint-8
  9 reward   0.00/  0.07/  1.00 len   7.87 saved tmp/ppo/froz/checkpoint_9/checkpoint-9
 10 reward   0.00/  0.09/  1.00 len   8.84 saved tmp/ppo/froz/checkpoint_10/checkpoint-10

ceteri

  1 reward -902.00/-751.75/-345.00 len 194.80 saved tmp/ppo/taxi/checkpoint_1/checkpoint-1
  2 reward -902.00/-751.85/-345.00 len 193.70 saved tmp/ppo/taxi/checkpoint_2/checkpoint-2
  3 reward -902.00/-725.72/-340.00 len 193.00 saved tmp/ppo/taxi/checkpoint_3/checkpoint-3
  4 reward -902.00/-705.04/-151.00 len 192.59 saved tmp/ppo/taxi/checkpoint_4/checkpoint-4
  5 reward -902.00/-682.85/-151.00 len 192.62 saved tmp/ppo/taxi/checkpoint_5/checkpoint-5
  6 reward -902.00/-643.69/-128.00 len 190.27 saved tmp/ppo/taxi/checkpoint_6/checkpoint-6
  7 reward -902.00/-585.58/-78.00 len 185.95 saved tmp/ppo/taxi/checkpoint_7/checkpoint-7
  8 reward -794.00/-524.43/-21.00 len 176.76 saved tmp/ppo/taxi/checkpoint_8/checkpoint-8
  9 reward -794.00/-482.32/-21.00 len 172.15 saved tmp/ppo/taxi/checkpoint_9/checkpoint-9
 10 reward -713.00/-443.42/-21.00 len 166.61 saved tmp/ppo/taxi/checkpoint_10/checkpoint-10

ceteri

  1 reward -200.00/-200.00/-200.00 len 200.00 saved tmp/ppo/moun/checkpoint_1/checkpoint-1
  2 reward -200.00/-200.00/-200.00 len 200.00 saved tmp/ppo/moun/checkpoint_2/checkpoint-2
  3 reward -200.00/-200.00/-200.00 len 200.00 saved tmp/ppo/moun/checkpoint_3/checkpoint-3
  4 reward -200.00/-200.00/-200.00 len 200.00 saved tmp/ppo/moun/checkpoint_4/checkpoint-4
  5 reward -200.00/-200.00/-200.00 len 200.00 saved tmp/ppo/moun/checkpoint_5/checkpoint-5
  6 reward -200.00/-200.00/-200.00 len 200.00 saved tmp/ppo/moun/checkpoint_6/checkpoint-6
  7 reward -200.00/-200.00/-200.00 len 200.00 saved tmp/ppo/moun/checkpoint_7/checkpoint-7
  8 reward -200.00/-200.00/-200.00 len 200.00 saved tmp/ppo/moun/checkpoint_8/checkpoint-8
  9 reward -200.00/-200.00/-200.00 len 200.00 saved tmp/ppo/moun/checkpoint_9/checkpoint-9

ceteri

  1 reward   9.00/ 22.65/ 63.00 len  22.65 saved tmp/ppo/cart/checkpoint_1/checkpoint-1
  2 reward  12.00/ 42.72/151.00 len  42.72 saved tmp/ppo/cart/checkpoint_2/checkpoint-2
  3 reward  12.00/ 68.17/322.00 len  68.17 saved tmp/ppo/cart/checkpoint_3/checkpoint-3
  4 reward  13.00/ 97.87/408.00 len  97.87 saved tmp/ppo/cart/checkpoint_4/checkpoint-4
  5 reward  13.00/131.53/500.00 len 131.53 saved tmp/ppo/cart/checkpoint_5/checkpoint-5
  6 reward  13.00/165.24/500.00 len 165.24 saved tmp/ppo/cart/checkpoint_6/checkpoint-6
  7 reward  13.00/202.48/500.00 len 202.48 saved tmp/ppo/cart/checkpoint_7/checkpoint-7
  8 reward  22.00/233.83/500.00 len 233.83 saved tmp/ppo/cart/checkpoint_8/checkpoint-8
  9 reward  22.00/271.82/500.00 len 271.82 saved tmp/ppo/cart/checkpoint_9/checkpoint-9
 10 reward  22.00/302.99/500.00 len 302.99 saved tmp/ppo/cart/checkpoint_10/checkpoint-10

ceteri

CHECKPOINT_ROOT = "tmp/ppo/froz"
shutil.rmtree(CHECKPOINT_ROOT, ignore_errors=True, onerror=None)

ray_results = os.getenv("HOME") + "/ray_results/"
shutil.rmtree(ray_results, ignore_errors=True, onerror=None)