davidmcclure

davidmcclure

import click

import pandas as pd

from pyspark.sql.functions import udf
from pyspark.sql import types as T

from geovec_data.utils import get_spark

davidmcclure

davidmcclure

{
  "$schema": "https://vega.github.io/schema/vega-lite/v2.4.3.json",
  "config": {"view": {"height": 300, "width": 400}},
  "data": {
    "values": [
      {
        "count": 216,
        "surname": "Strunk",
        "title": "The Elements of Style",
        "x": -2.187297821044922,

davidmcclure

{
  "corpus": "gale",
  "identifier": "XXX",
  "title": "XXX",
  "author_first": "XXX",
  "author_last": "XXX",
  "pub_year": 1900,
  "token": ["The", "raw", "tokens"],
  "tag": ["DT", "JJ", "NNS"]
}

davidmcclure

import torch
import csv

from torch import nn
from torch.nn import functional as F
from torch.autograd import Variable

from cached_property import cached_property

davidmcclure

https://docs.google.com/presentation/d/1su9sDVebcutqEqlpjP2prVUdJfsKhl5l8UClVC5_ZMA/edit?usp=sharing

davidmcclure

https://docs.google.com/presentation/d/1td9rxvUiMXfajR34iev8GoiDy9f2DDAaepM1fUxeKHw/edit?usp=sharing

davidmcclure

CREATE TABLE book_id (
	book_id INTEGER NOT NULL, 
	PRIMARY KEY (book_id)
);
CREATE TABLE chapter (
	scraped_at DATETIME NOT NULL, 
	book_id INTEGER NOT NULL, 
	chapter_number INTEGER NOT NULL, 
	chapter VARCHAR NOT NULL, 
	PRIMARY KEY (book_id, chapter_number), 

davidmcclure

+-----+--------+
|state|   count|
+-----+--------+
|   CA|34308508|
|   TX|21677337|
|   NY|16945315|
|   FL|13532303|
|   IL| 9117109|
|   GA| 7981133|
|   PA| 7367569|