conormm

R to python data wrangling snippets

The dplyr package in R makes data wrangling significantly easier. The beauty of dplyr is that, by design, the options available are limited. Specifically, a set of key verbs form the core of the package. Using these verbs you can solve a wide range of data problems effectively in a shorter timeframe. Whilse transitioning to Python I have greatly missed the ease with which I can think through and solve problems using dplyr in R. The purpose of this document is to demonstrate how to execute the key dplyr verbs when manipulating data using Python (with the pandas package).

dplyr is organised around six key verbs:

conormm

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import begin

l1_nodes = 200
l2_nodes = 100
final_layer_nodes = 10

# define placeholder for data
# also considered as the "visibale layer, the layer that we see"

conormm

def extract_tokens_plus_meta(doc:spacy.tokens.doc.Doc):
    """Extract tokens and metadata from individual spaCy doc."""
    return [
        (i.text, i.i, i.lemma_, i.ent_type_, i.tag_, 
         i.dep_, i.pos_, i.is_stop, i.is_alpha, 
         i.is_digit, i.is_punct) for i in doc
    ]

conormm

import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.functional as F
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import SubsetRandomSampler

import numpy as np
from sklearn.datasets import load_boston
from sklearn.preprocessing import StandardScaler

conormm

# pytorch embeddings

import torch
from torch.optim import Adam
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F

import pandas as pd
import numpy as np

conormm

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.utils import np_utils
import numpy as np

l1_nodes = 200
l2_nodes = 100
final_layer_nodes = 10

conormm

import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestRegressor
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.datasets import load_boston
sns.set_style("whitegrid")

X, y = load_boston(return_X_y=True)

conormm

(tidy_docs
 .groupby("doc_id")
 .apply(lambda x: x.assign(
     prev_token = lambda x: x.token.shift(1), 
     next_token = lambda x: x.token.shift(-1))
       )
 .reset_index(drop=True)
 .query("tag == 'POS'")
 .loc[:, ["doc_id", "prev_token", "token", "next_token"]]
)

conormm

def tidy_tokens(docs):
    """Extract tokens and metadata from list of spaCy docs."""
    
    cols = [
        "doc_id", "token", "token_order", "lemma", 
        "ent_type", "tag", "dep", "pos", "is_stop", 
        "is_alpha", "is_digit", "is_punct"
    ]
    
    meta_df = []

conormm

def calcualte_exceedance_probability(exceed_values, posterior):
    n = posterior.shape[1]
    ix = []
    probs_m = []
    probs_05 = []
    probs_95 = []
    for i in exceed_values:
        p = ((posterior>i).sum(1)/n)
        p05 = np.quantile(p, 0.01)
        p95 = np.quantile(p, 0.99)