This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# coding: utf-8 | |
# In[12]: | |
import pymongo | |
from datetime import timedelta | |
# In[4]: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
### Keybase proof | |
I hereby claim: | |
* I am jplsightm on github. | |
* I am sightmjpl (https://keybase.io/sightmjpl) on keybase. | |
* I have a public key ASBIFMlk-Tq7bpp2utgHxTy8gqzu_igdfm242gCs-FwAdwo | |
To claim this, I am signing this object: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
sudo apt install -y ssh | |
sudo apt install -y zsh | |
sudo apt install -y byobu | |
sudo apt-get -y update | |
sudo apt-get install -y \ | |
apt-transport-https \ | |
ca-certificates \ | |
curl \ | |
software-properties-common | |
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
from tabulate import tabulate | |
def pandas_df_to_markdown_table(df): | |
# Dependent upon ipython | |
# shamelessly stolen from https://stackoverflow.com/questions/33181846/programmatically-convert-pandas-dataframe-to-markdown-table | |
from IPython.display import Markdown, display | |
fmt = ['---' for i in range(len(df.columns))] | |
df_fmt = pd.DataFrame([fmt], columns=df.columns) | |
df_formatted = pd.concat([df_fmt, df]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from datetime import datetime, timedelta | |
import pandas as pd | |
def get_fw(date, fiscal_start=datetime(1970, 1,1), calendar_day=False): | |
""" | |
Obtain fiscal week from a datetime object. | |
:fiscal_start: Indicate the start of a fiscal year | |
:calendar_day: If False the first full week is Week 1. | |
""" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def process_files(path, extention, func, *args, **kwargs): | |
""" | |
Take a directory of files and apply a function to those files. | |
The first parameter of the function (`func`) must be a file name. This is typically | |
the file to parse to df before apply some function | |
""" | |
dfs = {} | |
for fname in os.listdir(path): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
def step_back_ts(frame, ts_col, shift): | |
timestamps = pd.DataFrame(frame[ts_col].iloc[shift:], columns=['timestamp']) | |
timestamps.reset_index(inplace=True, drop=True) | |
for i in range(shift): | |
timestamps.loc[len(timestamps)+1, 'timestamp'] = np.nan | |
return timestamps |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
def sensor_csv(frame, sensor_name, sensor_column, prefix, keep_columns, timestamp): | |
""" | |
frame = input frame | |
sensor_name = sensor name to filter on | |
sensor_column = column that contains the sensor name | |
prefix = prefilx to add to column (make themn unique) | |
keep_columns = what columns should be kept .... I am not doing any checks on data types, make this a list | |
timestamp = timestamp column |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pymongo | |
# get mongo sslog ids and ORDERIDS | |
ORDERIDs = {log['_id']: log['data']['fieldvalues']['ORDERID']['value'] | |
for log in sslog.find({'data.fieldvalues.ORDERID.value': {'$exists': True}}, {'data.fieldvalues.ORDERID.value': 1})} | |
# do some parsing because there was all sorts of badness - floats cast as strings, integers, etc | |
def order_id_to_string(_id, orderid): | |
try: | |
orderid = str(int(float(orderid))) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def mongo_objs(conn_str, database, tests=[lambda x: x.document_count({})]): | |
client = pymongo.MongoClient(conn_str) | |
db = client[database] | |
sslog = db.sslog | |
cycle = db.cycle | |
sslog_results = {} | |
cycle_results = {} | |
for test in tests: | |
try: | |
sslog_results[test.__name__] = test(sslog) |
OlderNewer