Skip to content

Instantly share code, notes, and snippets.

@jplsightm
jplsightm / update_ts.py
Created May 15, 2017 15:48
Updating timestamps in the site machine platform
# coding: utf-8
# In[12]:
import pymongo
from datetime import timedelta
# In[4]:
### Keybase proof
I hereby claim:
* I am jplsightm on github.
* I am sightmjpl (https://keybase.io/sightmjpl) on keybase.
* I have a public key ASBIFMlk-Tq7bpp2utgHxTy8gqzu_igdfm242gCs-FwAdwo
To claim this, I am signing this object:
@jplsightm
jplsightm / docker_setup_ubuntu.sh
Created April 25, 2018 16:55
Ubuntu - Quick docker setup
sudo apt install -y ssh
sudo apt install -y zsh
sudo apt install -y byobu
sudo apt-get -y update
sudo apt-get install -y \
apt-transport-https \
ca-certificates \
curl \
software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
@jplsightm
jplsightm / df_to_markdown.py
Last active January 31, 2023 07:32
Convert a Pandas Dataframe to Markdown
import pandas as pd
from tabulate import tabulate
def pandas_df_to_markdown_table(df):
# Dependent upon ipython
# shamelessly stolen from https://stackoverflow.com/questions/33181846/programmatically-convert-pandas-dataframe-to-markdown-table
from IPython.display import Markdown, display
fmt = ['---' for i in range(len(df.columns))]
df_fmt = pd.DataFrame([fmt], columns=df.columns)
df_formatted = pd.concat([df_fmt, df])
@jplsightm
jplsightm / fiscal_week.py
Last active June 5, 2018 14:30
Early Attempt at identifying fiscal week
from datetime import datetime, timedelta
import pandas as pd
def get_fw(date, fiscal_start=datetime(1970, 1,1), calendar_day=False):
"""
Obtain fiscal week from a datetime object.
:fiscal_start: Indicate the start of a fiscal year
:calendar_day: If False the first full week is Week 1.
"""
@jplsightm
jplsightm / csvs_to_dfs.py
Created June 5, 2018 13:32
Take a directory of files and apply a function to those files.
def process_files(path, extention, func, *args, **kwargs):
"""
Take a directory of files and apply a function to those files.
The first parameter of the function (`func`) must be a file name. This is typically
the file to parse to df before apply some function
"""
dfs = {}
for fname in os.listdir(path):
@jplsightm
jplsightm / step_back_ts.py
Created September 26, 2018 19:13
Sometimes you just need to shift items back in a dataframe (typically with times series data). This is a hackish way to do that :) Enjoy
import pandas as pd
def step_back_ts(frame, ts_col, shift):
timestamps = pd.DataFrame(frame[ts_col].iloc[shift:], columns=['timestamp'])
timestamps.reset_index(inplace=True, drop=True)
for i in range(shift):
timestamps.loc[len(timestamps)+1, 'timestamp'] = np.nan
return timestamps
@jplsightm
jplsightm / process_tall_records.py
Created July 2, 2019 04:07
For processing tall records. This is a non tested function. Use at your own risk.
import pandas as pd
def sensor_csv(frame, sensor_name, sensor_column, prefix, keep_columns, timestamp):
"""
frame = input frame
sensor_name = sensor name to filter on
sensor_column = column that contains the sensor name
prefix = prefilx to add to column (make themn unique)
keep_columns = what columns should be kept .... I am not doing any checks on data types, make this a list
timestamp = timestamp column
import pymongo
# get mongo sslog ids and ORDERIDS
ORDERIDs = {log['_id']: log['data']['fieldvalues']['ORDERID']['value']
for log in sslog.find({'data.fieldvalues.ORDERID.value': {'$exists': True}}, {'data.fieldvalues.ORDERID.value': 1})}
# do some parsing because there was all sorts of badness - floats cast as strings, integers, etc
def order_id_to_string(_id, orderid):
try:
orderid = str(int(float(orderid)))
def mongo_objs(conn_str, database, tests=[lambda x: x.document_count({})]):
client = pymongo.MongoClient(conn_str)
db = client[database]
sslog = db.sslog
cycle = db.cycle
sslog_results = {}
cycle_results = {}
for test in tests:
try:
sslog_results[test.__name__] = test(sslog)