yamyamyuo

import json
import datetime

'''Create an encoder subclassing JSON.encoder. 
Make this encoder aware of our classes (e.g. datetime.datetime objects) 
'''
class Encoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime):
            return obj.isoformat()

yamyamyuo

from argparse import ArgumentParser

parser = ArgumentParser()
parser.add_argument(
"--verbose",
help="turn on debug mode",
default=False,
action='store_true')
parser.add_argument(
"--process-lock-file", help="process file lock", required=True)

yamyamyuo

#!/bin/bash

for i in `seq 1 355`;
do
    data_date=`date -v -${i}d +%F`
    echo $data_date
done

yamyamyuo

# 找到某个bucket 下, 限定prefix下, 所有的key
import os
import boto3
bucket_name = 'tmp-bucket'

_s3 = boto3.resource('s3',region_name='cn-north-1')
print _s3
bucket = _s3.Bucket(bucket_name)
print bucket.name
all_objects = _s3.meta.client.list_objects(Bucket = bucket_name)

yamyamyuo

import os
from itertools import groupby


input2 = [
{'dept': '001', 'sku': 'foo', 'transId': 'uniqueId1', 'qty': 100},
{'dept': '001', 'sku': 'bar', 'transId': 'uniqueId2', 'qty': 200},
{'dept': '001', 'sku': 'foo', 'transId': 'uniqueId3', 'qty': 300},
{'dept': '002', 'sku': 'baz', 'transId': 'uniqueId4', 'qty': 400},
{'dept': '002', 'sku': 'baz', 'transId': 'uniqueId5', 'qty': 500},

yamyamyuo

# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

if sys.version_info >= (3, 0):
    from urllib.parse import urlparse
else:
    from urlparse import urlparse

yamyamyuo

import logging

logging.basicConfig(
level=logging.INFO,
format=
'%(asctime)s pid:%(process)d %(levelname)s (%(filename)s:%(lineno)d)- %(message)s'
)

yamyamyuo

"""
sort by column in multi level dataframes

"""
import pandas

data = {'active_days': [2,6,1, 2],
        'province': ['beijing', 'shanghai', 'shanghai','beijing'],
        '__timestamp': ['2018-09-17', '2018-09-24', '2018-08-20', '2018-09-17'],
        'sum(active_days_user)': [1033604, 256682, 3613940, 1]}

yamyamyuo

#!/bin/bash

docker rm $(docker ps -a -q)
docker rmi $(docker images -q)
docker rmi $(docker images --filter "dangling=true" -q --no-trunc)
docker images | grep "superset" | awk '{print $3}' | xargs docker rmi

yamyamyuo

prestodb/presto#2624