retrieve slack log data

updatedb.py

#!/usr/bin/env python

import requests,json
import pymongo
import datetime,time

token = "xoxp-your-token"

def main():
    client = pymongo.MongoClient("localhost", 27017)
    db = client.hellocpp
    channels = get_channels(token)
    for ch in channels["channels"]:
        update_database(ch, db)
    print("End")

def update_database(ch,db):
    print(ch["name"])
    chid = ch["id"]
    if db[chid].find_one() == None:
        last_ts = None
    else:
        msg = db[chid].find().sort("ts",-1).limit(1)[0]
        last_ts = msg["ts"]
    for hist in get_history(token, ch, None, last_ts):
        print("{} get_hist ok, {}".format(chid,len(hist["messages"])))
        for m in hist["messages"]:
            db[chid].update_one(m, {"$setOnInsert": m}, upsert=True)
            print(m)
    return

def get_history(token, ch, latest, oldest):
    chid = ch["id"]
    url = "https://slack.com/api/channels.history"
    payload = {
        "token"  : token,
        "channel": chid,
        "count"  : 1000,
    }
    if latest!=None:
        payload["lastest"]=latest

    if oldest==None:
        payload["oldest"]=ch["created"]
    else:
        payload["oldest"]=oldest

    ret = requests.post(url, params=payload)
    if ret.status_code != requests.codes.ok:
        print("status code ng")
        return False
    data = ret.json()

    if data["has_more"]==True:
        print("{} has_more".format(chid))
        ts = sorted([ m["ts"] for m in data["messages"] ])
        last_ts = ts[-1]
        return [data] + get_history(token, ch, None, float(last_ts))
    else:
        print("{} no_more".format(chid))
        return [data]

def get_channels(token):
    url = "https://slack.com/api/channels.list"
    payload = { "token": token, }
    ret = requests.post(url, params=payload)
    if ret.status_code != requests.codes.ok:
        return False
    data = json.loads(ret.text)
    return data

if __name__=="__main__":
    main()