Skip to content

Instantly share code, notes, and snippets.

@deknos
Created June 23, 2017 20:06
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save deknos/e4563da7ef22004539ed10736fa6233a to your computer and use it in GitHub Desktop.
Save deknos/e4563da7ef22004539ed10736fa6233a to your computer and use it in GitHub Desktop.
#!/usr/bin/python -Es
import sys
import os
import os.path
import json
import json_delta
import urllib2
import time
lastversionfile='last_version'
downloadurlprefix='https://multimobil-core.mvg.de/service/v5/networkState/networkState?MVG_RAD='
startnumber=1
curnumber=-1
wartezeit=1
if os.path.exists(lastversionfile):
# Ueberpruefe ob es schon einen stamp gibt, wo etwas heruntergeladen wurde
fh=open(lastversionfile)
line=fh.readline().strip('\n') # only one number
if line.isdigit():
curnumber=line
fh.close()
def getnewjsondata(url):
# laedt daten einer url und erzeugt ein json objekt
try:
response=urllib2.urlopen(url)
ujd = response.read()
return json.loads(ujd)
except :
print "HTTP connection had a failure"
return None
# Hole entweder die daten seit dem letzten stamp oder seit dem anfang
if curnumber != -1 :
importedjson = getnewjsondata(downloadurlprefix+curnumber)
else:
importedjson = getnewjsondata(downloadurlprefix+str(startnumber))
curnumber=startnumber
if importedjson == None:
print "First fetch failed, stop"
sys.exit(1)
def getnumberfromjsonstructure(ij):
# laedt stamp aus jsonobjekt
return ij['networkStateVersions']['MVG_RAD']
def writedata(number,jsonstruct):
fh=open(str(number), 'w')
fh.writelines(json.dumps(jsonstruct))
fh.close()
def savedataifdifferent(versionfile, newdata, curnum):
# hole stamp
newnum = getnumberfromjsonstructure(newdata)
# speichere falls anders
if newnum != curnum :
writedata(newnum,newdata)
writedata(versionfile,newnum)
return True, newnum
else:
return False, newnum
# sichert alte nummer und json objekt
lastnumber=curnumber
jsonprev=importedjson
different, curnumber = savedataifdifferent(lastversionfile,importedjson,curnumber)
# Prolog ist fertig, initiales json objekt ist fertig
# jetzt kann versucht werden, weitere daten zu laden und ein delta zu erzeugen
while 1:
time.sleep(wartezeit)
importedjson = getnewjsondata(downloadurlprefix+str(curnumber))
if importedjson == None:
continue
different, newnum = savedataifdifferent(lastversionfile,importedjson, curnumber)
if different == True:
diff = json_delta.diff(jsonprev,importedjson)
writedata(str(curnumber)+"-"+str(newnum),diff)
jsonprev=importedjson
curnumber=newnum
print "New Stamp:"+str(curnumber)+" generated also a diff"
else:
print "No new data was generated"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment