Skip to content

Instantly share code, notes, and snippets.

@ahmedengu
Last active January 15, 2019 23:57
Show Gist options
  • Save ahmedengu/bc05d20a225897f29b6a1d5a48432be2 to your computer and use it in GitHub Desktop.
Save ahmedengu/bc05d20a225897f29b6a1d5a48432be2 to your computer and use it in GitHub Desktop.
wilddog + ftp

Prerequisites:

  • Python 2 or 3
  • The following packages:
pip install six python-firebase

How to use:

python /path/to/wilddog1.py
"""
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1P3o0_YObRlHx5z3m0h0MLxO9bxBaGaeO
Dont forget to run the following command to install the required packages:
pip install six python-firebase
"""
# Fill the server information
ftp_server = 'ftp.cs.brown.edu'
ftp_user = ''
ftp_pass = ''
# The ftp upload folder
upload_folder = '/incoming'
# The local path to extract to
path = 'new'
# Wilddog Authentication
secret = ''
email = ''
# runs for ever?
forever = False # False: run just once, True: runs forever and check the updates ever 10 minutes
run_every = 10 * 60 # 10 minutes
# Delete the record from wilddog after download
delete_after_download = False # False: to keep, True: to delete
import os
import zipfile
from ftplib import FTP
from time import sleep
from firebase import firebase
from six.moves import urllib
def enter_dir(f, path):
try:
f.cwd(path)
except:
try:
f.mkd(path)
f.cwd(path)
except:
pass
print (path)
if __name__ == '__main__':
# connect
authentication = None
if email != '' and secret != '':
authentication = authentication = firebase.FirebaseAuthentication(secret, email)
wilddog = firebase.FirebaseApplication('https://site-eumt.wilddogio.com', authentication=authentication)
# checking for updates and save to last_customs.txt
# if there is an update check the listings
last_customs = ''
old_files = []
try:
with open('last_customs.txt', 'r') as file:
last_customs = file.readline()
except:
pass
while True:
result = wilddog.get('/listings_xml_aux/aux_xml_send_semaphore_customs', None)
with open('last_customs.txt', 'w') as file:
file.write(result)
if last_customs != result:
last_customs = result
result = wilddog.get('/listings_xml_customs_id', None)
print(result)
try:
# make sure that the local path exist
os.makedirs(path)
except OSError:
pass
# keep track of the old files
old_files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
# download the zip files and extract to path and delete the zip and listings_xml_customs_id from wilddog
for key in result:
value = result[key]
print("Downloading:", value)
zip_file = urllib.request.urlretrieve(value, os.path.join(path, value.split('/')[-1]))[0]
try:
with zipfile.ZipFile(zip_file, "r") as zip_ref:
zip_ref.extractall(path)
os.remove(zip_file)
if delete_after_download:
wilddog.delete('/listings_xml_customs_id', key)
except Exception as e:
print (e)
# upload only the new files to the ftp server
aftp = FTP(ftp_server, ftp_user, ftp_pass)
aftp.login()
enter_dir(aftp, upload_folder)
all_files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
new_files = set(all_files) - set(old_files)
for file_name in new_files:
with open(os.path.join(path, file_name), 'rb') as file:
print("uploading:", file_name)
aftp.storbinary('STOR %s' % file_name, file)
aftp.quit()
else:
print('No updates')
if forever is False:
print ('DONE')
break
print('Sleeping for', run_every)
sleep(run_every)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment