wilddog + ftp

readme.md

Prerequisites:

• Python 2 or 3
• The following packages:

pip install six python-firebase

How to use:

• Update the ftp information and delete_after_download
• Run online: https://colab.research.google.com/drive/1P3o0_YObRlHx5z3m0h0MLxO9bxBaGaeO
• To run the script once:

python /path/to/wilddog1.py

• To run every X minutes:

  • Use cron tasks: https://www.cyberciti.biz/faq/how-do-i-add-jobs-to-cron-under-linux-or-unix-oses/
  • Example (minutes after midnight, every day: https://crontab.guru/#5_0___* ):

  crontab -e
  5 0 * * * python /path/to/wilddog1.py
  

• To run forever:

  1. Change forever = False to forever = True
  2. Use nohup https://www.cyberciti.biz/tips/nohup-execute-commands-after-you-exit-from-a-shell-prompt.html
  • Example:

  nohup python /path/to/wilddog1.py >/dev/null 2>&1 &
  

  • To run it on startup

  crontab -e
  @reboot python /path/to/wilddog1.py
  

wilddog1.py

"""
Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1P3o0_YObRlHx5z3m0h0MLxO9bxBaGaeO

Dont forget to run the following command to install the required packages:
pip install six python-firebase
"""

# Fill the server information
ftp_server = 'ftp.cs.brown.edu'
ftp_user = ''
ftp_pass = ''

# The ftp upload folder
upload_folder = '/incoming'
# The local path to extract to
path = 'new'

# Wilddog Authentication
secret = ''
email = ''

# runs for ever?
forever = False  # False: run just once, True: runs forever and check the updates ever 10 minutes
run_every = 10 * 60  # 10 minutes

# Delete the record from wilddog after download
delete_after_download = False  # False: to keep, True: to delete

import os
import zipfile
from ftplib import FTP
from time import sleep

from firebase import firebase
from six.moves import urllib


def enter_dir(f, path):
    try:
        f.cwd(path)
    except:
        try:
            f.mkd(path)
            f.cwd(path)
        except:
            pass
    print (path)


if __name__ == '__main__':
    # connect
    authentication = None
    if email != '' and secret != '':
        authentication = authentication = firebase.FirebaseAuthentication(secret, email)
    wilddog = firebase.FirebaseApplication('https://site-eumt.wilddogio.com', authentication=authentication)

    # checking for updates and save to last_customs.txt
    # if there is an update check the listings
    last_customs = ''
    old_files = []
    try:
        with open('last_customs.txt', 'r') as file:
            last_customs = file.readline()
    except:
        pass

    while True:

        result = wilddog.get('/listings_xml_aux/aux_xml_send_semaphore_customs', None)

        with open('last_customs.txt', 'w') as file:
            file.write(result)

        if last_customs != result:
            last_customs = result
            result = wilddog.get('/listings_xml_customs_id', None)
            print(result)

            try:
                # make sure that the local path exist
                os.makedirs(path)
            except OSError:
                pass
            # keep track of the old files
            old_files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]

            # download the zip files and extract to path and delete the zip and listings_xml_customs_id from wilddog
            for key in result:
                value = result[key]
                print("Downloading:", value)
                zip_file = urllib.request.urlretrieve(value, os.path.join(path, value.split('/')[-1]))[0]
                try:
                    with zipfile.ZipFile(zip_file, "r") as zip_ref:
                        zip_ref.extractall(path)
                    os.remove(zip_file)
                    if delete_after_download:
                        wilddog.delete('/listings_xml_customs_id', key)
                except Exception as e:
                    print (e)

            # upload only the new files to the ftp server
            aftp = FTP(ftp_server, ftp_user, ftp_pass)
            aftp.login()
            enter_dir(aftp, upload_folder)

            all_files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
            new_files = set(all_files) - set(old_files)
            for file_name in new_files:
                with open(os.path.join(path, file_name), 'rb') as file:
                    print("uploading:", file_name)
                    aftp.storbinary('STOR %s' % file_name, file)

            aftp.quit()
        else:
            print('No updates')
        if forever is False:
            print ('DONE')
            break
        print('Sleeping for', run_every)
        sleep(run_every)