高速公路 ETC 資料下載程式，請自行修改相關參數

taiwanetc.py

import tarfile
import os
import datetime
import csv

import urllib.error

from urllib import request
from glob import glob
from multiprocessing.dummy import Pool

# import mysql.connector

def main():
  s = datetime.datetime(2016,3,1)
  e = datetime.datetime(2016,5,2)

  pool= Pool(20)
  
  # pool.map(download_and_extract_one_file, get_list(s, e))
  pool.map(handle_csv, glob(os.path.join(folder, 'temp/M03A/*/*/*.csv')))
  
  pool.close()
  pool.join()

def download_and_extract_one_file(url):
  f = download(url)
  
  try:
    with tarfile.open(f, 'r:gz') as tar:
      tar.extractall('temp/')
      os.remove(f)
  except FileNotFoundError:
    pass
  print(f.split('/')[-1])

def handle_csv(_from):
  to=os.path.join(folder, 'temp/all.csv')

  with open(to, 'a', encoding='utf8') as output:
    with open(_from, 'r', encoding='utf8') as input:
      input_rows = input.read().split("\n")
      for row in input_rows:
        row = row.split(",")

        if row[0] != '':
          row[0] = str(datetime.datetime.strptime(row[0], "%Y-%m-%d %H:%M").timestamp())
          output.write(",".join(row) + "\n")

  print(_from)

def handle_sql(path):
  connect = mysql.connector.connect(user='root', database='etc')
  cursor = connect.cursor()

  with open(path, 'r', encoding='utf8') as f:
    for row in csv.reader(f):
      cursor.execute('insert into flow (datetime, gantry, direction, vehicletype, number) values ("{}", "{}", "{}", {}, {})'.format(
        row[0],
        row[1],
        row[2],
        int(row[3]),
        int(row[4])
      ))
  connect.commit()
  print(path)

  cursor.close()
  connect.close()

def get_list(start, end):
  step = datetime.timedelta(days=1)
  url_template = 'http://tisvcloud.freeway.gov.tw/history/TDCS/M03A/M03A_{}.tar.gz'

  while end > start:
    yield url_template.format(start.strftime('%Y%m%d'))
    start = start + step

def download(url):
  path = os.path.join(folder, 'temp/' + url.split("/")[-1])
  try:
    request.urlretrieve(url, path)
  except ConnectionResetError:
    print('Connect Error')
  except urllib.error.HTTPError:
    print('File Not Found: ' + path)

  return path

if __name__ == '__main__':
  folder = os.path.dirname(os.path.realpath(__file__))
  
  main()