Created
June 21, 2018 07:40
-
-
Save sinanatra/29316f88b005eb4e1a7f28f6096c6fe0 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import urllib.request | |
| import scraperwiki | |
| import lxml.html | |
| import codecs | |
| import urllib | |
| import requests | |
| import urllib.request | |
| import datetime | |
| import os | |
| import time | |
| from bs4 import BeautifulSoup | |
| from urllib.request import urlopen | |
| from lxml.html import parse | |
| #modify this to have a starting boat | |
| first_id = '5114034' | |
| print("start") | |
| boatz = [] | |
| #never change this | |
| user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7' | |
| nearby = "https://www.marinetraffic.com/en/ais/index/ships/nearby/shipid:" | |
| url = nearby + first_id | |
| word = "shipid" | |
| headers={'User-Agent':user_agent,} | |
| request=urllib.request.Request(url,None,headers) | |
| response = urllib.request.urlopen(request) | |
| data = response.read() | |
| soup = BeautifulSoup(data) | |
| #find and then append all boats to an array | |
| for link in soup.find_all('td', href=False): | |
| b = str(link) | |
| try: | |
| a = b.split("mmsi:",1)[1].split('"',1)[0].split('/',1)[0] | |
| boatz.append(a) | |
| except: | |
| continue | |
| print("end") | |
| base_urlbase_ur = 'http://www.marinetraffic.com/en/ais/details/ships/' | |
| #modify this to add ships | |
| schoolIDs = boatz | |
| print ("start") | |
| # output file name, never remove the "a" or you will overwrite the tsv and not adding new lines | |
| output = open("output/"+"im-on-a-boat"+".tsv", "a") | |
| #loops through all the ships number and saves the .tsv file with coordinates | |
| for item in schoolIDs: | |
| try: | |
| next_link = base_url+item | |
| print(next_link) | |
| html = lxml.html.parse(next_link) | |
| #this is the structure of the elements in the html | |
| dates = [i.strip() for i in html.xpath("//div/strong/span/time/text()")] | |
| names = [i.strip() for i in html.xpath("//h1/text()")] | |
| location = [i.strip() for i in html.xpath("//a[contains(@class,'details_data_link')]/text()")] | |
| datas = [i.strip() for i in html.xpath("//div[contains(@class,'col-xs-6')]/div/b/text()")] | |
| weight = datas[6].replace("t","") + "" | |
| flag = datas[3].replace("t","") + "" | |
| boat_type = datas[4].replace("t","") + "" | |
| # mainly this is for future steps, it cleans the code, removes the \ and the ° symbols | |
| for date, loc,name, in zip(dates, location,names): | |
| geo = "" | |
| day = "" | |
| for i in loc: | |
| geo+= i.replace("°","").replace("/","\t") + "" | |
| for j in date: | |
| day+= j.replace(" ","\t").replace("-","\t") + "" | |
| print (day,geo,name,weight) | |
| output.write(name+"\t"+day + "\t" + geo +"\t"+str(weight)+"\t"+flag+"\t"+boat_type+"\n") | |
| except: | |
| continue | |
| output.close() | |
| print("end") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment