Skip to content

Instantly share code, notes, and snippets.

@sinanatra
Created June 21, 2018 07:40
Show Gist options
  • Select an option

  • Save sinanatra/29316f88b005eb4e1a7f28f6096c6fe0 to your computer and use it in GitHub Desktop.

Select an option

Save sinanatra/29316f88b005eb4e1a7f28f6096c6fe0 to your computer and use it in GitHub Desktop.
import urllib.request
import scraperwiki
import lxml.html
import codecs
import urllib
import requests
import urllib.request
import datetime
import os
import time
from bs4 import BeautifulSoup
from urllib.request import urlopen
from lxml.html import parse
#modify this to have a starting boat
first_id = '5114034'
print("start")
boatz = []
#never change this
user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
nearby = "https://www.marinetraffic.com/en/ais/index/ships/nearby/shipid:"
url = nearby + first_id
word = "shipid"
headers={'User-Agent':user_agent,}
request=urllib.request.Request(url,None,headers)
response = urllib.request.urlopen(request)
data = response.read()
soup = BeautifulSoup(data)
#find and then append all boats to an array
for link in soup.find_all('td', href=False):
b = str(link)
try:
a = b.split("mmsi:",1)[1].split('"',1)[0].split('/',1)[0]
boatz.append(a)
except:
continue
print("end")
base_urlbase_ur = 'http://www.marinetraffic.com/en/ais/details/ships/'
#modify this to add ships
schoolIDs = boatz
print ("start")
# output file name, never remove the "a" or you will overwrite the tsv and not adding new lines
output = open("output/"+"im-on-a-boat"+".tsv", "a")
#loops through all the ships number and saves the .tsv file with coordinates
for item in schoolIDs:
try:
next_link = base_url+item
print(next_link)
html = lxml.html.parse(next_link)
#this is the structure of the elements in the html
dates = [i.strip() for i in html.xpath("//div/strong/span/time/text()")]
names = [i.strip() for i in html.xpath("//h1/text()")]
location = [i.strip() for i in html.xpath("//a[contains(@class,'details_data_link')]/text()")]
datas = [i.strip() for i in html.xpath("//div[contains(@class,'col-xs-6')]/div/b/text()")]
weight = datas[6].replace("t","") + ""
flag = datas[3].replace("t","") + ""
boat_type = datas[4].replace("t","") + ""
# mainly this is for future steps, it cleans the code, removes the \ and the ° symbols
for date, loc,name, in zip(dates, location,names):
geo = ""
day = ""
for i in loc:
geo+= i.replace("°","").replace("/","\t") + ""
for j in date:
day+= j.replace(" ","\t").replace("-","\t") + ""
print (day,geo,name,weight)
output.write(name+"\t"+day + "\t" + geo +"\t"+str(weight)+"\t"+flag+"\t"+boat_type+"\n")
except:
continue
output.close()
print("end")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment