Skip to content

Instantly share code, notes, and snippets.

@nickjevershed

nickjevershed/expenses-pdf-scraper.py

Created July 23, 2014 09:30
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 2 You must be signed in to fork a gist
  • Save nickjevershed/1a41b52b410b197e9cbb to your computer and use it in GitHub Desktop.
Save nickjevershed/1a41b52b410b197e9cbb to your computer and use it in GitHub Desktop.
Download ZIP
A scraper for getting politicians' travel expenses from PDF
Raw
expenses-pdf-scraper.py
#!/usr/bin/env python
import scraperwiki
import urllib2
import lxml.etree
urls = ["http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ABBOTT_Tony.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ALBANESE_Anthony.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ALEXANDER_John.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BALDWIN_Bob.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BIRD_Sharon.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BISHOP_Bronwyn.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BOWEN_Chris.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BRADBURY_David.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BURKE_Tony.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CAMERON_Doug.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CARR_Bob.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CLARE_Jason.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CLAYDON_Sharon.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_COBB_John.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_COLEMAN_David.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_COMBET_Greg.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CONROY_Pat.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_COULTON_Mark.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_DASTYARI_Sam.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ELLIOT_Justine.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FAULKNER_John.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FERGUSON_Laurie.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FIERRAVANTI-WELLS_Concetta.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FITZGIBBON_Joel.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FLETCHER_Paul.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GARRETT_Peter.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GASH_Joanna.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GILLESPIE_David.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GRIERSON_Sharon.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HALL_Jill.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HARTSUYKER_Luke.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HAWKE_Alex.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HAYES_Chris.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HEFFERNAN_Bill.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HENDY_Peter.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HOCKEY_Joe.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HOGAN_Kevin.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HUSIC_Ed.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_JONES_Stephen.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_JOYCE_Barnaby.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_KELLY_Craig.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_KELLY_Mike.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LAUNDY_Craig.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LEY_Sussan.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MARKUS_Louise.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MATHESON_Russell.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MCCLELLAND_Robert.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MCCORMACK_Michael.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MCNAMARA_Karen.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MELHAM_Daryl.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MORRISON_Scott.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MURPHY_John.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_NASH_Fiona.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_OAKESHOTT_Robert.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_O%27NEILL_Deborah.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_O%27NEILL_Deborah.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_OWENS_Julie.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PAYNE_Marise.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PLIBERSEK_Tanya.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RHIANNON_Lee.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ROWLAND_Michelle.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RUDDOCK_Philip.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SAFFIN_Janelle.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SCHULTZ_Alby.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SCOTT_Fiona.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SINODINOS_Arthur.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_STEPHENS_Ursula.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SUDMALIS_Ann.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_TAYLOR_Angus.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_THISTLETHWAITE_Matt.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_THISTLETHWAITE_Matt.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_THOMSON_Craig.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_TURNBULL_Malcolm.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_VARVARIS_Nickolas.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WICKS_Lucy.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WILLIAMS_John.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WINDSOR_Tony.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ANDREWS_Kevin.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BANDT_Adam.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BILLSON_Bruce.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BROAD_Andrew.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BROADBENT_Russell.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BURKE_Anna.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BYRNE_Anthony.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CARR_Kim.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CHEESEMAN_Darren.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CHESTER_Darren.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CHESTERS_Lisa.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_COLLINS_Jacinta.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CONROY_Stephen.pdf","http://www.finance.gov.au/sites/default/files/P33_CREAN_Simon_0.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_DANBY_Michael.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_DI_NATALE_Richard.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_DREYFUS_Mark.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FEENEY_David.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FEENEY_David.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FERGUSON_Martin.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FIFIELD_Mitch.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FORREST_John.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FRYDENBERG_Josh.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GIBBONS_Steve.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GILES_Andrew.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GILLARD_Julia.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GRIFFIN_Alan.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HENDERSON_Sarah.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HUNT_Greg.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_JENKINS_Harry.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_KING_Catherine.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_KROGER_Helen.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MACKLIN_Jenny.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MADIGAN_John.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MARLES_Richard.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MARSHALL_Gavin.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MCGOWAN_Cathy.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MCKENZIE_Bridget.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MIRABELLA_Sophie.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MITCHELL_Rob.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_O%27CONNOR_Brendan.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_O%27DWYER_Kelly.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_O%27NEIL_Clare.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ROBB_Andrew.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RONALDSON_Michael.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ROXON_Nicola.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RYAN_Joanne.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RYAN_Scott.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SHORTEN_Bill.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SMITH_Tony.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SMYTH_Laura.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_STONE_Sharman.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SUKKAR_Michael.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SYMON_Mike.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_TEHAN_Dan.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_THOMSON_Kelvin.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_TILLEM_Mehmet.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_TUDGE_Alan.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_VAMVAKINOU_Maria.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WATTS_Tim.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WOOD_Jason.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ANDREWS_Karen.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BOSWELL_Ronald.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BOYCE_Sue.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BRANDIS_George.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BROUGH_Mal.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BUCHHOLZ_Scott.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CHALMERS_Jim.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CHRISTENSEN_George.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CIOBO_Steven.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_D%27ATH_Yvette.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_DUTTON_Peter.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_EMERSON_Craig.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ENTSCH_Warren.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FURNER_Mark.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GAMBARO_Teresa.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HOGG_John.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HOWARTH_Luke.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_JONES_Ewen.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_JOYCE_Barnaby.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_KATTER_Bob.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LAMING_Andrew.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LANDRY_Michelle.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LIVERMORE_Kirsten.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LUDWIG_Joe.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MACDONALD_Ian.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MACFARLANE_Ian.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MASON_Brett.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MCLUCAS_Jan.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MOORE_Claire.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_NEUMANN_Shayne.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_NEVILLE_Paul.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_O%27DOWD_Ken.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PALMER_Clive.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PERRETT_Graham.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PITT_Keith.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PRENTICE_Jane.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RIPOLL_Bernie.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ROBERT_Stuart.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ROY_Wyatt.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RUDD_Kevin.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SCOTT_Bruce.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SLIPPER_Peter.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SOMLYAY_Alexander.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SWAN_Wayne.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_TRUSS_Warren.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_VAN_MANEN_Bert.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_VASTA_Ross.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WATERS_Larissa.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BACK_Chris.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BISHOP_Julie.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BISHOP_Mark.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CASH_Michaelia.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CORMANN_Mathias.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CROOK_Tony.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_EGGLESTON_Alan.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_EVANS_Chris.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GOODENOUGH_Ian.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GRAY_Gary.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HAASE_Barry.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_IRONS_Steve.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_JENSEN_Dennis.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_JOHNSTON_David.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_KEENAN_Michael.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LINES_Sue.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LUDLAM_Scott.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MACTIERNAN_Alannah.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MARINO_Nola.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MOYLAN_Judi.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PARKE_Melissa.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PORTER_Christian.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PRATT_Louise.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PRICE_Melissa.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RANDALL_Don.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SIEWERT_Rachel.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SIMPKINS_Luke.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SMITH_Dean.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SMITH_Stephen.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_STERLE_Glenn.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WASHER_Mal.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WILSON_Rick.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WYATT_Ken.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BERNARDI_Cory.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BIRMINGHAM_Simon.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BRIGGS_Jamie.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BUTLER_Mark.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CHAMPION_Nick.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_EDWARDS_Sean.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ELLIS_Kate.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FARRELL_Don.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_FAWCETT_David.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GALLACHER_Alex.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GEORGANAS_Steve.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HANSON-YOUNG_Sarah.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MCEWEN_Anne.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PASIN_Tony.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PYNE_Christopher.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RAMSEY_Rowan.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RISHWORTH_Amanda.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_RUSTON_Anne.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SECKER_Patrick.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SOUTHCOTT_Andrew.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WILLIAMS_Matt.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WONG_Penny.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WORTLEY_Dana.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WRIGHT_Penny.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_XENOPHON_Nick.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ZAPPIA_Tony.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ABETZ_Eric.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_ADAMS_Dick.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BILYK_Catryna.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BROWN_Carol.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BUSHBY_David.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_COLBECK_Richard.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_COLLINS_Julie.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HUTCHINSON_Eric.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LYONS_Geoff.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_MILNE_Christine.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_NIKOLIC_Andrew.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PARRY_Stephen.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_POLLEY_Helen.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SIDEBOTTOM_Sid.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SINGH_Lisa.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_THORP_Lin.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_URQUHART_Anne.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WHISH-WILSON_Peter.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WHITELEY_Brett.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_WILKIE_Andrew.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_CROSSIN_Trish.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_GRIGGS_Natasha.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_PERIS_Nova.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SCULLION_Nigel.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SNOWDON_Warren.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_BRODTMANN_Gai.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_HUMPHRIES_Gary.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LEIGH_Andrew.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_LUNDY_Kate.pdf","http://www.finance.gov.au/sites/default/files//sites/default/files/P33_SESELJA_Zed.pdf"]
for url in urls:
pdfdata = urllib2.urlopen(url).read()
print "The pdf file has %d bytes" % len(pdfdata)
xmldata = scraperwiki.pdftoxml(pdfdata)
#print "After converting to xml it has %d bytes" % len(pdfdata)
#print "The first 2000 characters are: ", pdfdata
parser = lxml.etree.XMLParser(recover=True)
root = lxml.etree.XML(xmldata, parser)
pages = list(root)
#print "The pages are numbered:", [ page.attrib.get("number") for page in pages ]
# this function has to work recursively because we might have "<b>Part1 <i>part 2</i></b>"
def gettext_with_bi_tags(el):
res = [ ]
if el.text:
res.append(el.text)
for lel in el:
res.append("<%s>" % lel.tag)
res.append(gettext_with_bi_tags(lel))
res.append("</%s>" % lel.tag)
if el.tail:
res.append(el.tail)
return "".join(res)
#Get politician's name
politician = url.split('/')[-1].split('.pdf')[0].replace('_', ' ')
print politician
# check through the PDF to find the travel expense pages, store in list
travelExpPg = []
startHeight = []
for pageno, page in enumerate(pages):
for el in list(page):
print el.attrib, gettext_with_bi_tags(el)
if gettext_with_bi_tags(el).strip().replace(' ', '') == "<b>DomesticScheduledFares</b>":
travelExpPg.append(pageno)
startHeight.append(int(el.attrib['top']))
print travelExpPg
print startHeight
if not travelExpPg:
print "No expenses available"
else:
print "Travel expense pages: " + str(travelExpPg)
#Get the rows on each travel expense page
for i, pageno in enumerate(travelExpPg):
rows = []
for val, el in enumerate(list(pages[pageno])):
if el.tag == "text" and int(el.attrib['left']) > 45 and int(el.attrib['left']) < 48 and int(el.attrib['top']) > startHeight[i] and int(el.attrib['top']) < 1223:
rows.append(val)
#print rows
for i in xrange(0, len(rows)):
rangerows = []
if not rows:
print "rows empty"
else:
for x in xrange(rows[i-1], rows[i]):
rangerows.append(gettext_with_bi_tags(list(pages[pageno])[x]))
#print rangerows
rowlen = len(rangerows)
data = {}
for z in xrange(0, len(rangerows)):
data['politician'] = politician
data[str(z)] = rangerows[z]
data['key'] = str(rows[i]) + url + str(pageno) + politician
data['url'] = url
print "data:"
print data
if not data:
print "it's empty"
else:
scraperwiki.sqlite.save(unique_keys=["key"], data=data)
#Get the last row on each page
rangerows = []
if not rows:
print "rows empty"
else:
lastrowheight = int(list(pages[pageno])[rows[len(rows)-1]].attrib['top'])
for el in list(pages[pageno]):
if el.tag == "text" and int(el.attrib['top']) == lastrowheight:
rangerows.append(gettext_with_bi_tags(el))
#print rangerows
data = {}
for z in xrange(0, len(rangerows)):
data['politician'] = politician
data[str(z)] = rangerows[z]
data['key'] = str(rows[i]) + url + str(pageno) + politician
data['url'] = url
print "data:"
print data
if not data:
print "it's empty"
else:
scraperwiki.sqlite.save(unique_keys=["key"], data=data)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment