Created
December 19, 2013 04:13
-
-
Save nickjevershed/8034331 to your computer and use it in GitHub Desktop.
Scraper for Australian politician's expenses
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| import scraperwiki | |
| import urllib2 | |
| import lxml.etree | |
| urls = ["http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ABBOTT_Tony.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ALBANESE_Anthony.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ALEXANDER_John.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ARBIB_Mark.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BALDWIN_Bob.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BIRD_Sharon.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BISHOP_Bronwyn.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BOWEN_Chris.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BRADBURY_David.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BURKE_Tony.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CAMERON_Doug.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CLARE_Jason.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/COBB_John.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/COMBET_Greg.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/COONAN_Helen.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/COULTON_Mark.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ELLIOT_Justine.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FAULKNER_John.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FERGUSON_Laurie.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FIERRAVANTI-WELLS_Concetta.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FITZGIBBON_Joel.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FLETCHER_Paul.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FORSHAW_Michael.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GARRETT_Peter.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GASH_Joanna.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GRIERSON_Sharon.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HALL_Jill.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HARTSUYKER_Luke.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HAWKE_Alex.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HAYES_Chris.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HEFFERNAN_Bill.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HOCKEY_Joe.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HULL_Kay.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HUSIC_Ed.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HUTCHINS_Steve.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/IRWIN_Julia.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/JONES_Stephen.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/KELLY_Craig.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/KELLY_Mike.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/LEY_Sussan.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MARKUS_Louise.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MATHESON_Russell.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MCCLELLAND_Robert.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MCCORMACK_Michael.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MELHAM_Daryl.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MORRISON_Scott.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MURPHY_John.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/NASH_Fiona.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/NEAL_Belinda.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/OAKESHOTT_Robert.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/O'NEILL_Deb.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/OWENS_Julie.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/PAYNE_Marise.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/PLIBERSEK_Tanya.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/PRICE_Roger.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/RHIANNON_Lee.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ROWLAND_Michelle.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/RUDDOCK_Philip.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SAFFIN_Janelle.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SCHULTZ_Alby.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SINODINOS_Arthur.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/STEPHENS_Ursula.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/THISTLETHWAITE_Matthew.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/THOMSON_Craig.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/TURNBULL_Malcolm.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/WILLIAMS_John.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/WINDSOR_Tony.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ANDREWS_Kevin.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BANDT_Adam.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BILLSON_Bruce.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BROADBENT_Russell.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BURKE_Anna.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BYRNE_Anthony.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CARR_Kim.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CHEESEMAN_Darren.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CHESTER_Darren.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/COLLINS_Jacinta.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CONROY_Stephen.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CREAN_Simon.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/DANBY_Michael.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/DI_NATALE_Richard.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/DREYFUS_Mark.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FEENEY_David.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FERGUSON_Martin.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FIELDING_Steve.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FIFIELD_Mitch.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FORREST_John.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FRYDENBERG_Josh.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GEORGIOU_Petro.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GIBBONS_Steve.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GILLARD_Julia.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GRIFFIN_Alan.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HAWKER_David.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HUNT_Greg.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/JENKINS_Harry.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/KING_Catherine.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/KROGER_Helen.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MACKLIN_Jenny.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MADIGAN_John.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MARLES_Richard.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MARSHALL_Gavin.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MCGAURAN_Julian.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MCKENZIE_Bridget.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MIRABELLA_Sophie.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MITCHELL_Robert.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/O'CONNOR_Brendan.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/O'DWYER_Kelly.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ROBB_Andrew.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/RONALDSON_Michael.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ROXON_Nicola.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/RYAN_Scott.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SHORTEN_Bill.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SMITH_Tony.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SMYTH_Laura.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/STONE_Sharman.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SYMON_Mike.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/TANNER_Lindsay.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/TEHAN_Dan.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/THOMSON_Kelvin.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/TROETH_Judith.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/TUDGE_Alan.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/VAMVAKINOU_Maria.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/WOOD_Jason.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ANDREWS_Karen.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BEVIS_Arch.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BIDGOOD_James.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BOSWELL_Ronald.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BOYCE_Sue.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BRANDIS_George.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BUCHHOLZ_Scott.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CHRISTENSEN_George.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CIOBO_Steven.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/D'ATH_Yvette.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/DUTTON_Peter.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/EMERSON_Craig.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ENTSCH_Warren.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FURNER_Mark.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GAMBARO_Teresa.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HOGG_John.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/JONES_Ewen.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/JOYCE_Barnaby.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/KATTER_Bob.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/LAMING_Andrew.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/LINDSAY_Peter.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/LIVERMORE_Kirsten.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/LUDWIG_Joe.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MACDONALD_Ian.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MACFARLANE_Ian.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MASON_Brett.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MCLUCAS_Jan.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MOORE_Claire.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/NEUMANN_Shayne.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/NEVILLE_Paul.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/O'DOWD_Ken.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/PERRETT_Graham.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/PRENTICE_Jane.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/RAGUSE_Brett.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/REA_Kerry.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/RIPOLL_Bernie.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ROBERT_Stuart.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ROY_Wyatt.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/RUDD_Kevin.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SCOTT_Bruce.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SLIPPER_Peter.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SOMLYAY_Alexander.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SULLIVAN_Jon.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SWAN_Wayne.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/TROOD_Russell.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/TRUSS_Warren.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/TURNOUR_Jim.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/VAN_MANEN_Bert.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/VASTA_Ross.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/WATERS_Larissa.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ADAMS_Judith.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BACK_Chris.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BISHOP_Julie.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BISHOP_Mark.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CASH_Michaelia.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CORMANN_Mathias.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CROOK_Tony.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/EGGLESTON_Alan.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/EVANS_Christopher.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GRAY_Gary.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HAASE_Barry.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/IRONS_Steve.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/JACKSON_Sharryn.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/JENSEN_Dennis.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/JOHNSTON_David.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/KEENAN_Michael.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/LUDLAM_Scott.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MARINO_Nola.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MOYLAN_Judi.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/PARKE_Melissa.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/PRATT_Louise.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/RANDALL_Don.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SIEWERT_Rachel.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SIMPKINS_Luke.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SMITH_Stephen.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/STERLE_Glenn.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/WASHER_Mal.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/WYATT_Ken.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BERNARDI_Cory.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BIRMINGHAM_Simon.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BRIGGS_Jamie.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BUTLER_Mark.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CHAMPION_Nick.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/EDWARDS_Sean.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ELLIS_Kate.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FARRELL_Don.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FAWCETT_David.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FERGUSON_Alan.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/FISHER_Mary_Jo.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GALLACHER_Alex.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GEORGANAS_Steve.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HANSON-YOUNG_Sarah.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HURLEY_Annette.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MCEWEN_Anne.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MINCHIN_Nick.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/PYNE_Christopher.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/RAMSEY_Rowan.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/RISHWORTH_Amanda.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SECKER_Patrick.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SOUTHCOTT_Andrew.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/WONG_Penny.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/WORTLEY_Dana.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/WRIGHT_Penny.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/XENOPHON_Nick.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ZAPPIA_Tony.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ABETZ_Eric.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ADAMS_Dick.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BARNETT_Guy.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BILYK_Catryna.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BROWN_Bob.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BROWN_Carol.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BUSHBY_David.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CAMPBELL_Jodie.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/COLBECK_Richard.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/COLLINS_Julie.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/KERR_Duncan.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/LYONS_Geoff.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MILNE_Christine.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/O'BRIEN_Kerry.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/PARRY_Stephen.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/POLLEY_Helen.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SHERRY_Nick.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SIDEBOTTOM_Sid.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SINGH_Lisa.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/URQUHART_Anne.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/WILKIE_Andrew.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/CROSSIN_Trish.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/GRIGGS_Natasha.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HALE_Damian.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SCULLION_Nigel.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/SNOWDON_Warren.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/BRODTMANN_Gai.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/ELLIS_Annette.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/HUMPHRIES_Gary.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/LEIGH_Andrew.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/LUNDY_Kate.pdf","http://www.finance.gov.au/publications/parliamentarians-reporting/docs/T29/MCMULLAN_Bob.pdf"] | |
| urls2 = ["http://www.finance.gov.au/sites/default/files/P32_ABBOTT_Tony_0.pdf"] | |
| for url in urls: | |
| pdfdata = urllib2.urlopen(url).read() | |
| print "The pdf file has %d bytes" % len(pdfdata) | |
| xmldata = scraperwiki.pdftoxml(pdfdata) | |
| #print "After converting to xml it has %d bytes" % len(pdfdata) | |
| #print "The first 2000 characters are: ", pdfdata | |
| parser = lxml.etree.XMLParser(recover=True) | |
| root = lxml.etree.XML(xmldata, parser) | |
| pages = list(root) | |
| #print "The pages are numbered:", [ page.attrib.get("number") for page in pages ] | |
| # this function has to work recursively because we might have "<b>Part1 <i>part 2</i></b>" | |
| def gettext_with_bi_tags(el): | |
| res = [ ] | |
| if el.text: | |
| res.append(el.text) | |
| for lel in el: | |
| res.append("<%s>" % lel.tag) | |
| res.append(gettext_with_bi_tags(lel)) | |
| res.append("</%s>" % lel.tag) | |
| if el.tail: | |
| res.append(el.tail) | |
| return "".join(res) | |
| #Get politician's name | |
| politician = url.split('/')[-1].split('.pdf')[0].replace('_', ' ') | |
| print politician | |
| # check through the PDF to find the travel expense pages, store in list | |
| travelExpPg = [] | |
| startHeight = [] | |
| for pageno, page in enumerate(pages): | |
| for el in list(page): | |
| print el.attrib, gettext_with_bi_tags(el) | |
| if gettext_with_bi_tags(el).strip().replace(' ', '') == "<b>DomesticScheduledFares</b>": | |
| travelExpPg.append(pageno) | |
| startHeight.append(int(el.attrib['top'])) | |
| print travelExpPg | |
| print startHeight | |
| if not travelExpPg: | |
| print "No expenses available" | |
| else: | |
| print "Travel expense pages: " + str(travelExpPg) | |
| #Get the rows on each travel expense page | |
| for i, pageno in enumerate(travelExpPg): | |
| rows = [] | |
| for val, el in enumerate(list(pages[pageno])): | |
| if el.tag == "text" and int(el.attrib['left']) > 45 and int(el.attrib['left']) < 48 and int(el.attrib['top']) > startHeight[i] and int(el.attrib['top']) < 1223: | |
| rows.append(val) | |
| #print rows | |
| for i in xrange(0, len(rows)): | |
| rangerows = [] | |
| if not rows: | |
| print "rows empty" | |
| else: | |
| for x in xrange(rows[i-1], rows[i]): | |
| rangerows.append(gettext_with_bi_tags(list(pages[pageno])[x])) | |
| #print rangerows | |
| rowlen = len(rangerows) | |
| data = {} | |
| for z in xrange(0, len(rangerows)): | |
| data['politician'] = politician | |
| data[str(z)] = rangerows[z] | |
| data['key'] = str(rows[i]) + url + str(pageno) + politician | |
| data['url'] = url | |
| print "data:" | |
| print data | |
| if not data: | |
| print "it's empty" | |
| else: | |
| scraperwiki.sqlite.save(unique_keys=["key"], data=data) | |
| #Get the last row on each page | |
| rangerows = [] | |
| if not rows: | |
| print "rows empty" | |
| else: | |
| lastrowheight = int(list(pages[pageno])[rows[len(rows)-1]].attrib['top']) | |
| for el in list(pages[pageno]): | |
| if el.tag == "text" and int(el.attrib['top']) == lastrowheight: | |
| rangerows.append(gettext_with_bi_tags(el)) | |
| #print rangerows | |
| data = {} | |
| for z in xrange(0, len(rangerows)): | |
| data['politician'] = politician | |
| data[str(z)] = rangerows[z] | |
| data['key'] = str(rows[i]) + url + str(pageno) + politician | |
| data['url'] = url | |
| print "data:" | |
| print data | |
| if not data: | |
| print "it's empty" | |
| else: | |
| scraperwiki.sqlite.save(unique_keys=["key"], data=data) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment