\n \n \n \n \n\n
\n \n \n \n
; Convert history of the internet printable wiki into PDF<\/span><\/div>; source from http://www.nethistory.co.nz/index.php/Main_Page<\/span><\/div> <\/div>(<\/span>use<\/span> 'clojure<\/span>.<\/span>contrib<\/span>.<\/span>str-utils<\/span>)<\/span><\/div>(<\/span>import <\/span>'<\/span>(<\/span>java<\/span>.<\/span>io<\/span> ByteArrayInputStream<\/span> FileOutputStream<\/span> File<\/span>)<\/span><\/div>        '<\/span>(<\/span>javax<\/span>.<\/span>xml<\/span>.<\/span>parsers<\/span> DocumentBuilderFactory<\/span>))<\/span><\/div>(<\/span>load-file <\/span>"clj_web_crawler.clj"<\/span>)<\/span><\/div> <\/div>(<\/span>System/setProperty<\/span> "xr.text.aa-fontsize-threshhold"<\/span> "10"<\/span>)<\/span><\/div> <\/div>(<\/span>defn <\/span>replace-all<\/span><\/div>  "A simple bulk regex replacement function."<\/span><\/div>  [<\/span>st<\/span> reps<\/span>]<\/span><\/div>  (<\/span>reduce <\/span>#<\/span>(<\/span>.<\/span>replaceAll<\/span> %1<\/span> (<\/span>first <\/span>%2<\/span>)<\/span> (<\/span>second <\/span>%2<\/span>))<\/span> st<\/span> reps<\/span>))<\/span><\/div> <\/div>(<\/span>defn <\/span>file-name-for<\/span> [<\/span>page<\/span>]<\/span><\/div>  (<\/span>str <\/span>"cache/"<\/span> (<\/span>replace-all<\/span> page<\/span> {<\/span>"/"<\/span> "_"<\/span>})))<\/span><\/div> <\/div>(<\/span>defn <\/span>get-cached-or<\/span> [<\/span>page<\/span> type<\/span> get-uncached-content-with<\/span>]<\/span><\/div>  (<\/span>let <\/span>[<\/span>page-file<\/span> (<\/span>str <\/span>(<\/span>file-name-for<\/span> page<\/span>)<\/span> "-"<\/span> type<\/span>)]<\/span><\/div>       (<\/span>cond <\/span>(<\/span>.<\/span>exists<\/span> (<\/span>File<\/span>.<\/span> page-file<\/span>))<\/span> (<\/span>do<\/span><\/div>         (<\/span>println <\/span>"Reading cached page "<\/span> page-file<\/span>)<\/span> (<\/span>slurp <\/span>page-file<\/span>))<\/span><\/div>             :else<\/span> (<\/span>do<\/span><\/div>         (<\/span>let <\/span>[<\/span>content<\/span> (<\/span>get-uncached-content-with<\/span> page<\/span>)]<\/span><\/div>              (<\/span>.<\/span>write<\/span> (<\/span>FileOutputStream<\/span>.<\/span> page-file<\/span>)<\/span> (<\/span>.<\/span>getBytes<\/span> content<\/span> "UTF-8"<\/span>))<\/span><\/div>              (<\/span>println <\/span>"Saving cached page to "<\/span> page-file<\/span>)<\/span><\/div>              content<\/span><\/div>              )))))<\/span><\/div> <\/div>(<\/span>defn <\/span>get-page<\/span> [<\/span>page<\/span>]<\/span><\/div>  (<\/span>replace-all<\/span><\/div>    (<\/span>clj-web-crawler/crawl-response<\/span> "http://www.nethistory.co.nz"<\/span> (<\/span>str <\/span>"/index.php/"<\/span> page<\/span> "?printable=yes"<\/span>))<\/span><\/div>    {<\/span>"(\\t|\\n)"<\/span> ""<\/span>}))<\/span><\/div> <\/div>(<\/span>defn <\/span>get-cached-page<\/span> [<\/span>page<\/span>]<\/span><\/div>  (<\/span>get-cached-or<\/span> page<\/span> "raw"<\/span> (<\/span>fn <\/span>[<\/span>page<\/span>]<\/span> (<\/span>get-page<\/span> page<\/span>))))<\/span><\/div> <\/div>(<\/span>defn <\/span>get-page-body<\/span> [<\/span>page<\/span>]<\/span><\/div>  (<\/span>first <\/span>(<\/span>rest <\/span>(<\/span>re-split<\/span> #<\/span>"(.*<\\s*body[^>]*>)|(<\\s*/\\s*body\\s*\\>.+)"<\/span> (<\/span>get-cached-page<\/span> page<\/span>)))))<\/span><\/div> <\/div>(<\/span>defn <\/span>strip-crap<\/span> [<\/span>page<\/span>]<\/span><\/div>  (<\/span>get-cached-or<\/span> page<\/span> "stripped"<\/span> (<\/span>fn <\/span>[<\/span>page<\/span>]<\/span><\/div>                                     (<\/span>replace-all<\/span><\/div>                                       (<\/span>str <\/span>(<\/span>first <\/span>(<\/span>re-split<\/span> #<\/span>"<!-- NewPP limit report"<\/span> (<\/span>get-page-body<\/span> page<\/span>)))<\/span> "</div></div></div></div>"<\/span>)<\/span><\/div>                                       {<\/span><\/div>                                         "&nbsp;"<\/span> "&#160;"<\/span><\/div>                                         "<h3 id=\\"site.*/h3>"<\/span> ""<\/span><\/div>                                         "<div id=\\"jump-to-nav.*-->"<\/span> ""<\/span><\/div>                                         "<p>www.nethistory.co.nz</p>"<\/span> ""<\/span><\/div>                                         "<p>([A-Z'\\\\./\\\\s\\\\?!]{4,})([A-Z'\\\\./\\\\?!])"<\/span> "<h3>$1</h3><p>$2"<\/span><\/div>                                         }))))<\/span><\/div> <\/div>(<\/span>def <\/span>title<\/span> "Connecting the Clouds - The Internet in New Zealand"<\/span>)<\/span><\/div> <\/div>(<\/span>defn <\/span>get-content<\/span> [<\/span>page<\/span>]<\/span><\/div>  (<\/span>str <\/span>"<div style='page-break-before: always;'></div>"<\/span> (<\/span>strip-crap<\/span> page<\/span>)))<\/span><\/div> <\/div> <\/div>(<\/span>def <\/span>content<\/span> (<\/span>str<\/span><\/div>  "<html><head><title>"<\/span> title<\/span> "</title>"<\/span><\/div> <\/div>  "<style type='text/css'>"<\/span><\/div>  (<\/span>slurp <\/span>"history.css"<\/span>)<\/span><\/div>  "</style>"<\/span><\/div> <\/div>  "</head><body>"<\/span><\/div>  ;    "<div id='header' style=''>" title "</div>"<\/span><\/div>  ;    "<div id='footer' style=''>Page <span id='pagenumber'/> of <span id='pagecount'/></div>"<\/span><\/div>  "<h1>"<\/span> title<\/span> "</h1>"<\/span><\/div>  "<p>Written by Keith Newman</p>"<\/span><\/div>  (<\/span>get-content<\/span> "Preface"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_1_-_The_Tyranny_of_Distance_-_Reaching_out_to_the_World"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_2_-_Battling_with_Big_Iron_-_Unscrambling_the_Code"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_3_-_No.8_Wire_Networks_-_Patchwork_Quilt_of_Protocols"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_4_-_Nuclear_Free_Reforms_-_Nothing_is_What_it_Seems"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_5_-_Selling_the_Family_Jewels_-_Telecom_Holds_Back_the_Tide"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_6_-_Craving_for_Connection_-_Dawn_of_the_Dial-up_Community"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_7_-_Craving_for_Connection_II_-_The_Pioneering_ISPs"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_8_-_The_Rhythm_Method_-_Regulation_by_Litigation"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_9_-_Local_Loop_Languishing_-_Battling_Bandwidth_Blues"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_10_-_Diminished_Capacity_-_Whose_Foot_is_on_the_Hose%3F"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_11_-_Deluge_in_a_Paper_Cup_-_Knowledge_Wave_Wake-up"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_12_-_E-government_Lumbers_Online_-_Presenting_a_Public_Face"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_13_-_Clicks_and_Mortar_-_Beyond_Online_Pamphlets"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_14_-_Battle_of_the_Names_-_Taming_the_Domains"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_15_-_The_Proxy_Revolution_-_Changing_of_the_Guard"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_16_-_Cyberspace_Junk_-_Nailing_Net_Nasties"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_17_-_Bitstream_Boundaries_-_Sorting_out_Speed_Bumps"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_18_-_Download_Culture_-_Infotainment_on-Demand"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_19_-_Mobile_Momentum_-_Weaned_off_the_Wires"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_20_-_Digital_Refresh_Required_-_Government_Learns_to_Share"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_21_-_Broadband_Breakthrough_-_The_Battle_to_Unbundle"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_22_-_In_the_Recovery_Room_-_Remedial_Learning_Required"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_23_-_IP_Channel_Surfing_-_Digital_Vision_Evolving"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_24_-_Leaping_the_Loop_-_Cloud_Cover_Continues"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_25_-_Next_Step_Internet_-_High_Fibre_Diet_Required"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Chapter_26_-_20/20_Visionaries_-_Beyond_the_Sevens_Cs"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Footnotes"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Acronyms_and_Explanations"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Internet_in_New_Zealand_Timeline"<\/span>)<\/span><\/div>  (<\/span>get-content<\/span> "Postscript_-_Holding_Back_the_Tide_-_Obstacles_to_Advancement"<\/span>)<\/span><\/div>  "</body></html>"<\/span><\/div>  ))<\/span><\/div> <\/div> <\/div> <\/div>(<\/span>def <\/span>document-builder<\/span> (<\/span>.<\/span>newDocumentBuilder<\/span> (<\/span>DocumentBuilderFactory/newInstance<\/span>)))<\/span><\/div>(<\/span>def <\/span>mydoc<\/span> (<\/span>.<\/span>parse<\/span> document-builder<\/span> (<\/span>ByteArrayInputStream<\/span>.<\/span> (<\/span>.<\/span>getBytes<\/span> content<\/span> "UTF-8"<\/span>))))<\/span><\/div> <\/div>(<\/span>doto <\/span>(<\/span>org<\/span>.<\/span>xhtmlrenderer<\/span>.<\/span>pdf<\/span>.<\/span>ITextRenderer<\/span>.<\/span>)<\/span><\/div>  (<\/span>.<\/span>setDocument<\/span> mydoc<\/span> nil<\/span>)<\/span><\/div>  (<\/span>.<\/span>layout<\/span>)<\/span><\/div>  (<\/span>.<\/span>createPDF<\/span> (<\/span>FileOutputStream<\/span>.<\/span> "history.pdf"<\/span>)))<\/span><\/div><\/pre><\/div>\n          \n        <\/div>\n\n        \n          view raw<\/a>\n          history.clj<\/a>\n          This Gist<\/a> brought to you by GitHub<\/a>.\n        <\/div>\n      <\/div>\n    \n  \n<\/div>\n')