rufuspollock

// the location we want to GeoCode
var location = 'London';

// we are using MapQuest's Nominatim service
var geocode = 'http://open.mapquestapi.com/search?format=json&q=' + location;

// use jQuery to call the API and get the JSON results
$.getJSON(geocode, function(data) {
  // get lat + lon from first match
  var latlng = [data[0].lat, data[0].lon]

rufuspollock

Challenge 1

Your task: write a script to get a nice CSV file of natural gas prices.

Please publish your results in a git repo or a gist. Please include both script and your resulting data -- so the CSV files should be stored in the repo too!

More detail:

• Prices should be Henry Hub gas prices. Use EIA data here: http://www.eia.gov/dnav/ng/hist/rngwhhdm.htm
• Hint: you can get the data from any data source on the page ...

rufuspollock

jQuery(function ($) {
        var elem = $('#text-to-annotate');
        var account_id = '39fc339cf058bd22176771b3e3036609';
        var annotator_store = '/annostore' + '/api';
        var userid = '';
        var options = {};
        options.permissions = {};
        options.permissions.user = {
          'name': '194.104.70.73'
        };

rufuspollock

import os
import shutil

def convert():
    tmp = '/tmp/togif'
    if os.path.exists(tmp):
        print('Temp path %s already exists' % tmp)
        return
    os.makedirs(tmp)

rufuspollock

# python 3+
def convert_data_package_to_ckan_package(data_package):
    '''
    Documentation of CKAN metadata structure ...

    https://docs.ckan.org/en/2.8/api/index.html#ckan.logic.action.create.package_create
    https://docs.ckan.org/en/2.8/api/index.html#ckan.logic.action.create.resource_create
    '''
    out = dict(data_package)
    out['extras'] = []

rufuspollock

var nodeUtil = require("util"),
    PFParser = require("pdf2json")
;

var pdfParser = new PFParser();

pdfParser.on("pdfParser_dataReady", function(data) {
  console.log('here');
  console.log(data);
  console.log(data.data.Pages[0]);

rufuspollock

'''Simple script for creating demo data in CKAN

Requires existence of a tester user. You can create this by doing::

   paster create-test-data user
'''
import ckanclient

base_location = 'http://localhost:5000/api'
api_key = 'tester'

rufuspollock

Additions wanted - please just fork and add.

Tutorials

• Parsing PDFs by Thomas Levine
• [Get Started With Scraping – Extracting Simple Tables from PDF Documents][scoda-simple-tables]

Generic (PDF -> text)

rufuspollock

name: opsd-time-series

title: Time series

description: Load, wind and solar, prices in hourly resolution

long_description: This data package contains different kinds of time series data relevant for power system modelling, namely electricity consumption (load) for 36 European countries as well as...

homepage: http://data.open-power-system-data.org/time_series/2016-07-14/

rufuspollock

#!/usr/bin/env python
# A simple Python script to convert csv files to sqlite (with type guessing)
#
# @author: Rufus Pollock
# Placed in the Public Domain
import csv
import sqlite3

def convert(filepath_or_fileobj, dbpath, table='data'):
    if isinstance(filepath_or_fileobj, basestring):