Skip to content

Instantly share code, notes, and snippets.

View NTerpo's full-sized avatar

terpo NTerpo

View GitHub Profile
SELECT ?instrumentPermID ?instrumenthasName ?instrumenthasPermId ?assetClassPermId ?assetClassLabel ?assetClassBroader ?assetClassComment ?instrumentStatus ?primaryQuotehasPermId ?primaryQuotehasExchangeCode ?primaryQuotehasRic ?primaryQuotehasName ?primaryQuotehasExchangeTicker ?primaryQuotehasMic ?instrumentIsIssuedBy ?orgActivityStatus ?orgHasLatestOrganizationFoundedDate ?isIncorporatedIn ?isDomiciledIn ?orgName ?orgHasPrimaryBusinessSector ?orgHasPrimaryIndustryGroup ?orgHasURL ?orgHasOrganizationPrimaryQuote ?orgHasPrimaryInstrument ?orgHasHoldingClassification ?orgHasIPODate ?orgHasLEI ?orgPermId ?org
WHERE {
?org <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://permid.org/ontology/organization/Organization> .
?org <http://permid.org/ontology/financial/hasPrimaryInstrument> ?instrumentPermID .
?instrumentPermID <http://permid.org/ontology/common/hasName> ?instrumenthasName .
?instrumentPermID a <http://permid.org/ontology/financial/Instrument> .
?instr
<iframe frameborder="0" height="600" width="100%" src="https://data.opendatasoft.com/map/rsf_2017/?headless=true"></iframe>
<div class="container-fluid">
<div class="the-map">
<h1>
The map!
</h1>
<div class="ods-box">
<ods-dataset-context context="topmillionwebsites"
topmillionwebsites-dataset="top-million-websites">
<ods-map no-refit="true"
display-control="true"
SELECT ?PermID ?hasPermId ?rank ?prefLabel ?hasReportedTitle ?hasHolder ?hasPositionType ?hasPublicationStatus ?isPositionIn ?to ?familyName ?givenName ?holdsPosition ?hasTenureInOrganization ?additionalName ?hasGender ?bday ?honorificPrefix ?from ?hasQualification ?preferredName ?honorificSuffix ?withDegree ?fromInstitutionName ?inSubject ?isTenureIn ?dateOfDeath
WHERE { OPTIONAL { ?PermID <http://permid.org/ontology/common/hasPermId> ?hasPermId }
OPTIONAL { ?PermID <http://permid.org/ontology/person/rank> ?rank }
OPTIONAL { ?PermID <http://www.w3.org/2004/02/skos/core#prefLabel> ?prefLabel }
OPTIONAL { ?PermID <http://permid.org/ontology/person/hasReportedTitle> ?hasReportedTitle }
OPTIONAL { ?PermID <http://permid.org/ontology/person/hasHolder> ?hasHolder }
OPTIONAL { ?PermID <http://permid.org/ontology/person/hasPositionType> ?hasPositionType }
OPTIONAL { ?PermID <http://permid.org/ontology/common/hasPublicationStatus> ?hasPublicationStatus }
OPTIONA
$ git clone https://github.com/opensdmx/rsdmx.git
$ R
> install.packages("path/to/rsdmx", repos=NULL, type="source")
> packageVersion("rsdmx")
[1] ‘0.5.2’
> library(rsdmx)
> ulc.data <- readSDMX(file = "http://stats.oecd.org/restsdmx/sdmx.ashx/GetData/PDBI_I4/AUS+AUT+BEL+CAN+CHL+CZE+DNK+EST+FIN+FRA+DEU+GRC+HUN+IRL+ISR+ITA+JPN+KOR+LUX+MEX+NLD+NZL+NOR+POL+PRT+SVK+SVN+ESP+SWE+CHE+GBR+USA+EA19+EU28+NMEC+LVA+LTU+ZAF.I4_ANA_GVAHRS+I4_ANA_ULCH+I4_ANA_LCHRS.GRW+AVGRW.A_U+BNEXCL+B_E+BDE+C+F+GNEXCL+G_I+J+K+MN/all?startTime=2001&endTime=2015")
> ulc.dsd <- readSDMX(file = "http://stats.oecd.org/restsdmx/sdmx.ashx/GetDataStructure/PDBI_I4")
{
"language": "en",
"name": "Paris Data",
"description": "City of Paris Open Data portal",
"url": "http://opendata.paris.fr/",
"linked_portals": ["http://data.gouv.fr", "http://data.iledefrance.fr"],
"data_language": ["fr"],
"modified": "2016-03-04T13:44:44+00:00",
"themes": ["Culture, Heritage", "Education, Training, Research, Teaching", "Environment", "Transport, Movements", "Spatial Planning, Town Planning, Buildings, Equipment, Housing", "Health", "Economy, Business, SME, Economic development, Employment", "Services, Social", "Administration, Government, Public finances, Citizenship", "Justice, Safety, Police, Crime", "Sports, Leisure", "Accommodation, Hospitality Industry"],
"links": [
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>http://opendata.paris.fr/explore/dataset/liste_des_sites_des_hotspots_paris_wifi/</loc>
<lastmod>2014-06-16</lastmod>
</url>
<url>
<loc>http://opendata.paris.fr/explore/dataset/points-de-vote-du-budget-participatif/</loc>
<lastmod>2014-09-24</lastmod>
</url>
User-agent: Twitterbot
Disallow:
User-agent: *
Disallow: /logout
Disallow: /login
Disallow: /publish
Disallow: /explore/dataset/*/download/
Disallow: /explore/dataset/*/rss/
Disallow: /explore/dataset/*/atom/
@NTerpo
NTerpo / datacleaning.sh
Created January 29, 2016 16:56
some data cleaning shell
# Change extension for txt files to csv files
$ for f in *.txt
do
[ -f "$f" ] && mv "$f" "${f%txt}csv"
done
# Remove first line every csv files
$ for f in *.csv
do
[ -f "$f" ] && awk '{if (NR==1 || NR==2) next};1' "$f" >> "clean_$f"
- DL data as GeoJSON from ODS
- $ ogr2ogr -f "ESRI Shapefile" foo.shp foo.geojson OGRGeoJSON
- $ shp2pgsql foo.shp foo > foo.sql
- $ createdb gp
- $ psql -d gp
- gp=# CREATE EXTENSION postgis;
- $ psql -f foo.sql -d gp
- gp=# create table merged as
select id, ST_Union(geom)
from foo