Georgiana Bere georgiana-b
georgiana-b
/ create_lai_product_datasets.py
Created
April 22, 2020 08:54
Script to create output datasets from the ITC Leaf Area Index datasets
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import requests | |
| import json | |
| import os | |
| import sys | |
| import glob | |
| import datetime | |
| import pprint | |
| from bs4 import BeautifulSoup as Soup | |
| try: |
georgiana-b
/ elvis-distributed-config.json
Last active
July 5, 2018 14:00
Full distributed config of the elvis database in ODB sharding test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "@type": "d", | |
| "@version": 0, | |
| "readYourWrites": true, | |
| "newNodeStrategy": "static", | |
| "servers": { | |
| "@type": "d", | |
| "@version": 0, | |
| "boss": "MASTER", | |
| "*": "master", |
georgiana-b
/ topojson.json
Created
July 6, 2017 17:44
The self-contained version of the TopoJSON JSON schema here: https://github.com/nhuebel/TopoJSON_schema
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "$schema":"http://json-schema.org/draft-04/schema#", | |
| "title":"TopoJSON object", | |
| "description":"Schema for a TopoJSON object", | |
| "type":"object", | |
| "required":[ | |
| "type" | |
| ], | |
| "properties":{ | |
| "bbox":{ |
georgiana-b
/ csvs_with_duplicates_eu_structural_funds.py
Last active
June 21, 2017 20:37
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # -*- coding: utf-8 -*- | |
| from sqlalchemy import create_engine | |
| import pandas | |
| import os | |
| def has_duplicates(dataframe): | |
| duplicates = dataframe[dataframe.duplicated()] | |
| # Uncomment line below to see the duplicated rows in the dataframe | |
| # print(duplicates) | |
| return not duplicates.empty |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| id | name | slug | |
|---|---|---|---|
| ao | Angola | angola | |
| ag | Antigua and Barbuda | antigua-and-barbuda | |
| ar | Argentina | argentina | |
| am | Armenia | armenia | |
| aw | Aruba | aruba | |
| au | Australia | australia | |
| at | Austria | austria | |
| bd | Bangladesh | bangladesh | |
| be | Belgium | belgium |
georgiana-b
/ trial_index_mapping.json
Last active
January 21, 2017 13:36
`trial` index mapping as returned by ElasticSearch
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "trials_2017-01-21_1485004350175" : { | |
| "mappings" : { | |
| "trial" : { | |
| "dynamic_templates" : [ { | |
| "identifiers_values_arent_analyzed" : { | |
| "mapping" : { | |
| "index" : "not_analyzed", | |
| "type" : "string" | |
| }, |
georgiana-b
/ elasticsearch_response_match_all.json
Created
January 21, 2017 13:18
Trials from `trial` index contain fields that were not included in the mapping (`source`, `risks_of_bias`, `status`, `results_exemption_date` etc.)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "took" : 21, | |
| "timed_out" : false, | |
| "_shards" : { | |
| "total" : 5, | |
| "successful" : 5, | |
| "failed" : 0 | |
| }, | |
| "hits" : { | |
| "total" : 2, |
georgiana-b
/ multiple_nct_processor.py
Created
January 6, 2017 11:13
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # -*- coding: utf-8 -*- | |
| from __future__ import absolute_import | |
| from __future__ import division | |
| from __future__ import print_function | |
| from __future__ import unicode_literals | |
| import json | |
| def process(conf, conn): | |
| # import ipdb; ipdb.set_trace() |
georgiana-b
/ similar_columns.py
Last active
September 23, 2016 08:33
Find pairs of duplicated columns, both exact duplicates and percentual duplicates
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import pandas | |
| def find_exact_duplicate_columns(frame): | |
| """Find pairs of columns that are exact duplicates | |
| i.e. each value should match with the other's column value for all rows. | |
| Adapted from this thoughtful answer: http://stackoverflow.com/a/32961145 | |
| """ | |
| dups = [] | |
| columns = frame.columns |
georgiana-b
/ io_operation_on_closed_file.txt
Last active
August 12, 2016 15:53
+ corrupted double-linked list: 0x00007f91f0008990. This happens for http://data.gov.au/dataset/0de37904-43e0-4814-b21b-5b64fafefe6f/resource/1c48292a-9bfb-476c-850f-7b0da3c273fc/download/prodcom.csv with the code form this fork: https://github.com/georgiana-b/goodtables/tree/feature/parallel_batch_processing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| http://data.gov.au/dataset/0de37904-43e0-4814-b21b-5b64fafefe6f/resource/1c48292a-9bfb-476c-850f-7b0da3c273fc/download/prodcom.csv | |
| Traceback (most recent call last): | |
| File "/home/g/.virtualenvs/uk-spend/bin/dq", line 9, in <module> | |
| load_entry_point('data-quality==0.1.1', 'console_scripts', 'dq')() | |
| File "/home/g/.virtualenvs/uk-spend/lib/python3.4/site-packages/click/core.py", line 716, in __call__ | |
| return self.main(*args, **kwargs) | |
| File "/home/g/.virtualenvs/uk-spend/lib/python3.4/site-packages/click/core.py", line 696, in main | |
| rv = self.invoke(ctx) | |
| File "/home/g/.virtualenvs/uk-spend/lib/python3.4/site-packages/click/core.py", line 1060, in invoke | |
| return _process_result(sub_ctx.command.invoke(sub_ctx)) |
NewerOlder