Robin Linacre RobinL

## example_deduplicate_50k_synthetic.ipynb

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                RobinL
                / example_deduplicate_50k_synthetic.ipynb
            
            
              Last active
              June 24, 2023 10:23
            
              
                test_charts
              
          
      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.
      
    
## edges_data.json
[{"tf_adjusted_match_prob":0.990110793,"match_probability":0.9733507932,"source_dataset_l":"synthentic_data","unique_id_l":"Q15990270-11","source_dataset_r":"synthentic_data","unique_id_r":"Q15990270-13","surname_std_l":"searle","surname_std_r":"searle","forename1_std_l":"william","forename1_std_r":"william","forename2_std_l":"george","forename2_std_r":"georgina","forename3_std_l":null,"forename3_std_r":null,"forename4_std_l":null,"forename4_std_r":null,"forename5_std_l":null,"forename5_std_r":null,"gamma_surname_std":3,"gamma_forename1_std":3,"gamma_forename2_std":1,"occupation_l":null,"occupation_r":null,"gamma_occupation":-1,"dob_l":"1829-01-04","dob_r":null,"gamma_dob":-1,"postcode_l":"SE4 1TS","postcode_r":null,"lat_lng_l":{"lat":51.463683,"long":-0.029905},"lat_lng_r":null,"birth_place_l":"Bexley","birth_place_r":"Bexley","gamma_custom_postcode_distance_comparison":2,"cluster_l":"Q15990270","cluster_r":"Q15990270","match_key":5,"commit_hash":"20feacc","version":"v01","input_datasets":"uk_citizens_max_gr

## fails.py
import requests
import json
import pandas as pd
from io import StringIO
import time

url = "https://api.beta.ons.gov.uk/v1/filters?submitted=true"
ages = range(0,90)
ages = [str(a) for a in ages]
years = [2015,2016,2017]

## get_data.py
import requests
import json
import pandas as pd
from io import StringIO
import time

url = "https://api.beta.ons.gov.uk/v1/filters?submitted=true"

post = {
    "dataset": {

## spec.json
{"$schema": "https://vega.github.io/schema/vega/v3.0.json",
  "width": 500,
  "height": 600,
  "autosize": "none",
  "signals": [
    {
      "name": "translate0",
      "update": "width / 2"
    },
    {

## random_tasks.py
from airflow import DAG

from airflow.operators.bash_operator import BashOperator

from datetime import datetime, timedelta

default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'start_date': datetime(2018,10,1),

## gist:e13df2ee55aa6c1117d5d6b6371e9b69
{
  "quoteResponse": {
    "result": [
      {
        "currency": "USD",
        "esgPopulated": false,
        "exchange": "CCY",
        "exchangeDataDelayedBy": 0,
        "exchangeTimezoneName": "Europe/London",
        "exchangeTimezoneShortName": "GMT",

## testschema.json
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "type": "object",
  "title": "Metadata",
  "description": "MoJ Data Catalogue Metadata",
  "properties": {
    "id": {
      "type": "string",
      "title": "The ID of this table.  Unique identifier which enables a table's metadata to refer to another table using e.g. for a foreign key",
      "examples": [

## add_pk_to_sqlite_table.py
import re

def get_create_table_string(tablename, connection):
    sql = """
    select * from sqlite_master where name = "{}" and type = "table"
    """.format(tablename)
    result = connection.execute(sql)

    create_table_string = result.fetchmany()[0][4]
    return create_table_string

## .block
license: mit
	import requests
	import json
	import pandas as pd
	from io import StringIO
	import time

	url = "https://api.beta.ons.gov.uk/v1/filters?submitted=true"
	ages = range(0,90)
	ages = [str(a) for a in ages]
	years = [2015,2016,2017]
	{"$schema": "https://vega.github.io/schema/vega/v3.0.json",
	"width": 500,
	"height": 600,
	"autosize": "none",
	"signals": [
	{
	"name": "translate0",
	"update": "width / 2"
	},
	{
	from airflow import DAG

	from airflow.operators.bash_operator import BashOperator

	from datetime import datetime, timedelta

	default_args = {
	'owner': 'airflow',
	'depends_on_past': False,
	'start_date': datetime(2018,10,1),
	{
	"quoteResponse": {
	"result": [
	{
	"currency": "USD",
	"esgPopulated": false,
	"exchange": "CCY",
	"exchangeDataDelayedBy": 0,
	"exchangeTimezoneName": "Europe/London",
	"exchangeTimezoneShortName": "GMT",
	{
	"$schema": "http://json-schema.org/draft-07/schema#",
	"type": "object",
	"title": "Metadata",
	"description": "MoJ Data Catalogue Metadata",
	"properties": {
	"id": {
	"type": "string",
	"title": "The ID of this table. Unique identifier which enables a table's metadata to refer to another table using e.g. for a foreign key",
	"examples": [
	import re

	def get_create_table_string(tablename, connection):
	sql = """
	select * from sqlite_master where name = "{}" and type = "table"
	""".format(tablename)
	result = connection.execute(sql)

	create_table_string = result.fetchmany()[0][4]
	return create_table_string