rhanka

# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE
# -*- coding: utf-8 -*-
import dataiku
import pandas as pd, numpy as np
from dataiku import pandasutils as pdu
# from birdy.twitter import UserClient, StreamClient
import json

# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE
TWITTER_CONSUMER_KEY = "xxx"

rhanka

# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE
# -*- coding: utf-8 -*-
import dataiku
import os
import json
import pandas as pd, numpy as np
from dataiku import pandasutils as pdu
os.environ['http_proxy'] = ''
import dataikuapi

rhanka

/*
 * resource.filetype: migrate OLD_DOMAINS to NEW_DOMAIN
 * WARNING : it uses a basic 'replace' without a strict regex, be careful about the OLD_DOMAINS
 */

var count = 0;
var urlRegex = /^https?\:\/\/alpha.datalab.mi.*$/i;

db.dataset.find({'resources.filetype': 'file'}).forEach(function(dataset) {
    if (dataset.resources) {

rhanka

#!/bin/sh
#get all schemas of the dump
schemas=`cat $1 |  docker exec -i postgres pg_restore -l  | grep SCHEMA | sed 's/^.*\s//' | sort | uniq`

for schema in $schemas
do echo $schema
  for table in `cat $1 | docker exec -i postgres pg_restore -l | egrep "TABLE $schema" |sed "s/.*TABLE $schema //;s/ $schema//' | sort | uniq `
    do echo $table
      cat $1 | docker exec -i postgres pg_restore -t $table | perl -e 'while(<>){if (/^\\.\s*$/) {$dump=0} if ($dump==1) {s/\\N//g;print} if (/COPY/){$dump=1;s/.*\((.*)\).*/$1/;s/,\s*/\t/g;print}}'  > $table.csv
    done