This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE | |
# -*- coding: utf-8 -*- | |
import dataiku | |
import os | |
import json | |
import pandas as pd, numpy as np | |
from dataiku import pandasutils as pdu | |
os.environ['http_proxy'] = '' | |
import dataikuapi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
* resource.filetype: migrate OLD_DOMAINS to NEW_DOMAIN | |
* WARNING : it uses a basic 'replace' without a strict regex, be careful about the OLD_DOMAINS | |
*/ | |
var count = 0; | |
var urlRegex = /^https?\:\/\/alpha.datalab.mi.*$/i; | |
db.dataset.find({'resources.filetype': 'file'}).forEach(function(dataset) { | |
if (dataset.resources) { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE | |
# -*- coding: utf-8 -*- | |
from multiprocessing import Process, Queue | |
import dataiku | |
from dataiku.customrecipe import get_input_names_for_role | |
from dataiku.customrecipe import get_output_names_for_role | |
from dataiku.customrecipe import get_recipe_config | |
import itertools | |
import logging | |
import pandas as pd |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
#get all schemas of the dump | |
schemas=`cat $1 | docker exec -i postgres pg_restore -l | grep SCHEMA | sed 's/^.*\s//' | sort | uniq` | |
for schema in $schemas | |
do echo $schema | |
for table in `cat $1 | docker exec -i postgres pg_restore -l | egrep "TABLE $schema" |sed "s/.*TABLE $schema //;s/ $schema//' | sort | uniq ` | |
do echo $table | |
cat $1 | docker exec -i postgres pg_restore -t $table | perl -e 'while(<>){if (/^\\.\s*$/) {$dump=0} if ($dump==1) {s/\\N//g;print} if (/COPY/){$dump=1;s/.*\((.*)\).*/$1/;s/,\s*/\t/g;print}}' > $table.csv | |
done |
NewerOlder