edstenson

{
    "id": "9e97d9ac115e58fcb7ab",
    "definition_id": "cdd0b9bb266e8f3f2de405f6c1a2b23e",
    "name": "job-estimation-test",
    "start": 1325376001,
    "end": 1325894401,
    "created_at": 1363271961,
    "status": "running",
    "progress": 15,
    "sources": [

edstenson

{
    "count": 2,
    "subscriptions": [
        {
            "id": "d668655cfe5f93741ddcd30bb309a8c7",
            "output_type": "dynamodb",
            "name": "mypushsubscription1",
            "created_at": 1365430066,
            "hash": "42d388f8b1db997faaf7dab487f11290",
            "hash_type": "stream",

edstenson

{
    "id": "d468655cfe5f93741ddcd30bb309a8c7",
    "output_type": "dynamodb",
    "name": "mypushsubscription1",
    "created_at": 1365430066,
    "hash": "42d388f8b1db997faaf7dab487f11290",
    "hash_type": "stream",
    "output_params": {
        "table": "datasift-dynamodb",
        "region": "dynamodb.eu-west-1.amazonaws.com"

edstenson

{
    "count": 2,
    "page": 1,
    "pages": 1,
    "per_page": 20,
    "sources": [
        {
            "name": "",
            "source_type": "facebook_page",
            "status": "running",

edstenson

CREATE TABLE TWITTER (
    INTERACTION_ID  varchar(64),
    CONTENT         text,
    HASHTAGS        text[],
    MENTION_IDS     int[]
);

edstenson

[TWITTER]
INTERACTION_ID = interaction.id
CONTENT = twitter.text
HASHTAGS = twitter.hashtags (data_type: string-array)
MENTION_IDS = twitter.mention_ids (data_type: integer-array)

edstenson

curl -X POST 'https://api.datasift.com/v1/push/create' \
-d 'name=connectorpostgresql' \
-d 'hash=42d388f8b1db997faaf7dab487f11290' \
-d 'output_type=postgresql' \
-d 'output_params.host=db.example.com' \
-d 'output_params.port=3306' \
-d 'output_params.mapping=W2ludGVyYWN0aW9uXQ0KaW50ZXJhY3Rpb25faWQgPSBpbnRlcmFjdGlvbi5pZA0KY3JlYXRlZF9hdCA9IGludGVyYWN0aW9uLmNyZWF0ZWRfYXQgKGRhdGFfdHlwZTogZGF0ZXRpbWUsIHRyYW5zZm9ybTogZGF0ZXRpbWUpDQpjcmVhdGVkX2F0X3VuaXggPSBpbnRlcmFjdGlvbi5jcmVhdGVkX2F0IChkYXRhX3R5cGU6IGludGVnZXIsIHRyYW5zZm9ybTogdW5peHRpbWVzdGFtcCkNCnVzZXJuYW1lID0gaW50ZXJhY3Rpb24uYXV0aG9yLnVzZXJuYW1lIA0KaXNfcmV0d2VldCA9IHR3aXR0ZXIucmV0d2VldGVkLmlkIChkYXRhX3R5cGU6IGludGVnZXIsIHRyYW5zZm9ybTogZXhpc3RzKQ0KdHdpdHRlcl9sYW5nID0gdHdpdHRlci5sYW5nDQp0d2l0dGVyX2xhbmcgPSAgdHdpdHRlci5yZXR3ZWV0LmxhbmcNCmZpcnN0X3RhZyA9IGludGVyYWN0aW9uLnRhZ3MgKHRyYW5zZm9ybTogZmlyc3Qp' \
-d 'output_params.database=mydb' \
-d 'output_params.auth.username=dsdb' \
-d 'output_params.auth.password=dspassword' \

edstenson

import argparse
import base64
import urllib
import sys

# parse arguments

parser = argparse.ArgumentParser(description='Convert a .p12 file into a string a Google Big Query Push connector can use.')

parser.add_argument('-f', required=True, action='store', dest='fin', 

edstenson

curl -X POST 'https://api.datasift.com/v1/push/create' \
-d 'name=connectorgooglebigquery' \
-d 'hash=42d388f8b1db497faaf7dab487f11290' \
-d 'output_type=bigquery' \
-d 'output_params.project_id=000000000000' \
-d 'output_params.dataset_id=interactions' \
-d 'output_params.table_id=pushedinteractions' \
-d 'output_params.auth.client_id=000000000000.apps.googleusercontent.com' \
-d 'output_params.auth.service_account=000000000000@developer.gserviceaccount.com' \
-d "output_params.auth.key_file=`python ./p12tobigquery.py -f 3a46aede3a0a2d375b7bdb72743e7da7605fef5c-privatekey.p12`"

edstenson

[SUBSCRIPTION]
SUB_ID = subscription.id
SUB_HASH = subscription.hash
SUB_HASHTYPE = subscription.hash_type