Skip to content

Instantly share code, notes, and snippets.

@tvorogme

tvorogme/spec.py Secret

Created April 10, 2019 15:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save tvorogme/d139e550720fc2725fbd15adb5785a40 to your computer and use it in GitHub Desktop.
Save tvorogme/d139e550720fc2725fbd15adb5785a40 to your computer and use it in GitHub Desktop.
data = {'type': 'index',
'spec': {'dataSchema': {'dataSource': 'checkline',
'parser': {'type': 'string',
'parseSpec': {"format": "json",
'dimensionsSpec': {'dimensions': [{'name': 'CashCheckLineNo'.lower(),
'type': 'long'},
{'name': 'BasePrice'.lower(), 'type': 'float'},
{'name': 'Quantity'.lower(), 'type': 'double'},
{'name': 'BaseSum'.lower(), 'type': 'float'},
{'name': 'id_tt_cl'.lower(), 'type': 'long'},
{'name': 'id_tov_cl'.lower(), 'type': 'long'},
{'name': 'time_ch'.lower(), 'type': 'long'},
{'name': 'OperationType_cl'.lower(), 'type': 'long'},
{'name': 'znak'.lower(), 'type': 'long'},
{'name': 'LoadDateTime_cl'.lower(), 'type': 'long'},
{'name': 'date_add'.lower(), 'type': 'long'},
{'name': 'Price_retail'.lower(), 'type': 'float'},
{'name': 'id_kontr'.lower(), 'type': 'long'},
{'name': 'BonusCard_cl'.lower(), 'type': 'string'},
{'name': 'id_coupons_type2_card_tov'.lower(), 'type': 'long'},
{'name': 'id_lovepr_card_tov'.lower(), 'type': 'long'},
{'name': 'id_discount_chl'.lower(), 'type': 'long'},
{'name': 'set_price'.lower(), 'type': 'float'},
{'name': 'green_price'.lower(), 'type': 'long'}]},
'timestampSpec': {'column': 'date_ch', 'format': 'iso'}}},
'metricsSpec': [{'type': 'hyperUnique',
'name': 'CheckLineUID'.lower(),
'fieldName': 'CheckLineUID'.lower()},
{'type': 'hyperUnique', 'name': 'CheckUID'.lower(), 'fieldName': 'CheckUID'.lower()}],
'granularitySpec': {'type': 'uniform',
'segmentGranularity': 'day',
'queryGranularity': 'none',
'rollup': False}},
"transformSpec" : {
"filter" : None,
"transforms" : [ {
"type" : "expression",
"name" : "CashCheckLineNo".lower(),
"expression" : "(\"CashCheckLineNo\")"
}, {
"type" : "expression",
"name" : "BasePrice".lower(),
"expression" : "(\"BasePrice\")"
}, {
"type" : "expression",
"name" : "Quantity".lower(),
"expression" : "(\"Quantity\")"
} , {
"type" : "expression",
"name" : "BaseSum".lower(),
"expression" : "(\"BaseSum\")"
} , {
"type" : "expression",
"name" : "id_tt_cl".lower(),
"expression" : "(\"id_tt_cl\")"
} , {
"type" : "expression",
"name" : "id_tov_cl".lower(),
"expression" : "(\"id_tov_cl\")"
} , {
"type" : "expression",
"name" : "time_ch".lower(),
"expression" : "(\"time_ch\")"
} , {
"type" : "expression",
"name" : "znak".lower(),
"expression" : "(\"znak\")"
} , {
"type" : "expression",
"name" : "LoadDateTime_cl".lower(),
"expression" : "(\"LoadDateTime_cl\")"
} , {
"type" : "expression",
"name" : "date_add".lower(),
"expression" : "(\"date_add\")"
} , {
"type" : "expression",
"name" : "Price_retail".lower(),
"expression" : "(\"Price_retail\")"
} , {
"type" : "expression",
"name" : "id_kontr".lower(),
"expression" : "(\"id_kontr\")"
} , {
"type" : "expression",
"name" : "BonusCard_cl".lower(),
"expression" : "(\"BonusCard_cl\")"
}, {
"type" : "expression",
"name" : "id_coupons_type2_card_tov".lower(),
"expression" : "(\"id_coupons_type2_card_tov\")"
}, {
"type" : "expression",
"name" : "id_lovepr_card_tov".lower(),
"expression" : "(\"id_lovepr_card_tov\")"
}, {
"type" : "expression",
"name" : "id_discount_chl".lower(),
"expression" : "(\"id_discount_chl\")"
}, {
"type" : "expression",
"name" : "set_price".lower(),
"expression" : "(\"set_price\")"
}, {
"type" : "expression",
"name" : "green_price".lower(),
"expression" : "(\"green_price\")"
}, {
"type" : "expression",
"name" : "green_price".lower(),
"expression" : "(\"green_price\")"
}, {
"type" : "expression",
"name" : "CheckUID".lower(),
"expression" : "(\"CheckUID\")"
}, {
"type" : "expression",
"name" : "CheckLineUID".lower(),
"expression" : "(\"CheckLineUID\")"
} ]
},
'ioConfig': {'type': 'index',
'firehose': {
"type" : "ingestSegment",
"dataSource" : "checklines",
"interval" : "1999-01-01/3000-01-01"
}},
'appendToExisting': True},
'tuningConfig': {'type': 'index',
'targetPartitionSize': 9000000,
'maxRowsInMemory': 9000000,
'forceExtendableShardSpecs': True}}
task = post_task(spec)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment