Skip to content

Instantly share code, notes, and snippets.

Avatar

Yolanda Davis YolandaMDavis

View GitHub Profile
View hdf-nifi-cc-sandbox-start.sh
#!/bin/bash
echo "Deploying HDF Sandbox Container:"
until docker ps 2>&1| grep STATUS>/dev/null; do sleep 1; done; >/dev/null
docker ps -a | grep sandbox
if [ $? -eq 0 ]; then
docker start sandbox
else
docker run --name sandbox --hostname "sandbox-hdf.hortonworks.com" --privileged -d \
-p 12181:2181 \
-p 17010:16010 \
View TwitterJoltEl_PutFile_Mongo.xml
<?xml version="1.0" ?>
<template encoding-version="1.1">
<description>Example of using Expression Language with Jolt. Twitter data is obtained
and transformed then stored in a file as well as a Mongo repo.</description>
<groupId>142ee790-015c-1000-9a0d-a31aac7e0eb3</groupId>
<name>TwitterJoltEl_PutFile_Mongo</name>
<snippet>
<connections>
<id>dc4efe01-905c-39bb-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
@YolandaMDavis
YolandaMDavis / GetTwitterData-MongoSQL.xml
Created May 21, 2017
Managing Data Flow with NiFi - Demo Templates
View GetTwitterData-MongoSQL.xml
<?xml version="1.0" ?>
<template encoding-version="1.1">
<description></description>
<groupId>142ee790-015c-1000-9a0d-a31aac7e0eb3</groupId>
<name>GetTwitterDataSQLMongo-2017</name>
<snippet>
<connections>
<id>bcbbda47-e17a-32fb-0000-000000000000</id>
<parentGroupId>e5a17068-5da9-3260-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
@YolandaMDavis
YolandaMDavis / Chain Example
Created Feb 7, 2017
Twitter JOLT Specifications - Jolt 1.0
View Chain Example
[{
"operation": "shift",
"spec": {
"created_at": "created_date_time",
"id": "tweet_id",
"text": "tweet_text",
"*":"&"
}
},{
"operation": "default",
@YolandaMDavis
YolandaMDavis / CustomPropertyDemo.xml
Created Jul 15, 2016
Custom Property Support in EL Demo for NiFi
View CustomPropertyDemo.xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><template encoding-version="1.0"><description>This flow demonstrates use of custom properties in EL with UpdateAttributes processor and the PutFile processor</description><groupId>53d29129-3b42-40ab-a008-81ebb557b334</groupId><name>CustomPropertyDemo</name><snippet><connections><id>748758a2-2eb8-42fe-8fc6-d11d467c1e6c</id><parentGroupId>53d29129-3b42-40ab-a008-81ebb557b334</parentGroupId><backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold><backPressureObjectThreshold>10000</backPressureObjectThreshold><destination><groupId>53d29129-3b42-40ab-a008-81ebb557b334</groupId><id>4045eff6-fafc-4979-9138-b63d465c059a</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>success</selectedRelationships><source><groupId>53d29129-3b42-40ab-a008-81ebb557b334</groupId><id>70bb9f9d-0640-4122-bdc4-ce5e95cd27f6</id><type>PROCESSOR</type></source><zIndex>0</zIndex></
@YolandaMDavis
YolandaMDavis / twitter_chain_spec.json
Created Jul 10, 2016
Example Jolt Transform Specifications for Twitter Data
View twitter_chain_spec.json
[{
"operation": "shift",
"spec": {
"created_at": "created_date_time",
"id": "tweet_id",
"text": "tweet_text",
"user": {
"id": "user_id"
}
}
@YolandaMDavis
YolandaMDavis / JoltTransformJSON_Twitter.xml
Created Jul 10, 2016
NiFi Template for Twitter Streams Transformed with Jolt
View JoltTransformJSON_Twitter.xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><template><description></description><name>JoltTransformJSON_Twitter</name><snippet><connections><id>4b7692b8-a3f2-4efc-9105-dc10a1a32571</id><parentGroupId>1c00c635-485b-4e88-89b0-b84da3fd3e05</parentGroupId><backPressureDataSizeThreshold>10 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>100</backPressureObjectThreshold><destination><groupId>1c00c635-485b-4e88-89b0-b84da3fd3e05</groupId><id>694eaedb-5a0b-47ec-8fcb-64548799da78</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name>Transform Tweets</name><selectedRelationships>success</selectedRelationships><source><groupId>1c00c635-485b-4e88-89b0-b84da3fd3e05</groupId><id>8e566f04-31d3-4932-aea1-ba1aa445ec45</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>acf4bde3-2252-421a-9e69-69b6cc52e382</id><parentGroupId>1c00c635-485b-4e88-89b0-b84da3fd3e05</parentGroupId><backPressureDataSizeThreshold