Skip to content

Instantly share code, notes, and snippets.

Pierre Villard pvillard31

View GitHub Profile
View execute.sh
$ time curl --data-binary "@test.csv" -H "Content-Type: text/csv" -X POST https://minifi-csvjson-t546z2l6aq-uc.a.run.app
[ {
"name" : "Pierre Villard",
"company" : "Google"
} ]
real 0m0.725s
user 0m0.036s
sys 0m0.026s
View publish.sh
git clone git@github.com:pvillard31/nifi-cloudrun-example.git
cd nifi-cloudrun-example
gcloud auth configure-docker
# docker build -t [HOSTNAME]/[PROJECT-ID]/[IMAGE]:[TAG] .
docker build -t eu.gcr.io/my-gcp-project/minifi-csvjson:0.0.1 .
# docker push [HOSTNAME]/[PROJECT-ID]/[IMAGE]:[TAG] .
docker push eu.gcr.io/my-gcp-project/minifi-csvjson:0.0.1
View Dockerfile
FROM apache/nifi-minifi:latest
USER root
ENV MINIFI_HOME /opt/minifi/minifi-0.5.0
ADD config.yml $MINIFI_HOME/conf/config.yml
ADD *.nar $MINIFI_HOME/lib/
RUN chown -R minifi:minifi $MINIFI_HOME
View medium-cloudrun-2
# command to convert the template into yml file for MiNiFi
$MINIFI_HOME/bin/config.sh transform templateConvertRecord.xml config.yml
View medium-cloudrun-1
$ cat test.csv
name,company
Pierre Villard,Google
$ curl --data-binary "@test.csv" -H "Content-Type: text/csv" -X POST http://localhost:9090/
[ {
"name" : "Pierre Villard",
"company" : "Google"
} ]
View oozie.xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.2">
<description></description>
<groupId>2e083aef-0162-1000-80bc-8c6bbfc00ca8</groupId>
<name>oozie</name>
<snippet>
<processGroups>
<id>5a720561-d2b1-3f86-0000-000000000000</id>
<parentGroupId>16e377d4-8369-3464-0000-000000000000</parentGroupId>
<position>
View testKafka.xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.2">
<description></description>
<groupId>2e083aef-0162-1000-80bc-8c6bbfc00ca8</groupId>
<name>testKafka</name>
<snippet>
<processGroups>
<id>db42a463-039a-32b1-0000-000000000000</id>
<parentGroupId>16e377d4-8369-3464-0000-000000000000</parentGroupId>
<position>
@pvillard31
pvillard31 / MDD.xml
Created Aug 29, 2018
Template for Monitoring Driven Development in NiFi
View MDD.xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.2">
<description></description>
<groupId>8927f4c0-0160-1000-597a-ea764ccd81a7</groupId>
<name>MDD</name>
<snippet>
<connections>
<id>a2098494-cce9-3fa4-0000-000000000000</id>
<parentGroupId>a8352767-434f-3321-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
View workflow-xml-reader-writer-fork.xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.2">
<description></description>
<groupId>01621000-4fe2-1d05-8b0e-7111cb416310</groupId>
<name>post-xml-reader-writer-fork</name>
<snippet>
<connections>
<id>13c3b3c1-3610-3a1c-0000-000000000000</id>
<parentGroupId>8f7c4f43-5305-36c8-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
View avro-schema.json
{
"type" : "record",
"name" : "customers",
"namespace" : "customer",
"fields" : [ {
"name" : "customer_id",
"type" : "long"
}, {
"name" : "name",
"type" : "string"
You can’t perform that action at this time.