Skip to content

Instantly share code, notes, and snippets.

View pvillard31's full-sized avatar

Pierre Villard pvillard31

View GitHub Profile
@pvillard31
pvillard31 / DataFlowFunctionsQuickstart.json
Last active September 22, 2022 08:16
DataFlowFunctionsQuickstart.json
{
"flowContents": {
"identifier": "f7ec47e6-9057-30b0-abb7-0b38eb709ef6",
"instanceIdentifier": "a4c2cb58-0182-1000-565f-6ad654e140b8",
"name": "DataFlowFunctionsQuickstart",
"comments": "",
"position": {
"x": 536.0,
"y": 768.0
},
@pvillard31
pvillard31 / template.json
Created September 8, 2022 15:54
Template for deploying a Function App in Azure
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"subscriptionId": {
"type": "string"
},
"name": {
"type": "string"
},
@pvillard31
pvillard31 / syncDataFlow.sh
Created December 24, 2021 19:29
Script to be used with the NiFi Registry Event Script Hook for automatic sync with the Cloudera DataFlow Catalog
#!/bin/sh
set -e
# This script is used to automatically sync new flows into the DataFlow Catalog
# It assumes that the CDP CLI has been installed first. See:
# https://docs.cloudera.com/dataflow/cloud/cli/index.html
# It also assumes that the NiFi CLI has been properly configured to interact with the NiFi Registry instance. See:
# https://nifi.apache.org/docs/nifi-docs/html/toolkit-guide.html#nifi_CLI
NIFICLI="/opt/cloudera/parcels/CFM/TOOLKIT/bin/cli.sh"
<customers>
<customer customer_id="1" name="John Doe">
<addresses>
<address type="home" value="1 Apache NiFi street, Nifiville" />
<address type="work" value="10 MiNiFi avenue, Nifiville" />
</addresses>
<accounts>
<account account_id="1" shortname="perso" balance="1234.56" />
<account account_id="2" shortname="joint" balance="2345.67">
<transactions>
@pvillard31
pvillard31 / MDD.xml
Created August 29, 2018 16:15
Template for Monitoring Driven Development in NiFi
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.2">
<description></description>
<groupId>8927f4c0-0160-1000-597a-ea764ccd81a7</groupId>
<name>MDD</name>
<snippet>
<connections>
<id>a2098494-cce9-3fa4-0000-000000000000</id>
<parentGroupId>a8352767-434f-3321-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
import java.security.SignatureException
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec
def flowFile = session.get()
if (!flowFile) return
def static hmac(String data, String key) throws java.security.SignatureException
{
String result
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.3">
<description></description>
<groupId>2e083aef-0162-1000-80bc-8c6bbfc00ca8</groupId>
<name>FTP-UTF8-Test</name>
<snippet>
<processGroups>
<id>3982c999-cd5f-3ac7-0000-000000000000</id>
<parentGroupId>16e377d4-8369-3464-0000-000000000000</parentGroupId>
<position>
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.io.IOUtils
import java.nio.charset.*
import org.json.JSONObject;
import org.json.XML;
def flowFile = session.get()
if (!flowFile) return
flowFile = session.write(flowFile,
{ inputStream, outputStream ->
@Test
public void testNoMatchingRecords() throws InitializationException {
TestRunner runner = getRunner();
final JsonTreeReader jsonReader = new JsonTreeReader();
runner.addControllerService("reader", jsonReader);
runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaInferenceUtil.INFER_SCHEMA);
final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
runner.addControllerService("writer", jsonWriter);