Skip to content

Instantly share code, notes, and snippets.

@ricston-git
Last active August 28, 2018 12:24
Show Gist options
  • Save ricston-git/331f8afa39a05c125e88c528723472db to your computer and use it in GitHub Desktop.
Save ricston-git/331f8afa39a05c125e88c528723472db to your computer and use it in GitHub Desktop.
Gist for "Use cases for the readUrl function in DataWeave" blog-post
---
{
functionsOnObjects: {
getUniqueKeys: getUniqueKeys,
getKeys: getKeys
},
functionsOnStrings: {
substring: substringFunc
},
functionsOnArrays: {
tail: tail,
subArray: subArray
}
}
name alpha-2 alpha-3 country-code iso_3166-2 region sub-region region-code sub-region-code
Afghanistan AF AFG 004 ISO 3166-2:AF Asia Southern Asia 142 034
Åland Islands AX ALA 248 ISO 3166-2:AX Europe Northern Europe 150 154
Albania AL ALB 008 ISO 3166-2:AL Europe Southern Europe 150 039
Algeria DZ DZA 012 ISO 3166-2:DZ Africa Northern Africa 002 015
American Samoa AS ASM 016 ISO 3166-2:AS Oceania Polynesia 009 061
Andorra AD AND 020 ISO 3166-2:AD Europe Southern Europe 150 039
. . . . . . . . .
. . . . . . . . .
. . . . . . . . .
Wallis and Futuna WF WLF 876 ISO 3166-2:WF Oceania Polynesia 009 061
Western Sahara EH ESH 732 ISO 3166-2:EH Africa Northern Africa 002 015
Yemen YE YEM 887 ISO 3166-2:YE Asia Western Asia 142 145
Zambia ZM ZMB 894 ISO 3166-2:ZM Africa Eastern Africa 002 014
Zimbabwe ZW ZWE 716 ISO 3166-2:ZW Africa Eastern Africa 002 014
%dw 1.0
%output application/json
%var countries = readUrl("classpath://countries.csv", "application/csv")
%var countryDetails = (countries filter $.name == payload.country)[0]
---
country: {
name : payload.country,
code : countryDetails.alpha-3,
region: countryDetails.sub-region
}
{"country":
{
"name": "Andorra",
"code": "AND",
"region": "Southern Europe"
}
}
%dw 1.0
%output application/json
%var myLibrary = readUrl("classpath://library.dwl","application/dw")
---
{
sub: myLibrary.substring(payload.myString,3,6),
keys: myLibrary.getUniqueKeys(payload.myObject)
}
<flow name="etl-flow">
<file:inbound-endpoint doc:name="Extract Entity File" path="/path/to/files_to_extract" responseTimeout="10000"/>
<choice doc:name="Choice">
<when expression="#[payload.type == &quot;Person&quot;]">
<dw:transform-message doc:name="Transform Person File">
<dw:set-payload resource="classpath:Person.dwl"/>
</dw:transform-message>
</when>
<when expression="#[payload.type == &quot;Location&quot;]">
<dw:transform-message doc:name="Transform Location File">
<dw:set-payload resource="classpath:Location.dwl"/>
</dw:transform-message>
</when>
<when expression="#[payload.type == &quot;Product&quot;]">
<dw:transform-message doc:name="Transform Product File">
<dw:set-payload resource="classpath:Product.dwl"/>
</dw:transform-message>
</when>
<otherwise>
<logger level="INFO" doc:name="Logger"/>
</otherwise>
</choice>
<flow-ref doc:name="Load Entity Subflow" name="load-entity-subflow"/>
</flow>
<flow name="etl-flow">
<file:inbound-endpoint doc:name="Extract Entity File" path="/path/to/files_to_extract" responseTimeout="10000"/>
<dw:transform-message doc:name="Transform Entity File">
<dw:set-payload><![CDATA[
%dw 1.0
%output application/java
%var transform = readUrl("classpath://" ++ payload.type ++ ".dwl", "application/dw")
---
transform(payload)
]]></dw:set-payload>
</dw:transform-message>
<flow-ref doc:name="Load Entity Subflow" name="load-entity-subflow"/>
</flow>
{
"myString": "Hello World!",
"myObject": {"a": 1, "b": 2, "a": 3, "c": 4}
}
%dw 1.0
%output application/dw
//I suggest using application/dw as the output type, so that any possible output will be supported.
//Anyhow, the output MIME type used by the callers of this library will take precedence.
%function getUniqueKeys(obj)
getKeys(obj) distinctBy $
%function getKeys(obj)
obj pluck $$
%function tail(arr)
[] when (sizeOf arr) <= 1 otherwise arr[1..-1]
%function substringFunc(str,i,j)
str splitBy "" filter ($$>=i and $$<=j) joinBy ""
%function subArray(arr,i,j)
arr filter ($$>=posi and $$<=posj)
---
//Mapping the function names to the functions themselves.
//The exposed function name can be different from the actual function name,
//which is the case for the function substringFunc, whose exposed name is substring.
//Also, note that not all functions may be exposed, such as the getKeys function in this case.
{
getUniqueKeys: getUniqueKeys,
tail: tail,
substring: substringFunc,
subArray: subArray
}
{
"sub": "lo W",
"keys": ["a","b","c"]
}
%dw 1.0
%input payload application/java
%output application/dw
---
read(payload, "application/csv")
%dw 1.0
%output application/json
%var data = payload.data
---
payload.transformations mapObject
{($$) : read(data ++ " " ++ $)}
"name,surname,id\nJohn,Smith,574323\nMary,Cooper,729852"
{
"data": "[2,6,3,6,1]",
"transformations":
{
"plusOne": "map \$+1",
"removeDuplicatesAndSort": "distinctBy \$ orderBy \$",
"filterOutOddIndexes": "filter (\$\$ mod 2) == 0"
}
}
[
{
"name": "John",
"surname": "Smith",
"id": "574323"
},
{
"name": "Mary",
"surname": "Cooper",
"id": "729852"
}
]
{
"plusOne": [3,7,4,7,2],
"removeDuplicatesAndSort": [1,2,3,6],
"filterOutOddIndexes": [2,3,1]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment