Skip to content

Instantly share code, notes, and snippets.

@igeligel
Last active November 25, 2019 11:24
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save igeligel/c3a72f8bd16984368584e280ac85d5cf to your computer and use it in GitHub Desktop.
Save igeligel/c3a72f8bd16984368584e280ac85d5cf to your computer and use it in GitHub Desktop.
public class DynamicFrameDecodeBase64 {
public static DynamicFrame unBase64(
DynamicFrame df,
final String base64ColumnName,
final String dataColumnName
) {
RDD unboxRecords = df
.records()
.mapPartitions(
new MapFunctionUnbase64(base64ColumnName, dataColumnName),
true,
scala.reflect.ClassTag$.MODULE$.apply(DynamicRecord.class)
);
return DynamicFrame.newFrameWithErrors(
df,
unboxRecords,
df.getName(),
"",
null,
0,
0
);
}
}
public class MapFunctionUnbase64
extends scala.runtime.AbstractFunction1<scala.collection.Iterator<DynamicRecord>, scala.collection.Iterator<DynamicRecord>>
implements scala.Serializable {
public final scala.collection.Iterator<DynamicRecord> apply(
scala.collection.Iterator<DynamicRecord> iter
) {
Base64.Decoder base64decoder = Base64.getDecoder();
return iter.map(
new scala.runtime.AbstractFunction1<com.amazonaws.services.glue.DynamicRecord, DynamicRecord>() {
public final DynamicRecord apply(DynamicRecord record) {
String base64ColumnName = "base64";
String raw = record
.dropField(base64ColumnName)
.get()
.getValue()
.toString();
byte[] decodedString;
String data;
try {
decodedString =
base64decoder.decode(new String(raw).getBytes("UTF-8"));
data = new String(decodedString, StandardCharsets.UTF_8);
} catch (java.io.UnsupportedEncodingException e) {
data = "";
}
record.addField(
"data",
com.amazonaws.services.glue.types.StringNode.apply(data)
);
return record;
}
}
);
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment