Skip to content

Instantly share code, notes, and snippets.

{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VisualEditor0",
"Effect": "Deny",
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:ListBucketMultipartUploads",
@rjdudley
rjdudley / GenerateTimestamp.cs
Created September 25, 2018 03:45
U-SQL inline function to generate the timestamp from two fields.
public string GenerateTimestamp(string timestamp, string offset)
{
// timestamp comes from log file as [25/Dec/2017:00:14:58
// offset comes from log file as -0500]
// I need to trim the brackets and change the first : to a T
string stamp, part1, part2;
part1 = timestamp.Replace("[", "");
part2 = offset.Replace("]", "");
@rjdudley
rjdudley / gist:7df48e4792414b6c0b169b6738ba03de
Created September 25, 2018 03:42
First test of log parser in U-SQL
@input =
EXTRACT ip string,
identd string,
remote_user string,
timestamp string,
offset string,
method_file_protocol string,
response int,
size string,
domain string,
@rjdudley
rjdudley / gist:2b96f8a532fc4a9504d9f8ea02b27ef7
Created September 25, 2018 03:27
Boilerplate U-SQL User Defined Extractor
public override IEnumerable<IRow> Extract(IUnstructuredReader input, IUpdatableRow output)
{
string line;
//Read the input line by line
foreach (Stream current in input.Split(_row_delim))
{
using (StreamReader streamReader = new StreamReader(current, this._encoding))
{
line = streamReader.ReadToEnd().Trim();