Navigation Menu

Skip to content

Instantly share code, notes, and snippets.

View CurtHagenlocher's full-sized avatar

Curt Hagenlocher CurtHagenlocher

  • Microsoft
  • Mercer Island, WA
View GitHub Profile
@CurtHagenlocher
CurtHagenlocher / SplitByColumnHeader.m
Created April 7, 2015 14:44
This sample uses a header consisting of dashes to identify where the column boundaries are and then splits the file by those boundaries.
let
Source = Table.FromColumns({Lines.FromBinary(Web.Contents("http://www.aer.ca/data/WELLS/SPUDTHU.TXT"),null,null,1252)}),
Header = List.First(List.Select(Source[Column1], each Text.StartsWith(_, "--"))),
Positions = Text.PositionOf(Header, " ", -1),
Length = Text.Length(Header),
Lengths = List.Generate(
() => 0,
(i) => i <= List.Count(Positions),
(i) => i + 1,
(i) =>
let
Source = Text.FromBinary(File.Contents("D:\testdata\tweet.json")),
Eol = Text.PositionOf(Source, "#(lf)"),
Skipped = Text.Range(Source, Eol + 1),
Tweets = Json.Document(Skipped)
in
Tweets
@CurtHagenlocher
CurtHagenlocher / FilterByValues.m
Created May 14, 2015 21:00
Filter by values from another table
let
data = Table.FromColumns({{1, 2, 3, 4, 5}, {"Left", "Right", "Center", "Left", "Right"}}),
filter = Table.FromColumns({{"Right", "Center"}}),
filterValues = filter[Column1],
filtered = Table.SelectRows(data, each List.Contains(filterValues, [Column2]))
in
filtered
@CurtHagenlocher
CurtHagenlocher / gist:1917426
Created February 26, 2012 15:41
UrlBuilder.cs
public class UrlBuilder : IDynamicMetaObjectProvider
{
/// <summary>
/// Usage: UrlBuilder.Build(url, key1: value1, key2: value2)
/// </summary>
/// <remarks>
/// values will be converted to strings via "ToString()". Any url-encoding must happen before
/// this method is called. Pass null as the value at runtime in order to omit the parameter
/// entirely. Use String.Empty for query parameters which should be present but have no value.
/// If url is a string, the result will be a string. If it is a System.Uri, the result will
@CurtHagenlocher
CurtHagenlocher / gist:1917432
Created February 26, 2012 15:49
StringFormat.cs
public class StringFormat : IDynamicMetaObjectProvider
{
/// <summary>
/// Usage: StringFormat.Format(formatString, key1: value1, key2: value2)
/// or StringFormat.Format(formatProvider, formatString, key1: value1, key2: value2)
/// </summary>
public static readonly dynamic Format = new StringFormat();
private StringFormat()
{
@CurtHagenlocher
CurtHagenlocher / json.m
Created July 8, 2015 22:52
Format M data as JSON
let
ByteToHex = (i as number) as text =>
let
chars = "0123456789abcdef",
low = Text.Range(chars, Number.Mod(i, 16), 1),
high = Text.Range(chars, Number.RoundDown(i / 16), 1)
in high & low,
Json.EscapeChar = (text as text) as text =>
if text = """" or text = "\" or text = "/" then "\" & text
else if Character.ToNumber(text) < 32 then "\u00" & ByteToHex(Character.ToNumber(text))
@CurtHagenlocher
CurtHagenlocher / EnforceSchema.m
Created July 8, 2015 15:57
Demonstrates some mechanisms in M for applying or enforcing a predefined schema onto a table.
let
EnforceTypes = (table as table, schema as table) as table =>
let
map = (t) => if t = type list or t = type record or t = type any then null else t,
mapped = Table.TransformColumns(schema, {"Value", map}),
omitted = Table.SelectRows(mapped, each [Value] <> null),
transforms = Table.ToRows(omitted),
changedType = Table.TransformColumnTypes(table, transforms)
in
changedType,
@CurtHagenlocher
CurtHagenlocher / AzureTable.pq
Created August 13, 2019 14:20
Work-in-progress Azure Table Storage
section AzStorage;
[DataSource.Kind="TableStorage"]
shared TableStorage.Contents = (url) =>
Text.FromBinary(Web.Contents(url, [Headers=SignRequest(url)]));
[DataSource.Kind="TableStorage"]
shared FileStorage.Contents = (url) =>
Text.FromBinary(Web.Contents(url, [Headers=SignRequestBlob(url)]));
@CurtHagenlocher
CurtHagenlocher / RowExpression.From.pq
Created March 6, 2018 23:26
RowExpression.From as JSON
let
Value.FixType = (value, optional depth) =>
let
nextDepth = if depth = null then 3 else depth - 1,
result = if depth = 0 then null
else if value is type then TextType(value)
else if value is table then Table.TransformColumns(value, {}, @Value.FixType)
else if value is list then List.Transform(value, each @Value.FixType(_, nextDepth))
else if value is record then
Record.FromList(List.Transform(Record.ToList(value), each @Value.FixType(_, nextDepth)), Record.FieldNames(value))
section Section1;
Detect1000Languages = (input as list) as list =>
let
// TODO: support nulls, truncate text to avoid service limits
text = List.Buffer(input),
data = Table.FromColumns({text}, type table [text=text]),
indexed = Table.AddIndexColumn(data, "id", 1),
textId = Table.TransformColumnTypes(indexed, {{"id", type text}}),
body = [documents=Table.ReorderColumns(textId, {"id", "text"})],