CurtHagenlocher

using Apache.Arrow;
using Apache.Arrow.Types;
using Microsoft.Data.SqlClient;
using System.Data.Common;
using System.Data.SqlTypes;

namespace Loader
{
    internal class Program
    {

CurtHagenlocher

section Section1;

// Does a short search on Wikipedia. This is meant to stand in for a function that returns the navigation data.
GetWikipediaSearch = (term) => Json.Document(Web.Contents("https://en.wikipedia.org/w/api.php", [Query=[
    action="query",
    origin="*",
    format="json",
    generator="search",
    gsrnamespace="0",
    gsrlimit="5",

CurtHagenlocher

section AzStorage;

[DataSource.Kind="TableStorage"]
shared TableStorage.Contents = (url) =>
        Text.FromBinary(Web.Contents(url, [Headers=SignRequest(url)]));

[DataSource.Kind="TableStorage"]
shared FileStorage.Contents = (url) =>
        Text.FromBinary(Web.Contents(url, [Headers=SignRequestBlob(url)]));

CurtHagenlocher

(tableType as type, fn as function) =>
let
    TransformNode = (x) => Record.FromList(List.Transform(Record.FieldValues(x), TransformValue), Record.FieldNames(x)),
    TransformValue = (x) => if x is record then TransformRecord(x) else if x is list then TransformList(x) else x,
    TransformList = (x) => List.Transform(x, TransformValue),
    TransformRecord = (x) => 
        if x = RowExpression.Row then [Kind="Row"]
        else if x[Kind] = "Constant" then TransformConstant(x)
        else TransformNode(x),
    TransformConstant = (x) => if x[Value] is function and TryFunctionName(x[Value]) <> null then [Kind="Identifier", Name=TryFunctionName(x[Value])]

CurtHagenlocher

section Section1;

Detect1000Languages = (input as list) as list =>
    let
        // TODO: support nulls, truncate text to avoid service limits
        text = List.Buffer(input),
        data = Table.FromColumns({text}, type table [text=text]),
        indexed = Table.AddIndexColumn(data, "id", 1),
        textId = Table.TransformColumnTypes(indexed, {{"id", type text}}),
        body = [documents=Table.ReorderColumns(textId, {"id", "text"})],

CurtHagenlocher

let
    Value.FixType = (value, optional depth) =>
        let
            nextDepth = if depth = null then 3 else depth - 1,
            result = if depth = 0 then null
                else if value is type then TextType(value)
                else if value is table then Table.TransformColumns(value, {}, @Value.FixType)
                else if value is list then List.Transform(value, each @Value.FixType(_, nextDepth))
                else if value is record then
                    Record.FromList(List.Transform(Record.ToList(value), each @Value.FixType(_, nextDepth)), Record.FieldNames(value))

CurtHagenlocher

let
    Web.MultiPartPost = (url as text, parts as record) as binary =>
        let
            unique = Text.NewGuid(),
            boundary = "--" & unique,
            crlf = "#(cr)#(lf)",
            item = (name, value) =>
                let
                    filename = Value.Metadata(value)[name]?,
                    contentType = Value.Metadata(value)[type]?,

CurtHagenlocher

let
    ByteToHex = (i as number) as text =>
        let
            chars = "0123456789abcdef",
            low = Text.Range(chars, Number.Mod(i, 16), 1),
            high = Text.Range(chars, Number.RoundDown(i / 16), 1)
        in high & low,
    Json.EscapeChar = (text as text) as text =>
        if text = """" or text = "\" or text = "/" then "\" & text
        else if Character.ToNumber(text) < 32 then "\u00" & ByteToHex(Character.ToNumber(text))

CurtHagenlocher

let
    EnforceTypes = (table as table, schema as table) as table =>
        let
            map = (t) => if t = type list or t = type record or t = type any then null else t,
            mapped = Table.TransformColumns(schema, {"Value", map}),
            omitted = Table.SelectRows(mapped, each [Value] <> null),
            transforms = Table.ToRows(omitted),
            changedType = Table.TransformColumnTypes(table, transforms)
        in
            changedType,

CurtHagenlocher

let
    data = Table.FromColumns({{1, 2, 3, 4, 5}, {"Left", "Right", "Center", "Left", "Right"}}),
    filter = Table.FromColumns({{"Right", "Center"}}),
    filterValues = filter[Column1],
    filtered = Table.SelectRows(data, each List.Contains(filterValues, [Column2]))
in
    filtered