Skip to content

Instantly share code, notes, and snippets.

@71
Created September 12, 2021 10:21
Show Gist options
  • Save 71/bd2b4b3ef764773dd1d690c344eafe8a to your computer and use it in GitHub Desktop.
Save 71/bd2b4b3ef764773dd1d690c344eafe8a to your computer and use it in GitHub Desktop.
module Main
open FSharp.Data
open FSharp.Data.JsonExtensions
open Google.Cloud.Functions.Framework
open System
open System.Globalization
open System.Net
open System.Text.RegularExpressions
module Notion =
let private token = "secret_"
let private feedsTableId = ""
let private feedItemsTableId = ""
let private http httpMethod data path = async {
let data = match data with
| Some data -> Some <| TextRequest data
| None -> None
let! contents =
async {
try
return! Http.AsyncRequestString(
sprintf "https://api.notion.com/v1/%s" path,
httpMethod=httpMethod,
headers=[
"Authorization" , sprintf "Bearer %s" token
"Notion-Version" , "2021-05-13"
"Content-Type" , "application/json"
],
?body=data)
with
| :? WebException as e when e.Status = WebExceptionStatus.ProtocolError && (e.Response :?> HttpWebResponse).StatusCode = HttpStatusCode.Conflict ->
return "{}"
}
return JsonValue.Parse(contents)
}
let private get = http "GET" None
let private post path data = http "POST" (Some data) path
let private patch path data = http "PATCH" (Some data) path
type Feed =
{ Id : string
Url : string
ReleaseWindow : DateTimeOffset * DateTimeOffset
IntervalDays : int
RegExp : (Regex * string) option
ItemsCount : int }
let private reRe = Regex @"^/(.+)/(.+?)/$"
let private dateFormat = "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fffK"
let private toDateFromNotion s = DateTimeOffset.ParseExact(s, dateFormat, CultureInfo.InvariantCulture)
let private toDate s = DateTimeOffset.Parse(s, CultureInfo.InvariantCulture)
let private toString (d : DateTimeOffset) = d.ToString(dateFormat)
let getFeeds () = async {
let data =
sprintf """{
"filter": {
"property": "Release window",
"date": {
"on_or_before": "%s"
}
}
}""" (DateTime.Now.ToString("yyyy-MM-dd"))
let! json = post (sprintf "databases/%s/query" feedsTableId) data
return [
for result in json.["results"] do
let properties = result.["properties"]
let startDate =
properties.["Release window"].["date"].["start"].AsString() |> toDateFromNotion
let endDate =
match properties.["Release window"].["date"].["end"] with
| JsonValue.String endDate -> toDateFromNotion endDate
| _ -> startDate
yield {
Id = result.["id"].AsString()
Url = properties.["URL"].["url"].AsString()
ReleaseWindow = (startDate, endDate)
IntervalDays = properties.["Interval (days)"].["number"].AsInteger()
RegExp = [|
for rt in properties.["RegExp"].["rich_text"].AsArray() ->
rt.["text"].["content"].AsString()
|] |> String.concat ""
|> reRe.Match
|> function
| m when m.Success -> Some (Regex m.Groups.[1].Value, m.Groups.[2].Value)
| _ -> None
ItemsCount = 0
}
]
}
let updateFeeds feeds =
feeds
|> Seq.filter (fun feed ->
feed.ItemsCount > 0 || snd feed.ReleaseWindow < DateTimeOffset.Now
)
|> Seq.map (fun feed ->
let mutable feed = feed
while fst feed.ReleaseWindow < DateTimeOffset.Now do
let s, e = feed.ReleaseWindow
let s, e = s.AddDays(float feed.IntervalDays), e.AddDays(float feed.IntervalDays)
feed <- { feed with ReleaseWindow = (s, e) }
let startDate = "\"" + toString (fst feed.ReleaseWindow) + "\""
let endDate =
if fst feed.ReleaseWindow = snd feed.ReleaseWindow then
"null"
else
"\"" + toString (snd feed.ReleaseWindow) + "\""
patch (sprintf "pages/%s" feed.Id) (sprintf """{
"properties": {
"Release window": {
"id": "cNh^",
"type": "date",
"date": {
"start": %s,
"end": %s
}
}
}
}""" startDate endDate)
)
|> Async.Parallel
|> Async.Ignore
type FeedItem =
{ Feed : Feed
Text : string
Link : string
Date : DateTimeOffset }
let private regex pattern = Regex(pattern, RegexOptions.ECMAScript)
let private atomRe =
regex @"<entry>[\s\S]*?<title>(.+?)</title>[\s\S]*?<link.+?href=""(.+?)"".+?/>[\s\S]*?<updated>(.+?)</updated>"
let private rssRe =
regex @"<item>[\s\S]*?<title>(.+?)</title>[\s\S]*?<link>(.+?)</link>[\s\S]*?<pubDate>(.+?)</pubDate>"
let addFeedItems feeds =
feeds
|> List.filter (fun feed -> fst feed.ReleaseWindow <= DateTimeOffset.Now)
|> List.map (fun feed -> async {
let! doc = Http.AsyncRequestString feed.Url
printfn "Fetched document for feed %A" feed
let feedOffset = (fst feed.ReleaseWindow).Offset
let lastEnd = (snd feed.ReleaseWindow).AddDays(-float feed.IntervalDays)
let re = if doc.StartsWith("<feed") then atomRe else rssRe
let items = seq {
for m in re.Matches(doc) do
let text = m.Groups.[1].Value
let link = m.Groups.[2].Value
let date = (toDate m.Groups.[3].Value).ToOffset(feedOffset)
if date > lastEnd then
yield { Feed = feed; Text = text; Link = link; Date = date }
}
let items =
match feed.RegExp with
| Some (re, repl) ->
seq {
for item in items do
let repl = re.Replace(item.Text, repl)
if repl <> item.Text then
yield { item with Text = repl }
}
| None -> items
let escape (s: string) = s.Replace("\"", "\\\"").Replace("\n", "\\n")
let! results =
items
|> Seq.map (fun item ->
printfn "Add item %A" item
post "pages" (sprintf """{
"parent": { "database_id": "%s" },
"properties": {
"Title": { "title": [{ "text": { "content": "%s" } }] },
"URL": { "url": "%s" },
"Feed": { "relation": [{ "id": "%s" }] },
"Release time": { "date": { "start": "%s" } }
}
}""" feedItemsTableId (escape item.Text) (escape item.Link) item.Feed.Id (toString item.Date))
)
|> Async.Parallel
return { feed with ItemsCount = Seq.length results }
})
|> Async.Parallel
let updateAllFeeds () = async {
let! feeds = getFeeds()
do printfn "Fetched %d feeds." feeds.Length
let! feeds = addFeedItems feeds
do printfn "Added feed items."
do! updateFeeds feeds
do printfn "Updated feeds."
}
type Function() =
interface ICloudEventFunction with
member this.HandleAsync(_, _) = Notion.updateAllFeeds() |> Async.StartAsTask :> _
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<Compile Include="UpdateNotionFeeds.fs" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Google.Cloud.Functions.Hosting" Version="1.0.0" />
<PackageReference Include="FSharp.Data" Version="4.2.0" />
</ItemGroup>
</Project>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment