Skip to content

Instantly share code, notes, and snippets.

@cbattlegear
Created October 28, 2020 14:28
Show Gist options
  • Save cbattlegear/31861dfa1120a48ffbb42ce010008e43 to your computer and use it in GitHub Desktop.
Save cbattlegear/31861dfa1120a48ffbb42ce010008e43 to your computer and use it in GitHub Desktop.
Simple API proxy to allow Azure Data Factory REST API connector to use Link header pagination. It makes some assumptions about API structure so verify if works with your use case.
#r "Newtonsoft.Json"
using System.Net;
using System.Net.Http;
using System.Web;
using System.Text.RegularExpressions;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Primitives;
using Newtonsoft.Json;
/*********************************************************************/
/* Simple API proxy to allow Azure Data Factory REST API connector */
/* to use Link header pagination. It makes some assumptions about */
/* API structure so verify if works with your use case. */
/* */
/* Make sure to edit the api_url variable and the auth headers if */
/* you need them. */
/*********************************************************************/
public static async Task<IActionResult> Run(HttpRequest req, ILogger log)
{
log.LogInformation("C# HTTP trigger function Linker Header Proxy");
//Change this to the URL for the API you want to call
string api_url = "https://api.github.com/search/code?q=http+user:cbattlegear";
string next_page = "";
string apiResponse = "";
string page = req.Query["page"];
using (var httpClient = new HttpClient())
{
//If you need authentication to the API please uncomment and edit the header line below
//httpClient.DefaultRequestHeaders.Add("HeaderName", "HeaderValueProbablyYourKey");
httpClient.DefaultRequestHeaders.Add("User-Agent", "Azure Function - Link Header Proxy");
if(string.IsNullOrEmpty(page)) {
using (var response = await httpClient.GetAsync(api_url))
{
if((int)response.StatusCode != 200) {
log.LogWarning("Source API Threw status code " + response.StatusCode);
return new StatusCodeResult((int)response.StatusCode);
}
apiResponse = await response.Content.ReadAsStringAsync();
string link_header = response.Headers.GetValues("link").FirstOrDefault();
next_page = GetNextPageFromLinkHeader(link_header);
}
} else {
using (var response = await httpClient.GetAsync(api_url + "&page=" + page))
{
if((int)response.StatusCode != 200) {
log.LogWarning("Source API Threw status code " + response.StatusCode);
return new StatusCodeResult((int)response.StatusCode);
}
apiResponse = await response.Content.ReadAsStringAsync();
string link_header = response.Headers.GetValues("link").FirstOrDefault();
next_page = GetNextPageFromLinkHeader(link_header);
}
}
}
string responseMessage = next_page;
string full_url = "https://" + req.Host.ToString() + req.Path.ToString();
if(!string.IsNullOrEmpty(next_page))
{
req.HttpContext.Response.Headers.Add("X-next-page", full_url + "?page=" + next_page);
}
var returned = new ContentResult();
returned.Content = apiResponse;
returned.ContentType = "application/json";
return returned;
}
//Modified from https://gist.github.com/pimbrouwers/8f78e318ccfefff18f518a483997be29
public static string GetNextPageFromLinkHeader(string header) {
string next_page = "";
string next_link = "";
if (!string.IsNullOrWhiteSpace(header))
{
string[] linkStrings = header.Split(',');
if (linkStrings != null && linkStrings.Any())
{
foreach (string linkString in linkStrings)
{
var relMatch = Regex.Match(linkString, "(?<=rel=\").+?(?=\")", RegexOptions.IgnoreCase);
var linkMatch = Regex.Match(linkString, "(?<=<).+?(?=>)", RegexOptions.IgnoreCase);
if (relMatch.Success && linkMatch.Success)
{
string rel = relMatch.Value.ToUpper();
string link = linkMatch.Value;
switch (rel)
{
case "FIRST":
break;
case "PREV":
break;
case "NEXT":
next_link = link;
break;
case "LAST":
break;
}
}
}
}
}
int iqs = next_link.IndexOf('?');
string querystring = "";
if (iqs >= 0)
{
querystring = (iqs < next_link.Length - 1) ? next_link.Substring(iqs + 1) : String.Empty;
}
next_page = HttpUtility.ParseQueryString(querystring)["page"];
return next_page;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment