Skip to content

Instantly share code, notes, and snippets.

@YannickRe
Last active June 14, 2021 19:37
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save YannickRe/936d9d1a620421940ef99f69dc66c658 to your computer and use it in GitHub Desktop.
Save YannickRe/936d9d1a620421940ef99f69dc66c658 to your computer and use it in GitHub Desktop.
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<AzureFunctionsVersion>v2</AzureFunctionsVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.Search" Version="5.0.2" />
<PackageReference Include="morelinq" Version="3.0.0" />
</ItemGroup>
</Project>
#r "Newtonsoft.Json"
#r "Microsoft.WindowsAzure.Storage"
using System.Net;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Primitives;
using Newtonsoft.Json;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.Azure.Search;
using Microsoft.Azure.Search.Models;
using MoreLinq.Extensions;
private static HttpClient client;
private static readonly string BlogUrl = "";
private static readonly string ContentApiKey = "";
private static readonly string SearchServiceName = "";
private static readonly string SearchAdminApiKey = "";
public static async Task<IActionResult> Run(HttpRequest req, ILogger log)
{
log.LogInformation("Loading posts from blog");
var client = GetClient();
var response = await client.GetAsync($"{BlogUrl}/ghost/api/v2/content/posts/?include=tags,authors&formats=plaintext&limit=15&key={ContentApiKey}");
var json = await response.Content.ReadAsStringAsync();
dynamic data = JsonConvert.DeserializeObject(json);
List<dynamic> postList = data.posts.ToObject<List<dynamic>>();
log.LogInformation("Converting posts into the correct format");
var postsArray = postList.ConvertAll((p) => {
List<dynamic> tags = p.tags.ToObject<List<dynamic>>();
return new Post() {
Id = p.id,
Uuid = p.uuid,
Title = p.title,
Slug = p.slug,
PlainText = p.plaintext,
FeatureImage = p.feature_image,
CustomExcerpt = p.custom_excerpt,
Page = p.page,
Url = p.url,
CreatedAt = p.created_at,
UpdatedAt = p.updated_at,
PublishedAt = p.published_at,
Tags = tags.Select(t => t.slug.ToString()).Cast<string>().ToArray()
};
});
log.LogInformation("Creating search service client");
var searchClient = new SearchServiceClient(SearchServiceName, new SearchCredentials(SearchAdminApiKey));
log.LogInformation("Deleting Index");
await searchClient.Indexes.DeleteAsync("posts");
log.LogInformation("Creating Index");
var definition = new Index()
{
Name = "posts",
Fields = FieldBuilder.BuildForType<Post>(),
CorsOptions = new CorsOptions()
{
AllowedOrigins = new List<string>() { BlogUrl },
MaxAgeInSeconds = 300
}
};
await searchClient.Indexes.CreateAsync(definition);
log.LogInformation("BatchInsert Posts into Index");
var indexClient = searchClient.Indexes.GetClient("posts");
foreach(var objectBatch in postsArray.Batch(1000))
{
try
{
IndexBatch<Post> batch = IndexBatch.MergeOrUpload(objectBatch);
await indexClient.Documents.IndexAsync(batch);
}
catch (IndexBatchException e)
{
log.LogError(e, $"Failed to index some of the groups: {string.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key))}");
}
}
return(ActionResult)new OkResult();
}
private static HttpClient GetClient()
{
client = client ?? new HttpClient();
return client;
}
[SerializePropertyNamesAsCamelCase]
public class Post
{
[System.ComponentModel.DataAnnotations.Key]
[IsRetrievable(true)]
public string Id { get; set; }
[IsSearchable, IsRetrievable(true)]
public string Uuid { get; set; }
[IsSearchable, IsRetrievable(true)]
public string Title { get; set; }
[IsSearchable, IsRetrievable(true)]
public string Slug { get; set; }
[IsSearchable, IsRetrievable(true)]
public string PlainText { get; set; }
[IsSearchable, IsRetrievable(true)]
public string FeatureImage { get; set; }
[IsSearchable, IsRetrievable(true)]
public string CustomExcerpt { get; set; }
[IsSearchable, IsRetrievable(true)]
public string[] Tags { get; set; }
[IsRetrievable(true)]
public string Url { get; set; }
[IsFilterable, IsRetrievable(true)]
public bool? Page { get; set; }
[IsRetrievable(true)]
public DateTime? CreatedAt { get; set; }
[IsRetrievable(true)]
public DateTime? UpdatedAt { get; set; }
[IsRetrievable(true)]
public DateTime? PublishedAt { get; set; }
}
@SimonDarksideJ
Copy link

Updated the function script to use the newer Azure.Cognitive.Search client V11, which replaces the above V10 Azure.Search client
https://gist.github.com/SimonDarksideJ/6c4099900f1c66383a2ac00a1a322dfe

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment