Skip to content

Instantly share code, notes, and snippets.

@SimonDarksideJ
Forked from YannickRe/function.proj
Last active June 14, 2021 19:36
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save SimonDarksideJ/6c4099900f1c66383a2ac00a1a322dfe to your computer and use it in GitHub Desktop.
Save SimonDarksideJ/6c4099900f1c66383a2ac00a1a322dfe to your computer and use it in GitHub Desktop.
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<AzureFunctionsVersion>v3</AzureFunctionsVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.Search.Documents" Version="11.3.0" />
<PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="2.15.0" />
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.13" />
<PackageReference Include="morelinq" Version="3.3.2" />
</ItemGroup>
</Project>
using Azure;
using Azure.Core.Serialization;
using Azure.Search.Documents;
using Azure.Search.Documents.Indexes;
using Azure.Search.Documents.Indexes.Models;
using Azure.Search.Documents.Models;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Azure.WebJobs;
using Microsoft.Azure.WebJobs.Extensions.Http;
using Microsoft.Extensions.Logging;
using MoreLinq.Extensions;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading.Tasks;
public static class SearchIndexing
{
private static HttpClient client;
private static readonly string BlogUrl = "https://darkgenesisblogsite.azurewebsites.net/";
private static readonly string ContentApiKey = "b11d8dd52937fc3772065e8a79";
[FunctionName("SearchIndexing")]
public static async Task<IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("Loading posts from blog");
var client = GetClient();
var response = await client.GetAsync($"{BlogUrl}/ghost/api/v2/content/posts/?include=tags,authors&formats=plaintext&limit=15&key={ContentApiKey}");
var json = await response.Content.ReadAsStringAsync();
dynamic data = JsonConvert.DeserializeObject(json);
List<dynamic> postList = data.posts.ToObject<List<dynamic>>();
log.LogInformation("Converting posts into the correct format");
var postsArray = postList.ConvertAll((p) =>
{
List<dynamic> tags = p.tags.ToObject<List<dynamic>>();
return new Post()
{
Id = p.id,
Uuid = p.uuid,
Title = p.title,
Slug = p.slug,
PlainText = p.plaintext,
FeatureImage = p.feature_image,
CustomExcerpt = p.custom_excerpt,
Page = p.page,
Url = p.url,
CreatedAt = p.created_at,
UpdatedAt = p.updated_at,
PublishedAt = p.published_at,
Tags = tags.Select(t => t.slug.ToString()).Cast<string>().ToArray()
};
});
Uri endpoint = new Uri(Environment.GetEnvironmentVariable("SEARCH_ENDPOINT"));
AzureKeyCredential credential = new AzureKeyCredential(Environment.GetEnvironmentVariable("SEARCH_API_KEY"));
// Create serializer options with our converter to deserialize geographic points.
JsonSerializerOptions serializerOptions = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
SearchClientOptions clientOptions = new SearchClientOptions
{
Serializer = new JsonObjectSerializer(serializerOptions)
};
log.LogInformation("Creating search service client");
var searchClient = new SearchIndexClient(endpoint, credential);
log.LogInformation("Deleting Index");
await searchClient.DeleteIndexAsync("posts");
log.LogInformation("Creating Index");
var fieldBuilder = new FieldBuilder();
var definition = new SearchIndex("posts")
{
Fields = fieldBuilder.Build(typeof(Post)),
CorsOptions = new CorsOptions(new List<string>() { BlogUrl })
{
MaxAgeInSeconds = 300
}
};
await searchClient.CreateIndexAsync(definition);
log.LogInformation("BatchInsert Posts into Index");
var indexClient = searchClient.GetSearchClient("posts");
foreach (var objectBatch in postsArray.Batch(1000))
{
try
{
IndexDocumentsBatch<Post> batch = IndexDocumentsBatch.MergeOrUpload(objectBatch);
await indexClient.IndexDocumentsAsync(batch);
}
catch (RequestFailedException e)
{
log.LogError(e, $"Failed to index some of the groups");
}
}
return (ActionResult)new OkResult();
}
private static HttpClient GetClient()
{
client = client ?? new HttpClient();
return client;
}
public class Post
{
[SimpleField(IsKey = true)]
public string Id { get; set; }
[SearchableField(IsSortable = true)]
public string Uuid { get; set; }
[SearchableField()]
public string Title { get; set; }
[SearchableField()]
public string Slug { get; set; }
[SearchableField()]
public string PlainText { get; set; }
[SearchableField()]
public string FeatureImage { get; set; }
[SearchableField()]
public string CustomExcerpt { get; set; }
[SearchableField()]
public string[] Tags { get; set; }
[SimpleField()]
public string Url { get; set; }
[SimpleField(IsFilterable = true)]
public bool? Page { get; set; }
[SimpleField()]
public DateTime? CreatedAt { get; set; }
[SimpleField()]
public DateTime? UpdatedAt { get; set; }
[SimpleField()]
public DateTime? PublishedAt { get; set; }
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment