Skip to content

Instantly share code, notes, and snippets.

@onionhammer
Created May 8, 2023 18:06
Show Gist options
  • Save onionhammer/c019b04a7c4f058cd57183f0df738905 to your computer and use it in GitHub Desktop.
Save onionhammer/c019b04a7c4f058cd57183f0df738905 to your computer and use it in GitHub Desktop.
using System.Diagnostics.CodeAnalysis;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Orchestration;
using Microsoft.SemanticKernel.SemanticFunctions;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace Microsoft.SemanticKernel.KernelExtensions;
public static class MarkdownLoaderExtensions
{
private static readonly Regex s_asciiLettersDigitsUnderscoresRegex = new("^[0-9A-Za-z_]*$");
private static readonly IDeserializer yamlDeserializer;
static MarkdownLoaderExtensions()
{
yamlDeserializer = new DeserializerBuilder()
.WithNamingConvention(UnderscoredNamingConvention.Instance)
.Build();
}
internal static void ValidSkillName(string skillName)
{
if (!s_asciiLettersDigitsUnderscoresRegex.IsMatch(skillName))
{
ThrowInvalidName("skill name", skillName);
}
}
[DoesNotReturn]
private static void ThrowInvalidName(string kind, string name) =>
throw new KernelException(
KernelException.ErrorCodes.InvalidFunctionDescription,
$"A {kind} can contain only ASCII letters, digits, and underscores: '{name}' is not a valid name.");
/// <summary>
/// A kernel extension that allows to load Semantic Functions, defined by prompt templates stored in the filesystem.
/// A skill directory contains a set of subdirectories, one for each semantic function.
/// This extension requires the path of the parent directory (e.g. "d:\skills") and the name of the skill directory
/// (e.g. "OfficeSkill"), which is used also as the "skill name" in the internal skill collection.
///
/// Note: skill and function names can contain only alphanumeric chars and underscore.
///
/// Example:
/// D:\skills\ # parentDirectory = "D:\skills"
///
/// |__ OfficeSkill\ # skillDirectoryName = "SummarizeEmailThread"
///
/// |__ ScheduleMeeting.md # semantic function
/// |__ SummarizeEmailThread.md # semantic function
/// |__ MergeWordAndExcelDocs.md # semantic function
///
/// |__ XboxSkill\ # another skill, etc.
///
/// |__ MessageFriend.md
/// |__ LaunchGame.md
/// </summary>
/// <param name="kernel">Semantic Kernel instance</param>
/// <param name="parentDirectory">Directory containing the skill directory, e.g. "d:\myAppSkills"</param>
/// <param name="skillDirectoryNames">Name of the directories containing the selected skills, e.g. "StrategySkill"</param>
/// <returns>A list of all the semantic functions found in the directory, indexed by function name.</returns>
public static IDictionary<string, ISKFunction> ImportSemanticSkillFromMarkdown(
this IKernel kernel, string parentDirectory, params string[] skillDirectoryNames
)
{
var skill = new Dictionary<string, ISKFunction>();
foreach (string skillDirectoryName in skillDirectoryNames)
{
ValidSkillName(skillDirectoryName);
var skillDir = Path.Combine(parentDirectory, skillDirectoryName);
if (!Directory.Exists(skillDir))
throw new DirectoryNotFoundException($"Directory '{skillDir}' could not be found.");
var mdFiles = Directory.GetFiles(skillDir, "*.md", SearchOption.TopDirectoryOnly);
foreach (string file in mdFiles)
{
var functionName = Path.GetFileNameWithoutExtension(file);
// Parse the markdown file
var (frontMatter, prompt) = SplitMarkdownFile(file);
// Parse the frontmatter, if any
var config = frontMatter != null ?
TemplateConfigFromYamlText(frontMatter) :
new();
// Create the semantic function
var template = new PromptTemplate(prompt, config, kernel.PromptTemplateEngine);
var functionConfig = new SemanticFunctionConfig(config, template);
kernel.Log.LogTrace("Registering function {0}.{1} loaded from {2}", skillDirectoryName, functionName, file);
skill[functionName] = kernel.RegisterSemanticFunction(skillDirectoryName, functionName, functionConfig);
}
}
return skill;
}
/// <summary>
/// Get the front matter and body of a markdown file.
/// </summary>
public static (string? frontMatter, string body) SplitMarkdownFile(string path)
{
var markdown = File.ReadAllText(path);
string? frontMatter = null;
string? body = null;
if (markdown.StartsWith("---"))
{
var endOfFrontMatter = markdown.IndexOf("---", 3);
frontMatter = markdown.Substring(3, endOfFrontMatter - 3);
body = markdown.Substring(endOfFrontMatter + 3).Trim();
}
else
{
body = markdown;
}
return (frontMatter, body);
}
/// <summary>
/// Get the front matter and body of a markdown file.
/// </summary>
public static PromptTemplateConfig TemplateConfigFromYamlText(string frontMatter) =>
yamlDeserializer.Deserialize<PromptTemplateConfig>(frontMatter);
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment