MarkdownLoaderExtensions.cs

using System.Diagnostics.CodeAnalysis;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Orchestration;
using Microsoft.SemanticKernel.SemanticFunctions;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;

namespace Microsoft.SemanticKernel.KernelExtensions;

public static class MarkdownLoaderExtensions
{
    private static readonly Regex s_asciiLettersDigitsUnderscoresRegex = new("^[0-9A-Za-z_]*$");

    private static readonly IDeserializer yamlDeserializer;

    static MarkdownLoaderExtensions()
    {
        yamlDeserializer = new DeserializerBuilder()
            .WithNamingConvention(UnderscoredNamingConvention.Instance)
            .Build();
    }

    internal static void ValidSkillName(string skillName)
    {
        if (!s_asciiLettersDigitsUnderscoresRegex.IsMatch(skillName))
        {
            ThrowInvalidName("skill name", skillName);
        }
    }

    [DoesNotReturn]
    private static void ThrowInvalidName(string kind, string name) =>
        throw new KernelException(
            KernelException.ErrorCodes.InvalidFunctionDescription,
            $"A {kind} can contain only ASCII letters, digits, and underscores: '{name}' is not a valid name.");

    /// <summary>
    /// A kernel extension that allows to load Semantic Functions, defined by prompt templates stored in the filesystem.
    /// A skill directory contains a set of subdirectories, one for each semantic function.
    /// This extension requires the path of the parent directory (e.g. "d:\skills") and the name of the skill directory
    /// (e.g. "OfficeSkill"), which is used also as the "skill name" in the internal skill collection.
    ///
    /// Note: skill and function names can contain only alphanumeric chars and underscore.
    ///
    /// Example:
    /// D:\skills\                            # parentDirectory = "D:\skills"
    ///
    ///     |__ OfficeSkill\                  # skillDirectoryName = "SummarizeEmailThread"
    ///
    ///         |__ ScheduleMeeting.md        # semantic function
    ///         |__ SummarizeEmailThread.md   # semantic function
    ///         |__ MergeWordAndExcelDocs.md  # semantic function
    ///
    ///     |__ XboxSkill\                    # another skill, etc.
    ///
    ///         |__ MessageFriend.md
    ///         |__ LaunchGame.md
    /// </summary>
    /// <param name="kernel">Semantic Kernel instance</param>
    /// <param name="parentDirectory">Directory containing the skill directory, e.g. "d:\myAppSkills"</param>
    /// <param name="skillDirectoryNames">Name of the directories containing the selected skills, e.g. "StrategySkill"</param>
    /// <returns>A list of all the semantic functions found in the directory, indexed by function name.</returns>
    public static IDictionary<string, ISKFunction> ImportSemanticSkillFromMarkdown(
        this IKernel kernel, string parentDirectory, params string[] skillDirectoryNames
    )
    {
        var skill = new Dictionary<string, ISKFunction>();

        foreach (string skillDirectoryName in skillDirectoryNames)
        {
            ValidSkillName(skillDirectoryName);
            var skillDir = Path.Combine(parentDirectory, skillDirectoryName);
            if (!Directory.Exists(skillDir))
                throw new DirectoryNotFoundException($"Directory '{skillDir}' could not be found.");

            var mdFiles = Directory.GetFiles(skillDir, "*.md", SearchOption.TopDirectoryOnly);
            foreach (string file in mdFiles)
            {
                var functionName = Path.GetFileNameWithoutExtension(file);

                // Parse the markdown file
                var (frontMatter, prompt) = SplitMarkdownFile(file);

                // Parse the frontmatter, if any
                var config = frontMatter != null ?
                    TemplateConfigFromYamlText(frontMatter) :
                    new();

                // Create the semantic function
                var template = new PromptTemplate(prompt, config, kernel.PromptTemplateEngine);

                var functionConfig = new SemanticFunctionConfig(config, template);

                kernel.Log.LogTrace("Registering function {0}.{1} loaded from {2}", skillDirectoryName, functionName, file);
                skill[functionName] = kernel.RegisterSemanticFunction(skillDirectoryName, functionName, functionConfig);
            }
        }

        return skill;
    }

    /// <summary>
    /// Get the front matter and body of a markdown file.
    /// </summary>
    public static (string? frontMatter, string body) SplitMarkdownFile(string path)
    {
        var markdown = File.ReadAllText(path);

        string? frontMatter = null;
        string? body = null;

        if (markdown.StartsWith("---"))
        {
            var endOfFrontMatter = markdown.IndexOf("---", 3);
            frontMatter = markdown.Substring(3, endOfFrontMatter - 3);
            body = markdown.Substring(endOfFrontMatter + 3).Trim();
        }
        else
        {
            body = markdown;
        }

        return (frontMatter, body);
    }

    /// <summary>
    /// Get the front matter and body of a markdown file.
    /// </summary>
    public static PromptTemplateConfig TemplateConfigFromYamlText(string frontMatter) => 
        yamlDeserializer.Deserialize<PromptTemplateConfig>(frontMatter);
}