Scan over a directory that contains directories of expanded FHIR packages and extract a file called sds.json summarizing its contents

ProcessPackageCache.cs

using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using Hl7.Fhir.Model;
using Hl7.Fhir.Rest;
using Hl7.Fhir.Serialization;
using Hl7.Fhir.Utility;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Newtonsoft.Json;
using Task = System.Threading.Tasks.Task;

namespace QForms.Test
{
    [TestClass]
    public class ImportPackageCache : TestBase
    {
        [TestInitialize]
        public void Initialize()
        {
            Hl7.Fhir.FhirPath.ElementNavFhirExtensions.PrepareFhirSymbolTableFunctions();
        }

        private bool SkipFile(string path)
        {
            if (path.Contains("StructureDefinition", System.StringComparison.OrdinalIgnoreCase))
                return false;
            if (path.Contains("Questionnaire", System.StringComparison.OrdinalIgnoreCase))
                return false;
            if (path.Contains("SearchParameter", System.StringComparison.OrdinalIgnoreCase))
                return false;

            return true;

            if (path.EndsWith("package.json"))
                return true;
            if (path.EndsWith(".index.json"))
                return true;
            return false;
        }

        [TestMethod]
        public async Task ImportPackageCacheFiles_scanfilesonly()
        {
            // This is one hairy test, it will scan the package cache and read ALL content within
            // anything that is a resource type that this service can hold, it will load into the server.
            // only in fhirVersion 4.x
            var directories = System.IO.Directory.GetDirectories(@"E:\FHIR-Package-Cache");
            foreach (var packageFolder in directories)
            {
                if (packageFolder.Contains("fhir_5.0.0", System.StringComparison.OrdinalIgnoreCase))
                    continue;
                if (packageFolder.Contains("r5", System.StringComparison.OrdinalIgnoreCase))
                    continue;
                if (packageFolder.Contains("r3", System.StringComparison.OrdinalIgnoreCase))
                    continue;
                if (packageFolder.Contains("dstu2", System.StringComparison.OrdinalIgnoreCase))
                    continue;
                if (packageFolder.Contains("tx.support", System.StringComparison.OrdinalIgnoreCase))
                    continue;

                // scan all the files in this package directory
                var files = System.IO.Directory.GetFiles(packageFolder, "*.json", System.IO.SearchOption.AllDirectories);
                foreach (var filename in files)
                {
                    if (SkipFile(filename))
                        continue;
                    System.Console.WriteLine(filename);
                }
            }
        }

        [TestMethod]
        public async Task ImportPackageCacheFiles_parsefiles()
        {
            // This is one hairy test, it will scan the package cache and read ALL content within
            // anything that is a resource type that this service can hold, it will load into the server.
            // only in fhirVersion 4.x
            var directories = System.IO.Directory.GetDirectories(@"E:\FHIR-Package-Cache");
            var settings = new FhirJsonPocoDeserializerSettings()
            {
                Validator = null, // Since we can handle multiple issues, let this through
            };
            var ds = new FhirJsonPocoDeserializer(settings);
            long errors = 0;
            long success = 0;

            ExtractResults results = new ExtractResults();
            var sds = results.structureDefinition;

            foreach (var packageFolder in directories)
            {
                if (packageFolder.Contains("fhir_5.0.0", System.StringComparison.OrdinalIgnoreCase))
                    continue;
                if (packageFolder.Contains("r5", System.StringComparison.OrdinalIgnoreCase))
                    continue;
                if (packageFolder.Contains("r3", System.StringComparison.OrdinalIgnoreCase))
                    continue;
                if (packageFolder.Contains("dstu2", System.StringComparison.OrdinalIgnoreCase))
                    continue;
                if (packageFolder.Contains("tx.support", System.StringComparison.OrdinalIgnoreCase))
                    continue;

                // Check the package.json file to see if this is a 4.x package
                var packageFile = System.IO.Path.Combine(packageFolder, "package.json");
                if (!System.IO.File.Exists(packageFile))
                    continue;
                Firely.Fhir.Packages.PackageManifest package = null;
                using (var stream = System.IO.File.Open(packageFile, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
                {
                    using (StreamReader sr = new StreamReader(stream))
                    {
                        string packageJson = sr.ReadToEnd();
                        package = Firely.Fhir.Packages.PackageParser.ParseManifest(packageJson);
                        if (package.FhirVersions != null && !package.FhirVersions.Contains("4.0")
                            && !package.FhirVersions.Contains("4.3")
                            && !package.FhirVersions.Contains("4.0.0")
                            && !package.FhirVersions.Contains("4.0.1")
                            && !package.FhirVersions.Contains("4.3.0"))
                            continue;
                    }
                }

                // scan all the files in this package directory
                var files = System.IO.Directory.GetFiles(packageFolder, "*.json", System.IO.SearchOption.AllDirectories);
                var packageName = new DirectoryInfo(packageFolder).Name.Replace("_package.tgz", "");
                foreach (var filename in files)
                {
                    if (SkipFile(filename))
                        continue;
                    // System.Console.WriteLine(filename);
                    using (var stream = System.IO.File.Open(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
                    {
                        using (StreamReader sr = new StreamReader(stream))
                        {
                            try
                            {
                                var resource = ds.DeserializeResource(sr.ReadToEnd());
                                if (resource is StructureDefinition sd && sd.Differential != null)
                                {
                                    StructureDefinitionExtract sde = new StructureDefinitionExtract()
                                    {
                                        packageFilename = packageName,
                                        packageId = package?.Name,
                                        packageVersion = package?.Version,
                                        packageLicense = package?.License,
                                        filename = new FileInfo(filename).Name,
                                        url = sd.Url,
                                        version = sd.Version,
                                        status = sd.Status.GetLiteral(),
                                        copyright = sd.Copyright,
                                        copyrightLabel = sd.CopyrightLabel,
                                        publisher = sd.Publisher,
                                        title = sd.Title
                                    };
                                    foreach (var e in sd.Differential.Element.Where(e => e.Constraint.Any(c => !c.Key.StartsWith("dom-") && !c.Key.StartsWith("ext-") && !c.Key.StartsWith("ele-"))))
                                    {
                                        foreach (var c in e.Constraint.Where(c => !c.Key.StartsWith("dom-") && !c.Key.StartsWith("ext-") && !c.Key.StartsWith("ele-")))
                                        {
                                            System.Console.WriteLine($"Found expression in {filename} {c.Key}: {c.Expression}");
                                            results.expressionCount++;
                                            sde.invariant.Add(new ExpressionContext()
                                            {
                                                path = e.Path,
                                                key = c.Key,
                                                expression = c.Expression,
                                                human = c.Human
                                            });
                                        }
                                    }
                                    if (sde.invariant.Count > 0)
                                    {
                                        sds.Add(sde);
                                        System.Console.WriteLine($"Found {sde.invariant.Count} invariants in {sde.filename}");
                                    }
                                }
                                Interlocked.Increment(ref success);
                            }
                            catch (DeserializationFailedException exception)
                            {
                                // System.Console.WriteLine($"Error processing {filename}: {exception.Message}");
                                Interlocked.Increment(ref errors);
                            }
                        }
                    }
                }
            }
            System.Console.WriteLine();
            System.Console.WriteLine($"Final result: {success}/{errors}/{success + errors}");
            System.Console.WriteLine($"Expression Count: {results.expressionCount}");

            // Save sds to a JSON file called sds.json
            results.structureDefinitionCount = sds.Count;
            using (var stream = System.IO.File.Open("sds.json", System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.Read))
            {
                using (StreamWriter sw = new StreamWriter(stream))
                {
                    sw.Write(JsonConvert.SerializeObject(results, Newtonsoft.Json.Formatting.Indented));
                }
            }
        }
    }

    public class ExtractResults
    {
        public long structureDefinitionCount { get; set; }
        public long expressionCount { get; set; }

        public List<StructureDefinitionExtract> structureDefinition { get; private set; } = new List<StructureDefinitionExtract>();
    }

    public class StructureDefinitionExtract
    {
        public string packageFilename { get; init; }
        public string packageId { get; init; }
        public string packageVersion { get; init; }
        public string packageLicense { get; init; }
        public string filename { get; init; }
        public string url { get; init; }
        public string version { get; init; }
        public string status { get; init; }
        public string copyright { get; init; }
        public string copyrightLabel { get; init; }
        public string publisher { get; init; }
        public string title { get; init; }
        public List<ExpressionContext> invariant { get; private set; } = new List<ExpressionContext>();
    }

    public class ExpressionContext
    {
        public string path { get; init; }
        public string key { get; init; }
        public string expression { get; init; }
        public string human { get; init; }
    }
}