TIHan/unsafe_finder.cs

## unsafe_finder.cs
using System.IO;
using System.Runtime.Intrinsics;
using System.Collections.Concurrent;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Microsoft.CodeAnalysis.Text;
using System.Diagnostics;
using System.Text.Json.Serialization;
using System.Text.Json;
using System;

using Microsoft.CodeAnalysis.MSBuild;
using System.Text.RegularExpressions;
using static System.Runtime.InteropServices.JavaScript.JSType;
using CsvHelper.Configuration;
using System.Globalization;
using CsvHelper;
using CsvHelper.Configuration.Attributes;

namespace api_finder
{
    internal class Program
    {

        record Key(string FilePath, string Name);

        class Entry
        {
            public string? Release { get; set; }
            [Ignore]
            public string? FilePath { get; set; }
            public string? API { get; set; }
            public string? Name { get; set; }

            public int Count { get; set; }
        }

        class GroupedDataItem
        {
            public string? API { get; set; }
            public string? Name { get; set; }

            public int Count { get; set; }
        }

        class GroupedData
        {
            public string? FilePath { get; set; }
            public string? URL { get; set; }
            public List<GroupedDataItem> Usages { get; set; }
        }

        static object lockObj = new object();

        static void RecordUsage(string file, string memberName, Dictionary<Key, int> usage, Func<string, bool> filterMemberName)
        {
            var cleanPath =
                file.Replace("C:\\work\\runtime\\", "")
                .Replace("C:\\work\\history\\runtime_net5\\", "")
                .Replace("C:\\work\\history\\runtime_net6\\", "")
                .Replace("C:\\work\\history\\runtime_net7\\", "")
                .Replace("C:\\work\\history\\runtime_net8\\", "");
            var key =
                new Key(cleanPath, memberName);

            if (filterMemberName(memberName))
            {
                lock (lockObj)
                {
                    int count = 0;
                    if (!usage.TryGetValue(key, out count))
                        usage[key] = count = 0;

                    count++;
                    usage[key] = count;
                }
            }
        }

        static (IEnumerable<Entry>, IEnumerable<GroupedData>) ProcessUsage(Dictionary<Key, int> usage, string release, string api)
        {
            var entries = new List<Entry>();

            foreach (var item in usage)
            {
                entries.Add(
                    new Entry
                    {
                        Release = release,
                        FilePath = item.Key.FilePath,
                        API = api,
                        Name = item.Key.Name,
                        Count = item.Value
                    }
                );
            }

            var data = entries.OrderBy(x => x.Name).OrderBy(x => x.API).OrderBy(x => x.FilePath);
            var groupedData = data.GroupBy(x => x.FilePath);

            var groups = new List<GroupedData>();

            foreach (var group in groupedData)
            {
                var groupFilePath = "";
                var usages = new List<GroupedDataItem>();
                foreach (var item in group)
                {
                    groupFilePath = item.FilePath;
                    usages.Add(
                        new GroupedDataItem()
                        {
                            API = item.API,
                            Name = item.Name,
                            Count = item.Count
                        }
                    );
                }
                if (!string.IsNullOrEmpty(groupFilePath))
                {
                    groups.Add(
                        new GroupedData()
                        {
                            FilePath = groupFilePath,
                            URL = Path.Combine("https://github.com/dotnet/runtime/tree/main/", groupFilePath).Replace("\\", "/"),
                            Usages = usages
                        }
                    );
                }
            }

            var xs = data.GroupBy(x => (x.API, x.Name)).Select(x => new Entry() { API = x.Key.API, Name = x.Key.Name, Release = release, Count = x.Sum(x => x.Count) });

            return (xs, groups);
        }

        static void WriteGroups(string path, IEnumerable<GroupedData> groups)
        {
            var options = new JsonSerializerOptions();
            options.WriteIndented = true;
            File.WriteAllText(path, JsonSerializer.Serialize(groups, options));
        }

        static void PrintDataUsage(IEnumerable<Entry> data, string api)
        {
            var totals = data.GroupBy(x => x.Name).Select(x => (x.Key, x.Sum(x => x.Count))).OrderByDescending(x => x.Item2);

            var totalCount = totals.Sum(x => x.Item2);

            Console.WriteLine($"{api} - Total Count: {totalCount}");

            foreach (var (name, count) in totals)
            {
                Console.WriteLine($"{api}.{name} - Count: {count}");
            }
        }

        static IEnumerable<Entry> ProcessAndStore(Dictionary<Key, int> usage, string release, string api, string path)
        {
            var (data, groups) = ProcessUsage(usage, release, api);
            WriteGroups(path, groups);
            PrintDataUsage(data, api);
            Console.WriteLine("");
            return data;
        }
        static List<Entry> Run(string dirPath, string runtimeName)
        {
            try
            {
                var dirName = Path.GetDirectoryName(dirPath);
                var dir = Directory.EnumerateFiles(dirName, "*.cs", SearchOption.AllDirectories).Where(x => !x.Contains("tests\\"));

                var entries = new List<Entry>();
                var lockObj = new object();

                var unsafeUsage = new Dictionary<Key, int>();
                var memoryMarshalUsage = new Dictionary<Key, int>();
                var gcUsage = new Dictionary<Key, int>();
                var loadUnsafeUsage = new Dictionary<Key, int>();
                var pointerUsage = new Dictionary<Key, int>();

                Parallel.ForEach(dir, (file) =>
                {
                    var tree = CSharpSyntaxTree.ParseText(SourceText.From(File.ReadAllText(file)));

                    var exprs =
                        tree
                            .GetRoot()
                            .DescendantNodes((_) => true)
                            .OfType<MemberAccessExpressionSyntax>();

                    foreach (var expr in exprs)
                    {
                        var token = expr.Name.Identifier.GetPreviousToken();
                        if (!token.IsKind(SyntaxKind.DotToken))
                        {
                            continue;
                        }

                        token = token.GetPreviousToken();

                        if (token.IsKind(SyntaxKind.IdentifierToken))
                        {
                            switch (token.ValueText)
                            {
                                case "Unsafe":
                                    RecordUsage(file, expr.Name.Identifier.ValueText, unsafeUsage,
                                        memberName =>
                                        {
                                            switch (memberName)
                                            {
                                                case "AreSame":
                                                case "IsAddressGreaterThan":
                                                case "IsAddressLessThan":
                                                case "IsNullRef":
                                                case "NullRef":
                                                case "SizeOf":
                                                    return false;

                                                default:
                                                    return true;
                                            }
                                        }
                                    );
                                    break;

                                case "MemoryMarshal":
                                    RecordUsage(file, expr.Name.Identifier.ValueText, memoryMarshalUsage,
                                        memberName =>
                                        {
                                            switch (memberName)
                                            {
                                                case "ToEnumerable":
                                                case "TryGetString":
                                                    return false;

                                                default:
                                                    return true;
                                            }
                                        }
                                    );
                                    break;

                                case "GC":
                                    RecordUsage(file, expr.Name.Identifier.ValueText, gcUsage,
                                        memberName =>
                                        {
                                            switch (memberName)
                                            {
                                                case "AllocateUninitializedArray":
                                                    return true;

                                                default:
                                                    return false;
                                            }
                                        }
                                    );
                                    break;

                                default:
                                    var memberName = expr.Name.Identifier.ValueText;
                                    if (memberName == "LoadUnsafe")
                                    {
                                        RecordUsage(file, "LoadUnsafe", loadUnsafeUsage, (_) => true);
                                    }
                                    break;

                            }
                        }
                    }

                    var pointerTypes =
                        tree
                            .GetRoot()
                            .DescendantNodes((_) => true)
                            .OfType<PointerTypeSyntax>();

                    foreach (var ty in pointerTypes)
                    {
                        RecordUsage(file, "(type - '*')", pointerUsage, (_) => true);
                    }

                    var typeNames =
                        tree
                            .GetRoot()
                            .DescendantNodes((_) => true)
                            .OfType<TypeSyntax>()
                            .SelectMany(x => x.DescendantNodes((_) => true).OfType<IdentifierNameSyntax>());

                    foreach (var tyName in typeNames)
                    {
                        if (tyName.Identifier.ValueText == "IntPtr")
                        {
                            RecordUsage(file, "IntPtr", pointerUsage, (_) => true);
                        }
                        else if (tyName.Identifier.ValueText == "UIntPtr")
                        {
                            RecordUsage(file, "UIntPtr", pointerUsage, (_) => true);
                        }
                    }

                    var allExprs =
                        tree
                            .GetRoot()
                            .DescendantNodes((_) => true)
                            .OfType<ExpressionSyntax>();

                    foreach (var expr in allExprs)
                    {
                        if (expr.IsKind(SyntaxKind.PointerIndirectionExpression))
                        {
                            RecordUsage(file, "PointerIndirection", pointerUsage, (_) => true);
                        }
                        else if (expr.IsKind(SyntaxKind.AddressOfExpression))
                        {
                            RecordUsage(file, "AddressOf", pointerUsage, (_) => true);
                        }
                    }
                });

                var unsafeData = ProcessAndStore(unsafeUsage, runtimeName, "Unsafe", $"unsage_{runtimeName}.json");
                var memoryMarshalData = ProcessAndStore(memoryMarshalUsage, runtimeName, "MemoryMarshal", $"memory-marshal_{runtimeName}.json");
                var gcData = ProcessAndStore(gcUsage, runtimeName, "GC", $"gc_{runtimeName}.json");
                var loadUnsafeData = ProcessAndStore(loadUnsafeUsage, runtimeName, "(load-unsafe)", $"load-unsafe_{runtimeName}.json");
                var pointerData = ProcessAndStore(pointerUsage, runtimeName, "(pointer)", $"pointer_{runtimeName}.json");

                var all = new List<Entry>();
                all.AddRange(unsafeData);
                all.AddRange(memoryMarshalData);
                all.AddRange(gcData);
                all.AddRange(loadUnsafeData);
                all.AddRange(pointerData);
                return all;
            }
            catch (Exception e)
            {
                Console.Error.WriteLine(e.Message);
                throw;
            }
        }

        static unsafe int Main(string[] args)
        {
            if (args.Length == 0)
            {
                Console.Error.WriteLine("Invalid directory.");
                return 1;
            }

            var stopwatch = Stopwatch.StartNew();

            Console.WriteLine("net5 - commit from september 28, 2020");
            Console.WriteLine("=====================");
            var net5 = Run("C:\\work\\history\\runtime_net5\\src\\libraries", "net5");
            Console.WriteLine("==========================================");
            Console.WriteLine("");

            Console.WriteLine("release/net6.0");
            Console.WriteLine("=====================");
            var net6 = Run("C:\\work\\history\\runtime_net6\\src\\libraries", "net6");
            Console.WriteLine("==========================================");
            Console.WriteLine("");

            Console.WriteLine("release/net7.0");
            Console.WriteLine("=====================");
            var net7 = Run("C:\\work\\history\\runtime_net7\\src\\libraries", "net7");
            Console.WriteLine("==========================================");
            Console.WriteLine("");

            Console.WriteLine("release/net8.0");
            Console.WriteLine("=====================");
            var net8 = Run("C:\\work\\history\\runtime_net8\\src\\libraries", "net8");
            Console.WriteLine("==========================================");
            Console.WriteLine("");

            Console.WriteLine("main - current");
            Console.WriteLine("=====================");
            var current = Run(args[0], "current");
            Console.WriteLine("==========================================");
            Console.WriteLine("");

            Console.WriteLine($"-- FINISHED -- Time: {stopwatch.Elapsed.TotalSeconds}s");

            var all = new List<Entry>();
            all.AddRange(net5);
            all.AddRange(net6);
            all.AddRange(net7);
            all.AddRange(net8);
            all.AddRange(current);

            using (var writer = new StreamWriter("unsafe_data.csv"))
            using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
            {
                csv.WriteRecords(all);
            }

            return 0;
        }
    }
}
	using System.IO;
	using System.Runtime.Intrinsics;
	using System.Collections.Concurrent;
	using Microsoft.CodeAnalysis;
	using Microsoft.CodeAnalysis.CSharp;
	using Microsoft.CodeAnalysis.CSharp.Syntax;
	using Microsoft.CodeAnalysis.Text;
	using System.Diagnostics;
	using System.Text.Json.Serialization;
	using System.Text.Json;
	using System;

	using Microsoft.CodeAnalysis.MSBuild;
	using System.Text.RegularExpressions;
	using static System.Runtime.InteropServices.JavaScript.JSType;
	using CsvHelper.Configuration;
	using System.Globalization;
	using CsvHelper;
	using CsvHelper.Configuration.Attributes;

	namespace api_finder
	{
	internal class Program
	{

	record Key(string FilePath, string Name);

	class Entry
	{
	public string? Release { get; set; }
	[Ignore]
	public string? FilePath { get; set; }
	public string? API { get; set; }
	public string? Name { get; set; }

	public int Count { get; set; }
	}

	class GroupedDataItem
	{
	public string? API { get; set; }
	public string? Name { get; set; }

	public int Count { get; set; }
	}

	class GroupedData
	{
	public string? FilePath { get; set; }
	public string? URL { get; set; }
	public List<GroupedDataItem> Usages { get; set; }
	}

	static object lockObj = new object();

	static void RecordUsage(string file, string memberName, Dictionary<Key, int> usage, Func<string, bool> filterMemberName)
	{
	var cleanPath =
	file.Replace("C:\\work\\runtime\\", "")
	.Replace("C:\\work\\history\\runtime_net5\\", "")
	.Replace("C:\\work\\history\\runtime_net6\\", "")
	.Replace("C:\\work\\history\\runtime_net7\\", "")
	.Replace("C:\\work\\history\\runtime_net8\\", "");
	var key =
	new Key(cleanPath, memberName);

	if (filterMemberName(memberName))
	{
	lock (lockObj)
	{
	int count = 0;
	if (!usage.TryGetValue(key, out count))
	usage[key] = count = 0;

	count++;
	usage[key] = count;
	}
	}
	}

	static (IEnumerable<Entry>, IEnumerable<GroupedData>) ProcessUsage(Dictionary<Key, int> usage, string release, string api)
	{
	var entries = new List<Entry>();

	foreach (var item in usage)
	{
	entries.Add(
	new Entry
	{
	Release = release,
	FilePath = item.Key.FilePath,
	API = api,
	Name = item.Key.Name,
	Count = item.Value
	}
	);
	}

	var data = entries.OrderBy(x => x.Name).OrderBy(x => x.API).OrderBy(x => x.FilePath);
	var groupedData = data.GroupBy(x => x.FilePath);

	var groups = new List<GroupedData>();

	foreach (var group in groupedData)
	{
	var groupFilePath = "";
	var usages = new List<GroupedDataItem>();
	foreach (var item in group)
	{
	groupFilePath = item.FilePath;
	usages.Add(
	new GroupedDataItem()
	{
	API = item.API,
	Name = item.Name,
	Count = item.Count
	}
	);
	}
	if (!string.IsNullOrEmpty(groupFilePath))
	{
	groups.Add(
	new GroupedData()
	{
	FilePath = groupFilePath,
	URL = Path.Combine("https://github.com/dotnet/runtime/tree/main/", groupFilePath).Replace("\\", "/"),
	Usages = usages
	}
	);
	}
	}

	var xs = data.GroupBy(x => (x.API, x.Name)).Select(x => new Entry() { API = x.Key.API, Name = x.Key.Name, Release = release, Count = x.Sum(x => x.Count) });

	return (xs, groups);
	}

	static void WriteGroups(string path, IEnumerable<GroupedData> groups)
	{
	var options = new JsonSerializerOptions();
	options.WriteIndented = true;
	File.WriteAllText(path, JsonSerializer.Serialize(groups, options));
	}

	static void PrintDataUsage(IEnumerable<Entry> data, string api)
	{
	var totals = data.GroupBy(x => x.Name).Select(x => (x.Key, x.Sum(x => x.Count))).OrderByDescending(x => x.Item2);

	var totalCount = totals.Sum(x => x.Item2);

	Console.WriteLine($"{api} - Total Count: {totalCount}");

	foreach (var (name, count) in totals)
	{
	Console.WriteLine($"{api}.{name} - Count: {count}");
	}
	}

	static IEnumerable<Entry> ProcessAndStore(Dictionary<Key, int> usage, string release, string api, string path)
	{
	var (data, groups) = ProcessUsage(usage, release, api);
	WriteGroups(path, groups);
	PrintDataUsage(data, api);
	Console.WriteLine("");
	return data;
	}
	static List<Entry> Run(string dirPath, string runtimeName)
	{
	try
	{
	var dirName = Path.GetDirectoryName(dirPath);
	var dir = Directory.EnumerateFiles(dirName, "*.cs", SearchOption.AllDirectories).Where(x => !x.Contains("tests\\"));

	var entries = new List<Entry>();
	var lockObj = new object();

	var unsafeUsage = new Dictionary<Key, int>();
	var memoryMarshalUsage = new Dictionary<Key, int>();
	var gcUsage = new Dictionary<Key, int>();
	var loadUnsafeUsage = new Dictionary<Key, int>();
	var pointerUsage = new Dictionary<Key, int>();

	Parallel.ForEach(dir, (file) =>
	{
	var tree = CSharpSyntaxTree.ParseText(SourceText.From(File.ReadAllText(file)));

	var exprs =
	tree
	.GetRoot()
	.DescendantNodes((_) => true)
	.OfType<MemberAccessExpressionSyntax>();

	foreach (var expr in exprs)
	{
	var token = expr.Name.Identifier.GetPreviousToken();
	if (!token.IsKind(SyntaxKind.DotToken))
	{
	continue;
	}

	token = token.GetPreviousToken();

	if (token.IsKind(SyntaxKind.IdentifierToken))
	{
	switch (token.ValueText)
	{
	case "Unsafe":
	RecordUsage(file, expr.Name.Identifier.ValueText, unsafeUsage,
	memberName =>
	{
	switch (memberName)
	{
	case "AreSame":
	case "IsAddressGreaterThan":
	case "IsAddressLessThan":
	case "IsNullRef":
	case "NullRef":
	case "SizeOf":
	return false;

	default:
	return true;
	}
	}
	);
	break;

	case "MemoryMarshal":
	RecordUsage(file, expr.Name.Identifier.ValueText, memoryMarshalUsage,
	memberName =>
	{
	switch (memberName)
	{
	case "ToEnumerable":
	case "TryGetString":
	return false;

	default:
	return true;
	}
	}
	);
	break;

	case "GC":
	RecordUsage(file, expr.Name.Identifier.ValueText, gcUsage,
	memberName =>
	{
	switch (memberName)
	{
	case "AllocateUninitializedArray":
	return true;

	default:
	return false;
	}
	}
	);
	break;

	default:
	var memberName = expr.Name.Identifier.ValueText;
	if (memberName == "LoadUnsafe")
	{
	RecordUsage(file, "LoadUnsafe", loadUnsafeUsage, (_) => true);
	}
	break;

	}
	}
	}

	var pointerTypes =
	tree
	.GetRoot()
	.DescendantNodes((_) => true)
	.OfType<PointerTypeSyntax>();

	foreach (var ty in pointerTypes)
	{
	RecordUsage(file, "(type - '*')", pointerUsage, (_) => true);
	}

	var typeNames =
	tree
	.GetRoot()
	.DescendantNodes((_) => true)
	.OfType<TypeSyntax>()
	.SelectMany(x => x.DescendantNodes((_) => true).OfType<IdentifierNameSyntax>());

	foreach (var tyName in typeNames)
	{
	if (tyName.Identifier.ValueText == "IntPtr")
	{
	RecordUsage(file, "IntPtr", pointerUsage, (_) => true);
	}
	else if (tyName.Identifier.ValueText == "UIntPtr")
	{
	RecordUsage(file, "UIntPtr", pointerUsage, (_) => true);
	}
	}

	var allExprs =
	tree
	.GetRoot()
	.DescendantNodes((_) => true)
	.OfType<ExpressionSyntax>();

	foreach (var expr in allExprs)
	{
	if (expr.IsKind(SyntaxKind.PointerIndirectionExpression))
	{
	RecordUsage(file, "PointerIndirection", pointerUsage, (_) => true);
	}
	else if (expr.IsKind(SyntaxKind.AddressOfExpression))
	{
	RecordUsage(file, "AddressOf", pointerUsage, (_) => true);
	}
	}
	});

	var unsafeData = ProcessAndStore(unsafeUsage, runtimeName, "Unsafe", $"unsage_{runtimeName}.json");
	var memoryMarshalData = ProcessAndStore(memoryMarshalUsage, runtimeName, "MemoryMarshal", $"memory-marshal_{runtimeName}.json");
	var gcData = ProcessAndStore(gcUsage, runtimeName, "GC", $"gc_{runtimeName}.json");
	var loadUnsafeData = ProcessAndStore(loadUnsafeUsage, runtimeName, "(load-unsafe)", $"load-unsafe_{runtimeName}.json");
	var pointerData = ProcessAndStore(pointerUsage, runtimeName, "(pointer)", $"pointer_{runtimeName}.json");

	var all = new List<Entry>();
	all.AddRange(unsafeData);
	all.AddRange(memoryMarshalData);
	all.AddRange(gcData);
	all.AddRange(loadUnsafeData);
	all.AddRange(pointerData);
	return all;
	}
	catch (Exception e)
	{
	Console.Error.WriteLine(e.Message);
	throw;
	}
	}

	static unsafe int Main(string[] args)
	{
	if (args.Length == 0)
	{
	Console.Error.WriteLine("Invalid directory.");
	return 1;
	}

	var stopwatch = Stopwatch.StartNew();

	Console.WriteLine("net5 - commit from september 28, 2020");
	Console.WriteLine("=====================");
	var net5 = Run("C:\\work\\history\\runtime_net5\\src\\libraries", "net5");
	Console.WriteLine("==========================================");
	Console.WriteLine("");

	Console.WriteLine("release/net6.0");
	Console.WriteLine("=====================");
	var net6 = Run("C:\\work\\history\\runtime_net6\\src\\libraries", "net6");
	Console.WriteLine("==========================================");
	Console.WriteLine("");

	Console.WriteLine("release/net7.0");
	Console.WriteLine("=====================");
	var net7 = Run("C:\\work\\history\\runtime_net7\\src\\libraries", "net7");
	Console.WriteLine("==========================================");
	Console.WriteLine("");

	Console.WriteLine("release/net8.0");
	Console.WriteLine("=====================");
	var net8 = Run("C:\\work\\history\\runtime_net8\\src\\libraries", "net8");
	Console.WriteLine("==========================================");
	Console.WriteLine("");

	Console.WriteLine("main - current");
	Console.WriteLine("=====================");
	var current = Run(args[0], "current");
	Console.WriteLine("==========================================");
	Console.WriteLine("");

	Console.WriteLine($"-- FINISHED -- Time: {stopwatch.Elapsed.TotalSeconds}s");

	var all = new List<Entry>();
	all.AddRange(net5);
	all.AddRange(net6);
	all.AddRange(net7);
	all.AddRange(net8);
	all.AddRange(current);

	using (var writer = new StreamWriter("unsafe_data.csv"))
	using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
	{
	csv.WriteRecords(all);
	}

	return 0;
	}
	}
	}