apitoolkit.cs

using CloudNative.CloudEvents;
using Google.Cloud.PubSub.V1;
using Google.Apis.Auth.OAuth2;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace ApiToolkit
{
public class Payload
{
[JsonProperty("timestamp")]
public DateTime Timestamp { get; set; }
[JsonProperty("request_headers")]
public Dictionary<string, string[]> RequestHeaders { get; set; }
[JsonProperty("query_params")]
public Dictionary<string, string[]> QueryParams { get; set; }
[JsonProperty("path_params")]
public Dictionary<string, string> PathParams { get; set; }
[JsonProperty("response_headers")]
public Dictionary<string, string[]> ResponseHeaders { get; set; }
[JsonProperty("method")]
public string Method { get; set; }
[JsonProperty("sdk_type")]
public string SdkType { get; set; }
[JsonProperty("host")]
public string Host { get; set; }
[JsonProperty("raw_url")]
public string RawUrl { get; set; }
[JsonProperty("referer")]
public string Referer { get; set; }
[JsonProperty("project_id")]
public string ProjectId { get; set; }
[JsonProperty("url_path")]
public string UrlPath { get; set; }
[JsonProperty("response_body")]
public byte[] ResponseBody { get; set; }
[JsonProperty("request_body")]
public byte[] RequestBody { get; set; }
[JsonProperty("proto_minor")]
public int ProtoMinor { get; set; }
[JsonProperty("status_code")]
public int StatusCode { get; set; }
[JsonProperty("proto_major")]
public int ProtoMajor { get; set; }
[JsonProperty("duration")]
public TimeSpan Duration { get; set; }
}

public class Config
{
    [JsonProperty("debug")]
    public bool Debug { get; set; }
    [JsonProperty("verbose_debug")]
    public bool VerboseDebug { get; set; }
    [JsonProperty("root_url")]
    public string RootUrl { get; set; }
    [JsonProperty("api_key")]
    public string ApiKey { get; set; }
    [JsonProperty("project_id")]
    public string ProjectId { get; set; }
    [JsonProperty("redact_headers")]
    public List<string> RedactHeaders { get; set; }
    [JsonProperty("redact_request_body")]
    public List<string> RedactRequestBody { get; set; }
    [JsonProperty("redact_response_body")]
    public List<string> RedactResponseBody { get; set; }
}

public class ClientMetadata
{
    [JsonProperty("project_id")]
    public string ProjectId { get; set; }
    [JsonProperty("pubsub_project_id")]
    public string PubsubProjectId { get; set; }
    [JsonProperty("topic_id")]
    public string TopicId { get; set; }
    [JsonProperty("pubsub_push_service_account")]
    public JRaw PubsubPushServiceAccount { get; set; }
}

// public class Client
// {
//     private readonly PublisherClient _pubSubClient;
//     private readonly TopicName _topicName;
//     private readonly Config _config;
//     private readonly ClientMetadata _metadata;

//     public Client(PublisherClient pubSubClient, TopicName topicName, Config config, ClientMetadata metadata)
//     {


public static async Task<Client> NewClientAsync(CancellationToken cancellationToken, Config cfg)
{
    DotNetEnv.Env.Load(".env");
    var url = "https://app.apitoolkit.io";
    if (!string.IsNullOrEmpty(cfg.RootURL))
    {
        url = cfg.RootURL;
    }

    var headers = new Dictionary<string, string>
    {
        { "Authorization", $"Bearer {cfg.APIKey}" }
    };

    using var response = await url
        .AppendPathSegment("api/client_metadata")
        .WithHeaders(headers)
        .GetAsync(cancellationToken);

    if (!response.IsSuccessStatusCode)
    {
        throw new Exception($"Unable to query apitoolkit for client metadata: {response.StatusCode}");
    }

    var clientMetadata = await response.Content.ReadAsAsync<ClientMetadata>(cancellationToken);
    if (clientMetadata == null)
    {
        throw new Exception("Unable to unmarshal client metadata response");
    }

    var credentials = GoogleCredential
        .FromJson(clientMetadata.PubsubPushServiceAccount)
        .CreateScoped(PublisherService.Scope.Pubsub);

    var topic = await PublisherServiceApiClient
        .Create(credentials)
        .CreateTopicAsync(new Topic
        {
            Name = clientMetadata.TopicID
        }, cancellationToken: cancellationToken);

    var pubsubClient = await PublisherServiceApiClient
        .Create(credentials)
        .CreatePublisherAsync(new Publisher
        {
            Topic = topic.Name
        }, cancellationToken: cancellationToken);

    var cl = new Client
    {
        PubsubClient = pubsubClient,
        GoReqsTopic = topic,
        Config = cfg,
        Metadata = clientMetadata
    };
    cl.PublishMessage = cl.PublishMessageHandler;

    if (cl.Config.Debug)
    {
        Console.WriteLine("APIToolkit: client initialized successfully");
    }

    return cl;
}

public async Task PublishMessageAsync(Payload payload)
{
    if (goReqsTopic == null)
    {
        if (config.Debug)
        {
            Console.WriteLine("APIToolkit: topic is not initialized. Check client initialization");
        }
        throw new Exception("topic is not initialized");
    }

    var jsonPayload = JsonConvert.SerializeObject(payload);

    var message = new PubsubMessage
    {
        Data = ByteString.CopyFromUtf8(jsonPayload),
        PublishTime = Timestamp.FromDateTime(DateTime.UtcNow),
    };

    await goReqsTopic.PublishAsync(message);

    if (config.Debug)
    {
        Console.WriteLine("APIToolkit: message published to pubsub topic");

        if (config.VerboseDebug)
        {
            Console.WriteLine($"APIToolkit: {jsonPayload}");
        }
    }
}

public Payload BuildPayload(string SDKType, DateTime trackingStart, HttpRequestMessage req, int statusCode, byte[] reqBody, byte[] respBody, IDictionary<string, IEnumerable<string>> respHeader, IDictionary<string, string> pathParams, string urlPath)
{
    if (req == null || this == null || req.RequestUri == null)
    {
        // Early return with empty payload to prevent any null reference exceptions
        if (config.Debug)
        {
            Console.WriteLine("APIToolkit: null request or client or url while building payload.");
        }
        return new Payload();
    }
    string projectId = "";
    if (metadata != null)
    {
        projectId = metadata.ProjectId;
    }

    List<string> redactedHeaders = new List<string>();
    foreach (var v in config.RedactHeaders)
    {
        redactedHeaders.Add(v.ToLower());
    }

    TimeSpan since = DateTime.UtcNow.Subtract(trackingStart);
    return new Payload
    {
        Duration = since,
        Host = req.RequestUri.Host,
        Method = req.Method.Method,
        PathParams = null, // replace with appropriate code if necessary
        ProjectID = projectId,
        ProtoMajor = req.Version.Major,
        ProtoMinor = req.Version.Minor,
        QueryParams = req.RequestUri.ParseQueryString(),
        RawURL = req.RequestUri.AbsoluteUri,
        Referer = req.Headers.Referrer?.AbsoluteUri,
        RequestBody = Redact(reqBody, config.RedactRequestBody),
        RequestHeaders = RedactHeaders(req.Headers, redactedHeaders),
        ResponseBody = Redact(respBody, config.RedactResponseBody),
        ResponseHeaders = RedactHeaders(respHeader, redactedHeaders),
        SdkType = SDKType,
        StatusCode = statusCode,
        Timestamp = DateTime.UtcNow,
        URLPath = urlPath,
    };
}


using Newtonsoft.Json.Linq;

public static byte[] Redact(byte[] data, string[] redactList)
{
    var obj = JToken.Parse(System.Text.Encoding.UTF8.GetString(data));
    var config = new JsonPath.JsonPathConfig { UseXmlNotation = true };
    var path = new JsonPath.JsonPath(config);
    foreach (var key in redactList)
    {
        var results = path.Evaluate(key, obj);
        foreach (var result in results)
        {
            if (result.Value is JsonPath.Accessor accessor)
            {
                accessor.Value = "[CLIENT_REDACTED]";
            }
        }
    }
    return System.Text.Encoding.UTF8.GetBytes(obj.ToString());
}

public static IDictionary<string, IEnumerable<string>> RedactHeaders(IDictionary<string, IEnumerable<string>> headers, string[] redactList)
{
    foreach (var key in headers.Keys.ToArray())
    {
        if (Array.IndexOf(redactList, key.ToLower()) >= 0)
        {
            headers[key] = new string[] { "[CLIENT_REDACTED]" };
        }
    }
    return headers;
}

public static bool Find(string[] haystack, string needle)
{
    return Array.IndexOf(haystack, needle) >= 0;
}


public async Task GinMiddleware(HttpContext context)
{
    var start = DateTime.UtcNow;
    var request = context.Request;
    request.EnableBuffering(); // so we can read the body stream multiple times

    var requestBody = await new StreamReader(request.Body).ReadToEndAsync();
    request.Body.Position = 0; // reset the body stream to the beginning

    var responseBodyStream = new MemoryStream();
    var originalResponseBodyStream = context.Response.Body;
    context.Response.Body = responseBodyStream;

    await _next(context); // execute the next middleware in the pipeline

    responseBodyStream.Seek(0, SeekOrigin.Begin);
    var responseBody = await new StreamReader(responseBodyStream).ReadToEndAsync();
    responseBodyStream.Seek(0, SeekOrigin.Begin);

    var pathParams = new Dictionary<string, string>();
    foreach (var param in context.Request.RouteValues)
    {
        pathParams[param.Key] = param.Value.ToString();
    }

    var payload = BuildPayload(GoGinSDKType, start, context.Request, context.Response.StatusCode,
        requestBody, responseBody, context.Response.Headers.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.ToList()),
        pathParams, context.Request.Path);

    await PublishMessage(context, payload);

    // restore the original response body stream
    await responseBodyStream.CopyToAsync(originalResponseBodyStream);
    context.Response.Body = originalResponseBodyStream;
}

public Payload BuildPayload(string SDKType, DateTime trackingStart, HttpRequest req, int statusCode, byte[] reqBody, byte[] respBody, IHeaderDictionary respHeader, IDictionary<string, string> pathParams, string urlPath)
{
    if (req == null || req.HttpContext == null || req.HttpContext.Request == null || req.Scheme == null || req.Host == null)
    {
        // Early return with empty payload to prevent any null reference exceptions
        if (config.Debug)
        {
            Console.WriteLine("APIToolkit: null request or client or url while building payload.");
        }
        return new Payload();
    }

    string projectId = "";
    if (metadata != null)
    {
        projectId = metadata.ProjectId;
    }

    List<string> redactedHeaders = new List<string>();
    foreach (var v in config.RedactHeaders)
    {
        redactedHeaders.Add(v.ToLower());
    }

    TimeSpan since = DateTime.UtcNow.Subtract(trackingStart);

    var payload = new Payload
    {
        Duration = since,
        Host = req.Host.Host,
        Method = req.Method,
        PathParams = pathParams,
        ProjectID = projectId,
        ProtoMajor = req.ProtocolVersion.Major,
        ProtoMinor = req.ProtocolVersion.Minor,
        QueryParams = req.Query,
        RawURL = req.Scheme + "://" + req.Host + req.Path + req.QueryString,
        Referer = req.Headers["Referer"].ToString(),
        RequestBody = Redact(reqBody, config.RedactRequestBody),
        RequestHeaders = RedactHeaders(req.Headers, redactedHeaders),
        ResponseBody = Redact(respBody, config.RedactResponseBody),
        ResponseHeaders = RedactHeaders(respHeader, redactedHeaders),
        SdkType = SDKType,
        StatusCode = statusCode,
        Timestamp = DateTime.UtcNow,
        URLPath = urlPath,
    };

    return payload;
}