Skip to content

Instantly share code, notes, and snippets.

@joelverhagen
Created May 9, 2022 05:35
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save joelverhagen/dd9123a15716863bcfed21b2b30cb8d1 to your computer and use it in GitHub Desktop.
Save joelverhagen/dd9123a15716863bcfed21b2b30cb8d1 to your computer and use it in GitHub Desktop.
using System.Globalization;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using CsvHelper;
using CsvHelper.Configuration;
using CsvHelper.Configuration.Attributes;
using CsvHelper.TypeConversion;
var owner = "joelverhagen";
var repo = "MiniZip";
var workflowName = "Build";
var headShas = new HashSet<string> { "8be473b0ecaf85198430ae072d75055e5f1f6813", "69c8e595e490ce758c5d6e895066df0d944a06a8" };
var apiKey = "<TODO>";
using var httpClient = new HttpClient();
httpClient.BaseAddress = new Uri("https://api.github.com/");
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.v3+json"));
httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GitHubActionsDuration", "0.0.1"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($":{apiKey}")));
async Task<T> GetAsync<T>(string path)
{
Console.Write($"GET {path}");
try
{
using var response = await httpClient.GetAsync(path);
Console.Write($" {(int)response.StatusCode} {response.ReasonPhrase}");
if (response.Headers.TryGetValues("X-RateLimit-Remaining", out var rateLimitRemaining))
{
Console.Write($" (quota: {rateLimitRemaining.First()})");
}
Console.WriteLine();
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
return JsonSerializer.Deserialize<T>(content, new JsonSerializerOptions
{
Converters =
{
new NullableDateTimeConverterUsingDateTimeParse(),
new DateTimeConverterUsingDateTimeParse()
}
});
}
catch
{
Console.Error.WriteLine(" error");
throw;
}
}
var workflows = await GetAsync<WorkflowsResponse>($"/repos/{owner}/{repo}/actions/workflows");
var workflow = workflows.Workflows.Single(x => x.WorkflowName == workflowName);
var runIdToRun = new Dictionary<long, WorkflowRun>();
var perPage = 100;
var page = 1;
while (true)
{
var runs = await GetAsync<WorkflowsRunsResponse>($"/repos/{owner}/{repo}/actions/workflows/{workflow.WorkflowId}/runs?page={page}&per_page={perPage}");
foreach (var run in runs.Runs)
{
if (!headShas.Contains(run.RunHeadSha) || (run.RunConclusion is not null && run.RunConclusion != "success"))
{
continue;
}
runIdToRun[run.RunId] = run;
}
if (runs.Runs.Count < perPage)
{
break;
}
page++;
}
Console.WriteLine("Run counts:");
foreach (var group in runIdToRun.Values.GroupBy(x => x.RunHeadSha).OrderByDescending(x => x.Count()))
{
Console.WriteLine($" {group.Key}: {group.Count()} ({group.First().RunHeadBranch})");
}
var incompleteRuns = runIdToRun.Values.Where(x => x.RunConclusion is null);
if (incompleteRuns.Any())
{
Console.WriteLine($"There are {incompleteRuns.Count()} incomplete runs. Wait until they are complete.");
return;
}
using var outputFile = new FileStream($"out-{DateTimeOffset.Now:yyyy.MM.dd.HH.mm.ss}.csv", FileMode.Create);
using var fileWriter = new StreamWriter(outputFile);
using var csvWriter = new CsvWriter(fileWriter, CultureInfo.InvariantCulture);
csvWriter.Context.TypeConverterOptionsCache.AddOptions<DateTimeOffset>(new TypeConverterOptions
{
Formats = new[] { "yyyy-MM-dd HH:mm:ss.fff" }
});
csvWriter.WriteHeader<OutputRecord>();
csvWriter.NextRecord();
var headShaToOrder = new Dictionary<string, int>();
foreach (var run in runIdToRun.Values.OrderBy(x => x.RunCreatedAt))
{
headShaToOrder.TryGetValue(run.RunHeadSha, out var runOrder);
runOrder += 1;
headShaToOrder[run.RunHeadSha] = runOrder;
var jobs = await GetAsync<JobsResponse>(run.RunJobsUrl);
foreach (var job in jobs.Jobs)
{
foreach (var step in job.JobSteps)
{
csvWriter.WriteRecord(new OutputRecord(owner, repo, workflow, run, runOrder, job, step));
csvWriter.NextRecord();
}
}
}
class WorkflowsResponse
{
[JsonPropertyName("workflows")] public List<Workflow> Workflows { get; set; }
}
class Workflow
{
[JsonPropertyName("id")] public long WorkflowId { get; set; }
[JsonPropertyName("name")] public string WorkflowName { get; set; }
}
class WorkflowsRunsResponse
{
[JsonPropertyName("workflow_runs")] public List<WorkflowRun> Runs { get; set; }
}
class WorkflowRun
{
[JsonPropertyName("conclusion")] public string? RunConclusion { get; set; }
[JsonPropertyName("created_at")] public DateTimeOffset RunCreatedAt { get; set; }
[JsonPropertyName("head_branch")] public string RunHeadBranch { get; set; }
[JsonPropertyName("head_sha")] public string RunHeadSha { get; set; }
[JsonPropertyName("id")] public long RunId { get; set; }
[Ignore] [JsonPropertyName("jobs_url")] public string RunJobsUrl { get; set; }
[JsonPropertyName("name")] public string RunName { get; set; }
[JsonPropertyName("run_started_at")] public DateTimeOffset RunStartedAt { get; set; }
[JsonPropertyName("status")] public string RunStatus { get; set; }
[JsonPropertyName("updated_at")] public DateTimeOffset RunUpdatedAt { get; set; }
}
class JobsResponse
{
[JsonPropertyName("jobs")] public List<Job> Jobs { get; set; }
}
class Job
{
[JsonPropertyName("completed_at")] public DateTimeOffset? JobCompletedAt { get; set; }
[JsonPropertyName("conclusion")] public string? JobConclusion { get; set; }
[JsonPropertyName("id")] public long JobId { get; set; }
[JsonPropertyName("labels")] [TypeConverter(typeof(ToStringArrayConverter))] public List<string> JobLabels { get; set; }
[JsonPropertyName("name")] public string JobName { get; set; }
[JsonPropertyName("runner_name")] public string JobRunnerName { get; set; }
[JsonPropertyName("started_at")] public DateTimeOffset JobStartedAt { get; set; }
[JsonPropertyName("status")] public string JobStatus { get; set; }
[Ignore] [JsonPropertyName("steps")] public List<Step> JobSteps { get; set; }
}
class Step
{
[JsonPropertyName("completed_at")] public DateTimeOffset? StepCompletedAt { get; set; }
[JsonPropertyName("conclusion")] public string? StepConclusion { get; set; }
[JsonPropertyName("name")] public string StepName { get; set; }
[JsonPropertyName("number")] public int StepNumber { get; set; }
[JsonPropertyName("started_at")] public DateTimeOffset? StepStartedAt { get; set; }
[JsonPropertyName("status")] public string StepStatus { get; set; }
public double? StepDurationSeconds => StepCompletedAt.HasValue && StepStartedAt.HasValue ? (StepCompletedAt.Value - StepStartedAt.Value).TotalSeconds : null;
}
record OutputRecord(string Owner, string Repo, Workflow Workflow, WorkflowRun Run, int RunOrder, Job Job, Step Step);
class NullableDateTimeConverterUsingDateTimeParse : JsonConverter<DateTimeOffset?>
{
public override DateTimeOffset? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
var value = reader.GetString();
if (value is null)
{
return null;
}
return DateTimeOffset.Parse(value).ToUniversalTime();
}
public override void Write(Utf8JsonWriter writer, DateTimeOffset? value, JsonSerializerOptions options)
{
writer.WriteStringValue(value?.ToString());
}
}
class DateTimeConverterUsingDateTimeParse : JsonConverter<DateTimeOffset>
{
public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
return DateTimeOffset.Parse(reader.GetString()).ToUniversalTime();
}
public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options)
{
writer.WriteStringValue(value.ToString());
}
}
class ToStringArrayConverter : TypeConverter
{
public override object ConvertFromString(string text, IReaderRow row, MemberMapData memberMapData)
{
return new List<string>(text.Split(','));
}
public override string ConvertToString(object value, IWriterRow row, MemberMapData memberMapData)
{
return string.Join(',', ((List<string>)value).ToArray());
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment