Created
May 9, 2022 05:35
-
-
Save joelverhagen/dd9123a15716863bcfed21b2b30cb8d1 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System.Globalization; | |
using System.Net.Http.Headers; | |
using System.Text; | |
using System.Text.Json; | |
using System.Text.Json.Serialization; | |
using CsvHelper; | |
using CsvHelper.Configuration; | |
using CsvHelper.Configuration.Attributes; | |
using CsvHelper.TypeConversion; | |
var owner = "joelverhagen"; | |
var repo = "MiniZip"; | |
var workflowName = "Build"; | |
var headShas = new HashSet<string> { "8be473b0ecaf85198430ae072d75055e5f1f6813", "69c8e595e490ce758c5d6e895066df0d944a06a8" }; | |
var apiKey = "<TODO>"; | |
using var httpClient = new HttpClient(); | |
httpClient.BaseAddress = new Uri("https://api.github.com/"); | |
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.v3+json")); | |
httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("GitHubActionsDuration", "0.0.1")); | |
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($":{apiKey}"))); | |
async Task<T> GetAsync<T>(string path) | |
{ | |
Console.Write($"GET {path}"); | |
try | |
{ | |
using var response = await httpClient.GetAsync(path); | |
Console.Write($" {(int)response.StatusCode} {response.ReasonPhrase}"); | |
if (response.Headers.TryGetValues("X-RateLimit-Remaining", out var rateLimitRemaining)) | |
{ | |
Console.Write($" (quota: {rateLimitRemaining.First()})"); | |
} | |
Console.WriteLine(); | |
response.EnsureSuccessStatusCode(); | |
var content = await response.Content.ReadAsStringAsync(); | |
return JsonSerializer.Deserialize<T>(content, new JsonSerializerOptions | |
{ | |
Converters = | |
{ | |
new NullableDateTimeConverterUsingDateTimeParse(), | |
new DateTimeConverterUsingDateTimeParse() | |
} | |
}); | |
} | |
catch | |
{ | |
Console.Error.WriteLine(" error"); | |
throw; | |
} | |
} | |
var workflows = await GetAsync<WorkflowsResponse>($"/repos/{owner}/{repo}/actions/workflows"); | |
var workflow = workflows.Workflows.Single(x => x.WorkflowName == workflowName); | |
var runIdToRun = new Dictionary<long, WorkflowRun>(); | |
var perPage = 100; | |
var page = 1; | |
while (true) | |
{ | |
var runs = await GetAsync<WorkflowsRunsResponse>($"/repos/{owner}/{repo}/actions/workflows/{workflow.WorkflowId}/runs?page={page}&per_page={perPage}"); | |
foreach (var run in runs.Runs) | |
{ | |
if (!headShas.Contains(run.RunHeadSha) || (run.RunConclusion is not null && run.RunConclusion != "success")) | |
{ | |
continue; | |
} | |
runIdToRun[run.RunId] = run; | |
} | |
if (runs.Runs.Count < perPage) | |
{ | |
break; | |
} | |
page++; | |
} | |
Console.WriteLine("Run counts:"); | |
foreach (var group in runIdToRun.Values.GroupBy(x => x.RunHeadSha).OrderByDescending(x => x.Count())) | |
{ | |
Console.WriteLine($" {group.Key}: {group.Count()} ({group.First().RunHeadBranch})"); | |
} | |
var incompleteRuns = runIdToRun.Values.Where(x => x.RunConclusion is null); | |
if (incompleteRuns.Any()) | |
{ | |
Console.WriteLine($"There are {incompleteRuns.Count()} incomplete runs. Wait until they are complete."); | |
return; | |
} | |
using var outputFile = new FileStream($"out-{DateTimeOffset.Now:yyyy.MM.dd.HH.mm.ss}.csv", FileMode.Create); | |
using var fileWriter = new StreamWriter(outputFile); | |
using var csvWriter = new CsvWriter(fileWriter, CultureInfo.InvariantCulture); | |
csvWriter.Context.TypeConverterOptionsCache.AddOptions<DateTimeOffset>(new TypeConverterOptions | |
{ | |
Formats = new[] { "yyyy-MM-dd HH:mm:ss.fff" } | |
}); | |
csvWriter.WriteHeader<OutputRecord>(); | |
csvWriter.NextRecord(); | |
var headShaToOrder = new Dictionary<string, int>(); | |
foreach (var run in runIdToRun.Values.OrderBy(x => x.RunCreatedAt)) | |
{ | |
headShaToOrder.TryGetValue(run.RunHeadSha, out var runOrder); | |
runOrder += 1; | |
headShaToOrder[run.RunHeadSha] = runOrder; | |
var jobs = await GetAsync<JobsResponse>(run.RunJobsUrl); | |
foreach (var job in jobs.Jobs) | |
{ | |
foreach (var step in job.JobSteps) | |
{ | |
csvWriter.WriteRecord(new OutputRecord(owner, repo, workflow, run, runOrder, job, step)); | |
csvWriter.NextRecord(); | |
} | |
} | |
} | |
class WorkflowsResponse | |
{ | |
[JsonPropertyName("workflows")] public List<Workflow> Workflows { get; set; } | |
} | |
class Workflow | |
{ | |
[JsonPropertyName("id")] public long WorkflowId { get; set; } | |
[JsonPropertyName("name")] public string WorkflowName { get; set; } | |
} | |
class WorkflowsRunsResponse | |
{ | |
[JsonPropertyName("workflow_runs")] public List<WorkflowRun> Runs { get; set; } | |
} | |
class WorkflowRun | |
{ | |
[JsonPropertyName("conclusion")] public string? RunConclusion { get; set; } | |
[JsonPropertyName("created_at")] public DateTimeOffset RunCreatedAt { get; set; } | |
[JsonPropertyName("head_branch")] public string RunHeadBranch { get; set; } | |
[JsonPropertyName("head_sha")] public string RunHeadSha { get; set; } | |
[JsonPropertyName("id")] public long RunId { get; set; } | |
[Ignore] [JsonPropertyName("jobs_url")] public string RunJobsUrl { get; set; } | |
[JsonPropertyName("name")] public string RunName { get; set; } | |
[JsonPropertyName("run_started_at")] public DateTimeOffset RunStartedAt { get; set; } | |
[JsonPropertyName("status")] public string RunStatus { get; set; } | |
[JsonPropertyName("updated_at")] public DateTimeOffset RunUpdatedAt { get; set; } | |
} | |
class JobsResponse | |
{ | |
[JsonPropertyName("jobs")] public List<Job> Jobs { get; set; } | |
} | |
class Job | |
{ | |
[JsonPropertyName("completed_at")] public DateTimeOffset? JobCompletedAt { get; set; } | |
[JsonPropertyName("conclusion")] public string? JobConclusion { get; set; } | |
[JsonPropertyName("id")] public long JobId { get; set; } | |
[JsonPropertyName("labels")] [TypeConverter(typeof(ToStringArrayConverter))] public List<string> JobLabels { get; set; } | |
[JsonPropertyName("name")] public string JobName { get; set; } | |
[JsonPropertyName("runner_name")] public string JobRunnerName { get; set; } | |
[JsonPropertyName("started_at")] public DateTimeOffset JobStartedAt { get; set; } | |
[JsonPropertyName("status")] public string JobStatus { get; set; } | |
[Ignore] [JsonPropertyName("steps")] public List<Step> JobSteps { get; set; } | |
} | |
class Step | |
{ | |
[JsonPropertyName("completed_at")] public DateTimeOffset? StepCompletedAt { get; set; } | |
[JsonPropertyName("conclusion")] public string? StepConclusion { get; set; } | |
[JsonPropertyName("name")] public string StepName { get; set; } | |
[JsonPropertyName("number")] public int StepNumber { get; set; } | |
[JsonPropertyName("started_at")] public DateTimeOffset? StepStartedAt { get; set; } | |
[JsonPropertyName("status")] public string StepStatus { get; set; } | |
public double? StepDurationSeconds => StepCompletedAt.HasValue && StepStartedAt.HasValue ? (StepCompletedAt.Value - StepStartedAt.Value).TotalSeconds : null; | |
} | |
record OutputRecord(string Owner, string Repo, Workflow Workflow, WorkflowRun Run, int RunOrder, Job Job, Step Step); | |
class NullableDateTimeConverterUsingDateTimeParse : JsonConverter<DateTimeOffset?> | |
{ | |
public override DateTimeOffset? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) | |
{ | |
var value = reader.GetString(); | |
if (value is null) | |
{ | |
return null; | |
} | |
return DateTimeOffset.Parse(value).ToUniversalTime(); | |
} | |
public override void Write(Utf8JsonWriter writer, DateTimeOffset? value, JsonSerializerOptions options) | |
{ | |
writer.WriteStringValue(value?.ToString()); | |
} | |
} | |
class DateTimeConverterUsingDateTimeParse : JsonConverter<DateTimeOffset> | |
{ | |
public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) | |
{ | |
return DateTimeOffset.Parse(reader.GetString()).ToUniversalTime(); | |
} | |
public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options) | |
{ | |
writer.WriteStringValue(value.ToString()); | |
} | |
} | |
class ToStringArrayConverter : TypeConverter | |
{ | |
public override object ConvertFromString(string text, IReaderRow row, MemberMapData memberMapData) | |
{ | |
return new List<string>(text.Split(',')); | |
} | |
public override string ConvertToString(object value, IWriterRow row, MemberMapData memberMapData) | |
{ | |
return string.Join(',', ((List<string>)value).ToArray()); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment