Skip to content

Instantly share code, notes, and snippets.

@Mark-Simulacrum
Created July 4, 2018 16:20
Show Gist options
  • Save Mark-Simulacrum/2a3cf12527d541a29f66316c55ec54f8 to your computer and use it in GitHub Desktop.
Save Mark-Simulacrum/2a3cf12527d541a29f66316c55ec54f8 to your computer and use it in GitHub Desktop.
crater hunter, some parts taken from github.com/rust-lang-nursery/crater
extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate futures;
extern crate percent_encoding;
extern crate reqwest;
extern crate serde_json;
extern crate tokio_core;
extern crate url;
use std::cmp::min;
use std::collections::HashMap;
use std::fmt::Write;
use std::fs;
use std::path::PathBuf;
macro_rules! string_enum {
(pub enum $name:ident { $($item:ident => $str:expr,)* }) => {
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Copy, Clone)]
pub enum $name {
$($item,)*
}
impl ::std::str::FromStr for $name {
type Err = ();
fn from_str(s: &str) -> Result<$name, ()> {
Ok(match s {
$($str => $name::$item,)*
s => panic!("invalid {}: {}", stringify!($name), s),
})
}
}
impl ::std::fmt::Display for $name {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
write!(f, "{}", self.to_str())
}
}
impl $name {
pub fn to_str(&self) -> &'static str {
match *self {
$($name::$item => $str,)*
}
}
pub fn possible_values() -> &'static [&'static str] {
&[$($str,)*]
}
}
}
}
use reqwest::Client;
fn main() -> Result<(), Box<std::error::Error>> {
let mut nll: TestResults = serde_json::from_slice(&fs::read("results-nll-1.json")?)?;
let master: TestResults = serde_json::from_slice(&fs::read("results-pr-51762.json")?)?;
let mut master_map = HashMap::new();
for (idx, res) in master.crates.iter().enumerate() {
assert!(master_map.insert(res.name.clone(), idx).is_none());
}
for res in &mut nll.crates {
if !master_map.contains_key(&res.name) {
// skip
continue;
}
let master_res = &master.crates[master_map[&res.name]];
if let Some(result) = &master_res.runs[0] {
res.runs[0] = Some(result.clone());
let cmp = match (result.res, res.runs[1].clone().unwrap().res) {
(TestResult::BuildFail, TestResult::BuildFail) => Comparison::SameBuildFail,
(TestResult::TestFail, TestResult::TestFail) => Comparison::SameTestFail,
(TestResult::TestSkipped, TestResult::TestSkipped) => Comparison::SameTestSkipped,
(TestResult::TestPass, TestResult::TestPass) => Comparison::SameTestPass,
(TestResult::BuildFail, TestResult::TestFail)
| (TestResult::BuildFail, TestResult::TestSkipped)
| (TestResult::BuildFail, TestResult::TestPass)
| (TestResult::TestFail, TestResult::TestPass) => Comparison::Fixed,
(TestResult::TestPass, TestResult::TestFail)
| (TestResult::TestPass, TestResult::BuildFail)
| (TestResult::TestSkipped, TestResult::BuildFail)
| (TestResult::TestFail, TestResult::BuildFail) => Comparison::Regressed,
(TestResult::TestFail, TestResult::TestSkipped)
| (TestResult::TestPass, TestResult::TestSkipped)
| (TestResult::TestSkipped, TestResult::TestFail)
| (TestResult::TestSkipped, TestResult::TestPass) => {
panic!("can't compare");
}
};
res.res = cmp;
}
if let Some(run) = &mut res.runs[0] {
run.log = format!(
"https://cargobomb-reports.s3.amazonaws.com/pr-51762/{}",
run.log
);
}
if let Some(run) = &mut res.runs[1] {
run.log = format!(
"https://cargobomb-reports.s3.amazonaws.com/nll-1/{}",
run.log
);
}
}
let mut count = 0;
for res in &nll.crates {
if res.res == Comparison::Regressed {
if let Some(_) = &res.runs[1] {
let path = PathBuf::from(format!("logs/nll-1/{}", res.name));
if !path.exists() {
count += 1;
}
}
}
}
let mut logs = Vec::new();
let client = Client::new();
for res in &nll.crates {
let path = PathBuf::from(format!("logs/nll-1/{}", res.name));
if path.exists() {
continue;
}
if res.res == Comparison::Regressed {
if let Some(nll_run) = &res.runs[1] {
fs::create_dir_all(&path)?;
let url = format!("{}/log.txt", nll_run.log).replace("+", "%2B");
let mut response = client.get(&url).send()?;
if response.status().is_success() {
let content = response.text()?;
fs::write(path.join("nll.log"), content)?;
count -= 1;
if count % 100 == 0 {
println!("crates left: {}", count);
}
} else {
panic!("could not request {}: {:?}", nll_run.log, response);
}
}
}
}
for res in &nll.crates {
let path = PathBuf::from(format!("logs/nll-1/{}", res.name));
let log = match fs::read_to_string(path.join("nll.log")) {
Ok(s) => s,
Err(_) => continue,
};
if !log.contains("internal compiler error") {
continue;
}
logs.push((res, log));
}
let mut errors = Vec::new();
for (res, log) in &logs {
const ICE: &str = "internal compiler error";
let error = log.lines()
.find(|l| l.contains(&format!("{}: ", ICE)))
.unwrap_or_else(|| {
panic!("could not find ICE in log for {}", res.name);
});
let source = if error.contains("universal_regions.rs:825") {
let start = error.find(&format!("{}: ", ICE)).unwrap();
let start = start + format!("{}: ", ICE).len();
let end = start + error[start..].find(": ").unwrap();
&error[start..end]
} else {
let start = error.find(&format!("{}: ", ICE)).unwrap();
let start = start + format!("{}: ", ICE).len();
&error[start..]
};
errors.push((source, res.name.clone(), res));
}
errors.sort_by_key(|c| c.0.clone());
let mut current_category = String::new();
let mut report_md = String::new();
for (source, _, res) in errors {
let log_url = format!("{}/log.txt", res.runs[1].as_ref().unwrap().log).replace("+", "%2B");
let category = source[..min(60, source.len())].to_owned();
if category != current_category {
writeln!(report_md, "#### {}", category)?;
current_category = category;
}
writeln!(
report_md,
" - [{}]({}): [log]({})",
res.name, res.url, log_url,
)?;
}
fs::write("report.md", report_md)?;
println!("nll results: {}", nll.crates.len());
println!("master results: {}", master.crates.len());
let out = serde_json::to_string(&nll)?;
fs::write("results-merged.json", &out)?;
Ok(())
}
#[derive(Serialize, Deserialize)]
pub struct TestResults {
crates: Vec<CrateResult>,
}
#[derive(Serialize, Deserialize)]
struct CrateResult {
name: String,
url: String,
res: Comparison,
runs: [Option<BuildTestResult>; 2],
}
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
enum Comparison {
Regressed,
Fixed,
Skipped,
Unknown,
SameBuildFail,
SameTestFail,
SameTestSkipped,
SameTestPass,
}
#[derive(Clone, Serialize, Deserialize)]
struct BuildTestResult {
res: TestResult,
log: String,
}
string_enum!(pub enum TestResult {
BuildFail => "build-fail",
TestFail => "test-fail",
TestSkipped => "test-skipped",
TestPass => "test-pass",
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment