Skip to content

Instantly share code, notes, and snippets.

@nattaylor

nattaylor/out.php

Last active July 20, 2018 11:53
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save nattaylor/8ed8c65eda4dc7c966498cf70f1008fd to your computer and use it in GitHub Desktop.
Save nattaylor/8ed8c65eda4dc7c966498cf70f1008fd to your computer and use it in GitHub Desktop.
Download ZIP
Zoning Board of Appeal Decisions - City of Boston
Raw
out.php
<!DOCTYPE html>
<html>
<head>
<title>Zoning Board of Appeal Decisions - City of Boston</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
body {
font-family: sans-serif;
}
table {
border-collapse: collapse;
}
td, th {
border:1px solid #ccc;
}
</style>
</head>
<body>
<h1>Zoning Board of Appeal Decisions - City of Boston</h1>
<p>This an aggregation of the information available at <a href="https://www.boston.gov/departments/inspectional-services/zoning-board-appeal-decisions">https://www.boston.gov/departments/inspectional-services/zoning-board-appeal-decisions</a>. Appeals that haven't been fully aggregated yet may appear in the tables not in the details, and the details may contain typos due to the aggregation process. For more information, contact <a href="mailto:nattaylor@gmail.com">nattaylor@gmail.com</a>.</p>
<?php
$appeals = array_map(function($line){return explode("\t",$line);} ,explode("\n", file_get_contents('appeals.tsv')));
$totals = array();
foreach($appeals as $appeal) {
if($appeal[2] == "defer") continue;
if(!isset($totals[$appeal[4]])) {
$totals[$appeal[4]] = array("approvals"=>0, "denials"=>0, "total"=>0, "ratio"=>0.0);
}
if($appeal[2]=="approve") {
$totals[$appeal[4]]["approvals"]+=1;
} else if ($appeal[2]=="deny") {
$totals[$appeal[4]]["denials"]+=1;
}
$totals[$appeal[4]]["total"]+=1;
$totals[$appeal[4]]["ratio"]=strval(round($totals[$appeal[4]]["approvals"]/$totals[$appeal[4]]["total"],2)*100)."%";
}
ksort($totals);
echo "<p><strong>Table of Contents</strong></p><ul>\n";
foreach(range(1,21) as $ward) {
echo "<li><a href=\"#ward$ward\">Ward $ward</a></li>";
}
echo "<li><a href=\"#details\">Details</a></li>";
echo "</ul>\n";
echo "<h2>Summary</h2>";
echo "<table>\n<thead><tr><th>Ward</th><th>Approvals</th><th>Denials</th><th>Total</th><th>Rate</th></tr></thead>\n<tbody>\n";
foreach($totals as $ward=>$total) {
echo "<tr><td>$ward</td><td>".implode("</td><td>", $total)."</td></tr>".PHP_EOL;
}
echo "</tbody></table>".PHP_EOL;
foreach(range(1,21) as $ward) {
echo "<h2 id=\"ward$ward\">Ward $ward</h2>".PHP_EOL;
echo "<table>\n<thead><tr><th>Appeal</th><th>Date</th><th>Vote</th><th>Address</th></tr></thead>\n<tbody>\n";
foreach($appeals as $appeal) {
if($appeal[2] == "defer") continue;
if($appeal[4] == "Ward $ward") {
array_pop($appeal);
$appeal[0] = "<a href=\"#{$appeal[0]}\">{$appeal[0]}</a>";
echo "<tr><td>".implode("</td><td>", $appeal)."</td></tr>".PHP_EOL;
}
}
echo "</tbody></table>".PHP_EOL;
}
echo "<h2 id=\"details\">Details</h2>\n";
exec("php -f process.php list.txt",$output);
echo implode("\n", $output);
?>
<a href="#">Return to top &Uparrow;</a>
</body>
</html>
Raw
process-site.php
<?php
/** Parses the data from https://www.boston.gov/departments/inspectional-services/zoning-board-appeal-decisions
* $str needs to be of this form:
January 30, 2018
APPROVED DECISIONS
BOA-775971 268 Bremen Street, Ward 1
BOA-766613 109 Beech Street, Ward 20
BOA-734095 34 Concord Square, Ward 4
DENIED DECISIONS
BOA-747215 117 Bolton Street, Ward 6
BOA-700987 5 Cypher Street, Ward 6
BOA-736227 18 Greenwich Street, Ward 15
*/
$str = "";
//Split on date
$hearings = preg_split("/\n([A-Z][a-z]* [0-9]{1,2}, [0-9]{4})\n/", $str, -1, PREG_SPLIT_DELIM_CAPTURE);
for($i=1; $i<count($hearings); $i+=2) {
$monthMapping = array("January"=>'01', "February"=>'02', "March"=>'03', "April"=>'04', "May"=>'05', "June"=>'06', "July"=>'07', "August"=>'08', "September"=>'09', "October"=>'10', "November"=>'11', "December"=>'12');
preg_match("/(January|February|March|April|May|June|July|August|September|October|November|December) ([0-9]{1,2}), ([0-9]{4})/", $hearings[$i], $dateMatches);
$date = "{$dateMatches[3]}-{$monthMapping[$dateMatches[1]]}-{$dateMatches[2]}";
$splitHearings = preg_split("/DENIED DECISIONS/", $hearings[$i+1]);
$splitHearings[0] = str_replace("APPROVED DECISIONS\n", "", $splitHearings[0]);
for($j=0;$j<2;$j++) {
if(!isset($splitHearings[$j])) {continue;}
$appeals = explode("\n", trim($splitHearings[$j]));
$vote = ($j == 0) ? "approve" : "deny";
foreach($appeals as $appeal) {
$parts = preg_split("/(BOA-[0-9]*) /", $appeal, -1, PREG_SPLIT_DELIM_CAPTURE);
//$parts = array_slice($parts, 1);
$parts = array_slice(array_merge(array($parts[1], $date, $vote), preg_split("/, (Ward.*)/", $parts[2], -1, PREG_SPLIT_DELIM_CAPTURE)),0,-1);
echo "<tr><td>".implode("</td><td>", $parts)."</td></tr>".PHP_EOL;
}
}
}
Raw
process.php
<?php
/**
* Parse a Zoning Board hearing minutes into structured data.
* - https://www.boston.gov/departments/inspectional-services/zoning-board-appeal#meeting-minutes
*
*/
$files = $argv[1] == "list.txt" ? explode("\n", file_get_contents("list.txt")) : array($argv[1]);
$mergedCases = array();
foreach ($files as $file) {
/**
* Ugly date parsing.
*/
$monthMapping = array("january"=>'01', "february"=>'02', "march"=>'03', "april"=>'04', "may"=>'05', "june"=>'06', "july"=>'07', "august"=>'08', "september"=>'09', "october"=>'10', "november"=>'11', "december"=>'12');
preg_match("/(january|february|march|april|may|june|july|august|september|october|november|december)_([0-9]{1,2})_([0-9]{4})/", $file, $dateMatches);
$date = "{$dateMatches[3]}-{$monthMapping[$dateMatches[1]]}-{$dateMatches[2]}";
preg_match_all("/.*?\nCase:.*?/s", preg_replace("/\n[0-9]{1,2}\n\n/", " ", strip_tags(file_get_contents($file))), $output_array);
/**
* Loop to insert newlines as delimiters
*/
$split = array_map(function($case){
$case = str_replace("\n", " ", $case);
$fields = array(
"Case",
"Address",
"Article(s)",
"Purpose",
"Discussion",
"Testimony",
"Documents/Exhibits",
"Discussion/Vote",
"Vote",
"Applicant",
"Ward"
);
foreach($fields as $field) {
$case = str_replace($field.": ", "\n$field: ", $case);
}
//Special Cases
$case = str_replace("Address:", "\nAddress: ", $case);
$case = str_replace("Votes:", "\nVote: ", $case);
$case = preg_replace("/, Ward ([0-9]{1,2})/", "\nWard: $1", $case);
$case = preg_replace("/Case:$/", "", $case);
return "Case:$case";
}, $output_array[0]);
/**
* Loop to parse out the values
*/
$cases = array_map(function($case) use ($date) {
$parsedCase = array();
$lines = explode("\n", $case);
$fields = array(
"Case",
"Address",
"Article\(s\)",
"Purpose",
"Discussion",
"Testimony",
"Documents\/Exhibits",
"Discussion\/Vote",
"Vote",
"Applicant",
"Ward"
);
foreach($lines as $line) {
foreach($fields as $field) {
preg_match("/^$field: (.*)/", $line, $matches);
if( count($matches)>0 ) {
$parsedCase[$field]=rtrim($matches[1]," ,");
if($field == "Vote") {
if (strstr($matches[1], "deny") || strstr($matches[1], "denial")) {
$parsedCase['vote_heuristic']='deny';
} else if (strstr($matches[1], "defer")) {
$parsedCase['vote_heuristic']='defer';
} else if (strstr($matches[1], "approv")) {
$parsedCase['vote_heuristic']='approve';
}else {
$parsedCase['vote_heuristic']='unknown';
}
}
continue;
}
}
}
$parsedCase['Date']=$date;
return $parsedCase;
}, $split);
if(count($files)==1) {
foreach ($cases as $case) {
if ($case['vote_heuristic'] == 'defer') {
continue;
}
//case, date, vote_heuristic, address, ward
$structuredCase = array($case['Case'],$case['Date'],$case['vote_heuristic'],$case['Address'],"Ward ".$case['Ward']);
echo implode(" ", $structuredCase).PHP_EOL;
}
exit();
} else {
$mergedCases = array_merge($cases, $mergedCases);
}
}
foreach ($mergedCases as $case) {
$fields = array(
"Date",
"Ward",
"Applicant",
"Address",
"Article\(s\)",
"Purpose",
"Discussion",
"Testimony",
"Documents\/Exhibits",
"Discussion\/Vote",
"Vote"
);
if(strlen($case["Case"])>10) continue;
echo "<h3 id=\"{$case["Case"]}\">{$case["Case"]}</h3>\n";
foreach ($fields as $field) {
if(isset($case[$field])) {
echo "<p><strong>$field:</strong> {$case[$field]}</p>\n";
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment