Diffing UCR codes

README.md

🚓 wtdiff? iucr code edition

contents

raw files

• existing_codes.csv – data underlying the current version of python-iucr
• updated_codes.csv – updated codes provided in dec 2016 by the illinois state police

generated files

• existing_codes.enum.csv - raw file with line numbers
• updated_codes.enum.csv - raw file with line numbers
• different_statute.csv – ucr codes where the statute is different in the updated data
• new_code.csv – ucr codes in the update that are not in the existing data
• retired_code.csv – ucr codes where the corresponding statute has been repealed

refreshing the data

install csvkit (pip install csvkit), then

make clean
make all

diff.py

import csv
import sys

reader = csv.DictReader(sys.stdin)

out_header = ['code', 'offense', 'existing_ilcs', 'existing_line_number',
              'updated_ilcs', 'updated_line_number']

writer = csv.DictWriter(sys.stdout, fieldnames=out_header)
writer.writeheader()

next(reader)

for row in reader:
    if row['ilcs_reference'] != row['ILCS REFERENCE']:
        diff_row = {
            'code': row['CODE'],
            'offense': row['CATEGORY'],
            'existing_ilcs': row['ilcs_reference'],
            'existing_line_number': row['existing_codes_line_number'],
            'updated_ilcs': row['ILCS REFERENCE'],
            'updated_line_number': row['updated_codes_line_number'],
        }
        writer.writerow(diff_row)

different_statute.csv

existing_codes.csv

existing_codes.enum.csv

Makefile

SHELL := bash

OUTFILES=new_code.csv different_statute.csv retired_code.csv

.INTERMEDIATE : joined_codes.csv changed_codes.csv

all : $(OUTFILES)

clean :
	rm $(OUTFILES)

%.enum.csv : %.csv
	csvgrep -c 1 -r '.*' -l $< | \
	perl -pe 's/line_number/$*_line_number/' > $@

joined_codes.csv : updated_codes.enum.csv existing_codes.enum.csv
	csvjoin --left -c CODE,code $^ > $@

changed_codes.csv : joined_codes.csv
	cat $< | python diff.py > $@

new_code.csv : changed_codes.csv
	csvgrep -c existing_line_number -r '\d' -i $< > $@

different_statute.csv : changed_codes.csv
	csvgrep -c existing_line_number -r '\d' $< | \
	csvgrep -c updated_ilcs -m "retained for historical purposes" -i > $@

retired_code.csv : changed_codes.csv
	csvgrep -c updated_ilcs -m "retained for historical purposes" $< > $@

new_code.csv

retired_code.csv

updated_codes.csv

updated_codes.enum.csv