import_irs_soi_to_dolt.sh

# Init a new repo
dolt init

# Import 2011 data
dolt checkout -b 2011
dolt schema import column_mappings 2011_mapping.csv
dolt table import -r column_mappings 2011_mapping.csv

dolt add column_mappings
dolt commit -m "import 2011 column mappings"

dolt schema import --create --float-threshold 1 --map 2011_mapping.json  --pks state,zip,agi_category allagi zipcode2011/allagi.csv
dolt table import -m 2011_mapping.json -r allagi zipcode2011/allagi.csv

dolt add allagi
dolt commit -m "import 2011 allagi data"

dolt schema import --create --float-threshold 1 --map 2011_mapping.json  --pks state,zip,agi_category allnoagi zipcode2011/allnoagi.csv
dolt table import -m 2011_mapping.json -r allnoagi zipcode2011/allnoagi.csv

dolt add allnoagi
dolt commit -m "import 2011 allnoagi data"

dolt schema import --create --float-threshold 1 --map 2011_mapping.json  --pks state,zip,agi_category noagi zipcode2011/noagi.csv
dolt table import -m 2011_mapping.json -r noagi zipcode2011/noagi.csv

dolt add noagi
dolt commit -m "import 2011 noagi data"

dolt checkout master
dolt merge 2011


# Import 2012 data
dolt checkout -b 2012
dolt table import -r column_mappings 2012_mapping.csv

dolt add column_mappings
dolt commit -m "import 2012 column mappings"

dolt table rm noagi
dolt add noagi
dolt commit -m "remove data not in 2012 dataset"

dolt schema import --update --float-threshold 1 --map 2012_mapping.json  --pks state,zip,agi_category allagi zipcode2012/allagi.csv
dolt table import -m 2012_mapping.json -r allagi zipcode2012/allagi.csv

dolt add allagi
dolt commit -m "import 2012 allagi data"

dolt schema import --update --float-threshold 1 --map 2012_mapping.json  --pks state,zip,agi_category allnoagi zipcode2012/allnoagi.csv
dolt table import -m 2012_mapping.json -r allnoagi zipcode2012/allnoagi.csv

dolt add allnoagi
dolt commit -m "import 2012 allnoagi data"

dolt checkout master
dolt merge 2012


# Import 2013 data
dolt checkout -b 2013
dolt table import -r column_mappings 2013_mapping.csv

dolt add column_mappings
dolt commit -m "import 2013 column mappings"

dolt schema import --update --float-threshold 1 --map 2013_mapping.json  --pks state,zip,agi_category allagi zipcode2013/allagi.csv
dolt table import -m 2013_mapping.json -r --continue allagi zipcode2013/allagi.csv

dolt add allagi
dolt commit -m "import 2013 allagi data"

dolt schema import --update --float-threshold 1 --map 2013_mapping.json  --pks state,zip,agi_category allnoagi zipcode2013/allnoagi.csv
dolt table import -m 2013_mapping.json -r --continue allnoagi zipcode2013/allnoagi.csv

dolt add allnoagi
dolt commit -m "import 2013 allnoagi data"

dolt schema import --update --float-threshold 1 --map 2013_mapping.json  --pks state,zip,agi_category zipcodenoagi zipcode2013/zipcodenoagi.csv
dolt table import -m 2013_mapping.json -r --continue zipcodenoagi zipcode2013/zipcodenoagi.csv

dolt add zipcodenoagi
dolt commit -m "import 2013 zipcodenoagi data"

dolt schema import --update --float-threshold 1 --map 2013_mapping.json  --pks state,zip,agi_category zipcodeagi zipcode2013/zipcodeagi.csv
dolt table import -m 2013_mapping.json -r --continue zipcodeagi zipcode2013/zipcodeagi.csv

dolt add zipcodeagi
dolt commit -m "import 2013 zipcodeagi data"

dolt checkout master
dolt merge 2013


# Import 2014 data
dolt checkout -b 2014
dolt table import -r column_mappings 2014_mapping.csv

dolt add column_mappings
dolt commit -m "update 2014 column mappings"

dolt table rm zipcodeagi
dolt table rm zipcodenoagi

dolt schema import --update --float-threshold 1 --map 2014_mapping.json  --pks state,zip,agi_category allagi zipcode2014/allagi.csv
dolt table import -m 2014_mapping.json -r --continue allagi zipcode2014/allagi.csv

dolt add allagi
dolt commit -m "import 2014 allagi data"

dolt schema import --update --float-threshold 1 --map 2014_mapping.json  --pks state,zip,agi_category allnoagi zipcode2014/allnoagi.csv
dolt table import -m 2014_mapping.json -r --continue allnoagi zipcode2014/allnoagi.csv

dolt add allnoagi
dolt commit -m "import 2014 allnoagi data"

dolt checkout master
dolt merge 2014


# Import 2015 data
dolt checkout -b 2015
dolt table import -r column_mappings 2015_mapping.csv

dolt add column_mappings
dolt commit -m "update 2015 column mappings"

dolt schema import --update --float-threshold 1 --map 2015_mapping.json  --pks state,zip,agi_category allagi zipcode2015/allagi.csv
dolt table import -m 2015_mapping.json -r --continue allagi zipcode2015/allagi.csv

dolt add allagi
dolt commit -m "import 2015 allagi data"

dolt schema import --update --float-threshold 1 --map 2015_mapping.json  --pks state,zip,agi_category allnoagi zipcode2015/allnoagi.csv
dolt table import -m 2015_mapping.json -r --continue allnoagi zipcode2015/allnoagi.csv

dolt add allnoagi
dolt commit -m "import 2015 allnoagi data"

dolt checkout master
dolt merge 2015


# Import 2016 data
dolt checkout -b 2016
dolt table import -r column_mappings 2016_mapping.csv

dolt add column_mappings
dolt commit -m "update 2016 column mappings"

dolt schema import --update --keep-types --float-threshold 1 --map 2016_mapping.json  --pks state,zip,agi_category allagi zipcode2016/allagi.csv
dolt table import -m 2016_mapping.json -r --continue allagi zipcode2016/allagi.csv
# In 2016 and only in 2016 did they change zip from "00000" to "0".  Change back for consistency
dolt sql -q "UPDATE allagi SET zip = '00000' WHERE zip = '0'"

dolt add allagi
dolt commit -m "import 2016 allagi data"

dolt schema import --update --keep-types --float-threshold 1 --map 2016_mapping.json  --pks state,zip,agi_category allnoagi zipcode2016/allnoagi.csv
dolt table import -m 2016_mapping.json -r --continue allnoagi zipcode2016/allnoagi.csv
# In 2016 and only in 2016 did they change zip from "00000" to "0".  Change back for consistency
dolt sql -q "UPDATE allnoagi SET zip = '00000' WHERE zip = '0'"

dolt add allnoagi
dolt commit -m "import 2016 allnoagi data"

dolt checkout master
dolt merge 2016


# Import 2017 data
dolt checkout -b 2017
dolt table import -r column_mappings 2017_mapping.csv

dolt add column_mappings
dolt commit -m "update 2017 column mappings"

dolt schema import --update --float-threshold 1 --map 2017_mapping.json  --pks state,zip,agi_category allagi zipcode2017/allagi.csv
dolt table import -m 2017_mapping.json -r --continue allagi zipcode2017/allagi.csv

dolt add allagi
dolt commit -m "import 2017 allagi data"

dolt schema import --update --float-threshold 1 --map 2017_mapping.json  --pks state,zip,agi_category allnoagi zipcode2017/allnoagi.csv
dolt table import -m 2017_mapping.json -r --continue allnoagi zipcode2017/allnoagi.csv

dolt add allnoagi
dolt commit -m "import 2017 allnoagi data"

dolt checkout master
dolt merge 2017