Skip to content

Instantly share code, notes, and snippets.

@haasad

haasad/biosphere_inconsistencies32to33.ipynb

Created October 25, 2016 13:41
Show Gist options
  • Save haasad/fda0e69da08959ab0b2e0368089f6bc4 to your computer and use it in GitHub Desktop.
Save haasad/fda0e69da08959ab0b2e0368089f6bc4 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
biosphere_inconsistencies32to33.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import os\n",
"import pickle\n",
"import json\n",
"import itertools\n",
"import collections\n",
"import pandas as pd\n",
"import brightway2 as bw\n",
"from bw2data.utils import recursive_str_to_unicode\n",
"from lxml import objectify"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Inconsistencies between ecoinvent ElementaryExchanges.xml 3.2 to 3.3"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# modified from bw2io.importers.ecospold2_biosphere\n",
"EMISSIONS_CATEGORIES = {\n",
" \"air\": \"emission\",\n",
" \"soil\": \"emission\",\n",
" \"water\": \"emission\",\n",
"}\n",
"\n",
"def extract(file_name):\n",
" def extract_flow_data(o):\n",
" ds = {\n",
" 'categories': (\n",
" o.compartment.compartment.text,\n",
" o.compartment.subcompartment.text\n",
" ),\n",
" 'code': o.get('id'),\n",
" 'name': o.name.text,\n",
" 'database': 'biosphere3',\n",
" 'exchanges': [],\n",
" 'unit': o.unitName.text,\n",
" }\n",
" ds[u\"type\"] = EMISSIONS_CATEGORIES.get(\n",
" ds['categories'][0], ds['categories'][0]\n",
" )\n",
" return ds\n",
"\n",
" fp = file_name\n",
" root = objectify.parse(open(fp, encoding='utf-8')).getroot()\n",
" return recursive_str_to_unicode([extract_flow_data(ds)\n",
" for ds in root.iterchildren()])"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"4018"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"biosphere_data32 = extract('ecoinvent elementary flows 3.2.xml')\n",
"len(biosphere_data32)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"4028"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"biosphere_data33 = extract('ecoinvent elementary flows 3.3.xml')\n",
"len(biosphere_data33)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"codes32 = {b['code'] for b in biosphere_data32}\n",
"codes33 = {b['code'] for b in biosphere_data33}"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"6"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"only32 = [b for b in biosphere_data32 if b['code'] in codes32.difference(codes33)]\n",
"len(only32)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>categories</th>\n",
" <th>code</th>\n",
" <th>database</th>\n",
" <th>exchanges</th>\n",
" <th>name</th>\n",
" <th>type</th>\n",
" <th>unit</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>(air, unspecified)</td>\n",
" <td>6900ccc1-4b7a-422a-9fad-7c31eb1fc46f</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Carbon dioxide, non-fossil, from calcination</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>(social, unspecified)</td>\n",
" <td>4dc23cfd-f444-41ee-a799-c0521b8e9145</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>venting of argon, crude, liquid</td>\n",
" <td>social</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>(social, unspecified)</td>\n",
" <td>48f9d166-c0ed-42a7-b3ea-854a16d76ad8</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>residual wood, dry</td>\n",
" <td>social</td>\n",
" <td>m3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>(social, unspecified)</td>\n",
" <td>f490bb47-46f6-4b2b-917f-6aa2be756db2</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>residual hardwood, wet</td>\n",
" <td>social</td>\n",
" <td>m3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>(social, unspecified)</td>\n",
" <td>61d8fdff-2140-408e-ba21-8e569aee8fc6</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>residual softwood, wet</td>\n",
" <td>social</td>\n",
" <td>m3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>(social, unspecified)</td>\n",
" <td>360cb202-1cd7-4836-b599-e04fc47d78d4</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>venting of nitrogen, liquid</td>\n",
" <td>social</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" categories code database \\\n",
"0 (air, unspecified) 6900ccc1-4b7a-422a-9fad-7c31eb1fc46f biosphere3 \n",
"1 (social, unspecified) 4dc23cfd-f444-41ee-a799-c0521b8e9145 biosphere3 \n",
"2 (social, unspecified) 48f9d166-c0ed-42a7-b3ea-854a16d76ad8 biosphere3 \n",
"3 (social, unspecified) f490bb47-46f6-4b2b-917f-6aa2be756db2 biosphere3 \n",
"4 (social, unspecified) 61d8fdff-2140-408e-ba21-8e569aee8fc6 biosphere3 \n",
"5 (social, unspecified) 360cb202-1cd7-4836-b599-e04fc47d78d4 biosphere3 \n",
"\n",
" exchanges name type unit \n",
"0 [] Carbon dioxide, non-fossil, from calcination emission kg \n",
"1 [] venting of argon, crude, liquid social kg \n",
"2 [] residual wood, dry social m3 \n",
"3 [] residual hardwood, wet social m3 \n",
"4 [] residual softwood, wet social m3 \n",
"5 [] venting of nitrogen, liquid social kg "
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pd.DataFrame(only32)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"16"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"only33 = [b for b in biosphere_data33 if b['code'] in codes33.difference(codes32)]\n",
"len(only33)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>categories</th>\n",
" <th>code</th>\n",
" <th>database</th>\n",
" <th>exchanges</th>\n",
" <th>name</th>\n",
" <th>type</th>\n",
" <th>unit</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>b10642d6-cd29-4cf0-9ebe-755902c9a3ae</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Myclobutanil</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>2b3cf153-0dd6-4b79-b0d4-b4df832f3a7e</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Chlorpyrifos methyl</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>ff32cb2a-4d63-435e-8787-58436bd36a83</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Boscalid</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>936015d2-e29c-4c0e-a4d0-cb3680e14452</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Fluazifop</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>910b7532-d541-45a5-a535-03f54997726e</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Acetamiprid</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>5e86b7ae-1d51-485c-be33-8c12c4ce4d2e</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Kaolin</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>794a9d6b-1ed0-486b-a6a9-d8a575c0aa9a</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Fosetyl</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>fd9e2be1-a651-4c40-b0bb-e7daa4f56ad4</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Chloropicrin</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>90ba480e-f2f2-4ee8-8299-da906df00058</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Acrinathrin</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>9682172f-8194-4efc-86e0-d81d189514a7</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Pirimiphos methyl</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>(natural resource, in ground)</td>\n",
" <td>478e8437-1c21-4032-8438-872a6b5ddcdf</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Water, unspecified natural origin</td>\n",
" <td>natural resource</td>\n",
" <td>m3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>63c262e9-5fd4-4160-98aa-cb0b3d0dc2d7</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Cyhalothrin</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>(soil, agricultural)</td>\n",
" <td>a0544534-e298-46a9-bbd2-949902ac1487</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>Flurochloridone</td>\n",
" <td>emission</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>(social, unspecified)</td>\n",
" <td>089c30c3-5028-4ef7-b800-48fea3528b7e</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>venting of argon, crude, liquid</td>\n",
" <td>social</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>(social, unspecified)</td>\n",
" <td>cff8fda2-d951-49ff-b5c1-a5b4965ceb6b</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>residual wood, dry</td>\n",
" <td>social</td>\n",
" <td>m3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>(social, unspecified)</td>\n",
" <td>e2709804-ea1f-40f4-851f-831ab3eba01e</td>\n",
" <td>biosphere3</td>\n",
" <td>[]</td>\n",
" <td>venting of nitrogen, liquid</td>\n",
" <td>social</td>\n",
" <td>kg</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" categories code \\\n",
"0 (soil, agricultural) b10642d6-cd29-4cf0-9ebe-755902c9a3ae \n",
"1 (soil, agricultural) 2b3cf153-0dd6-4b79-b0d4-b4df832f3a7e \n",
"2 (soil, agricultural) ff32cb2a-4d63-435e-8787-58436bd36a83 \n",
"3 (soil, agricultural) 936015d2-e29c-4c0e-a4d0-cb3680e14452 \n",
"4 (soil, agricultural) 910b7532-d541-45a5-a535-03f54997726e \n",
"5 (soil, agricultural) 5e86b7ae-1d51-485c-be33-8c12c4ce4d2e \n",
"6 (soil, agricultural) 794a9d6b-1ed0-486b-a6a9-d8a575c0aa9a \n",
"7 (soil, agricultural) fd9e2be1-a651-4c40-b0bb-e7daa4f56ad4 \n",
"8 (soil, agricultural) 90ba480e-f2f2-4ee8-8299-da906df00058 \n",
"9 (soil, agricultural) 9682172f-8194-4efc-86e0-d81d189514a7 \n",
"10 (natural resource, in ground) 478e8437-1c21-4032-8438-872a6b5ddcdf \n",
"11 (soil, agricultural) 63c262e9-5fd4-4160-98aa-cb0b3d0dc2d7 \n",
"12 (soil, agricultural) a0544534-e298-46a9-bbd2-949902ac1487 \n",
"13 (social, unspecified) 089c30c3-5028-4ef7-b800-48fea3528b7e \n",
"14 (social, unspecified) cff8fda2-d951-49ff-b5c1-a5b4965ceb6b \n",
"15 (social, unspecified) e2709804-ea1f-40f4-851f-831ab3eba01e \n",
"\n",
" database exchanges name type \\\n",
"0 biosphere3 [] Myclobutanil emission \n",
"1 biosphere3 [] Chlorpyrifos methyl emission \n",
"2 biosphere3 [] Boscalid emission \n",
"3 biosphere3 [] Fluazifop emission \n",
"4 biosphere3 [] Acetamiprid emission \n",
"5 biosphere3 [] Kaolin emission \n",
"6 biosphere3 [] Fosetyl emission \n",
"7 biosphere3 [] Chloropicrin emission \n",
"8 biosphere3 [] Acrinathrin emission \n",
"9 biosphere3 [] Pirimiphos methyl emission \n",
"10 biosphere3 [] Water, unspecified natural origin natural resource \n",
"11 biosphere3 [] Cyhalothrin emission \n",
"12 biosphere3 [] Flurochloridone emission \n",
"13 biosphere3 [] venting of argon, crude, liquid social \n",
"14 biosphere3 [] residual wood, dry social \n",
"15 biosphere3 [] venting of nitrogen, liquid social \n",
"\n",
" unit \n",
"0 kg \n",
"1 kg \n",
"2 kg \n",
"3 kg \n",
"4 kg \n",
"5 kg \n",
"6 kg \n",
"7 kg \n",
"8 kg \n",
"9 kg \n",
"10 m3 \n",
"11 kg \n",
"12 kg \n",
"13 kg \n",
"14 m3 \n",
"15 kg "
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pd.DataFrame(only33)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Ecoinvent 3.3"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"with open('../load_all_system_models/data/ei33/cutoff33.pickle', 'rb') as i:\n",
" cutoff33 = pickle.load(i)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"biosphere_exchanges33 = [e for e in itertools.chain.from_iterable([d['exchanges'] for d in cutoff33]) \n",
" if e['type']=='biosphere']"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"only33_codes = {i['code'] for i in only33}"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"Counter({('Acetamiprid', '910b7532-d541-45a5-a535-03f54997726e'): 8,\n",
" ('Acrinathrin', '90ba480e-f2f2-4ee8-8299-da906df00058'): 75,\n",
" ('Boscalid', 'ff32cb2a-4d63-435e-8787-58436bd36a83'): 81,\n",
" ('Chloropicrin', 'fd9e2be1-a651-4c40-b0bb-e7daa4f56ad4'): 75,\n",
" ('Chlorpyrifos methyl', '2b3cf153-0dd6-4b79-b0d4-b4df832f3a7e'): 86,\n",
" ('Cyhalothrin', '63c262e9-5fd4-4160-98aa-cb0b3d0dc2d7'): 5,\n",
" ('Fluazifop', '936015d2-e29c-4c0e-a4d0-cb3680e14452'): 6,\n",
" ('Flurochloridone', 'a0544534-e298-46a9-bbd2-949902ac1487'): 4,\n",
" ('Fosetyl', '794a9d6b-1ed0-486b-a6a9-d8a575c0aa9a'): 2,\n",
" ('Kaolin', '5e86b7ae-1d51-485c-be33-8c12c4ce4d2e'): 2,\n",
" ('Myclobutanil', 'b10642d6-cd29-4cf0-9ebe-755902c9a3ae'): 9,\n",
" ('Pirimiphos methyl', '9682172f-8194-4efc-86e0-d81d189514a7'): 3,\n",
" ('Water, unspecified natural origin',\n",
" '478e8437-1c21-4032-8438-872a6b5ddcdf'): 9})"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"collections.Counter([(e['name'], e['flow']) for e in biosphere_exchanges33\n",
" if e['flow'] in only33_codes])"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"with open('ecoinvent 33 new biosphere.json', 'r') as i:\n",
" ei33_new_biosphere = json.load(i)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"missing_codes = only33_codes.difference({i['code'] for i in ei33_new_biosphere})"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"Counter({('Water, unspecified natural origin',\n",
" '478e8437-1c21-4032-8438-872a6b5ddcdf'): 9})"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"collections.Counter([(e['name'], e['flow']) for e in biosphere_exchanges33\n",
" if e['flow'] in missing_codes])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Ecoinvent 3.2"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"with open('../load_all_system_models/data/ei32/cutoff32.pickle', 'rb') as i:\n",
" cutoff32 = pickle.load(i)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"only32_codes = {i['code'] for i in only32}"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"biosphere_exchanges32 = [e for e in itertools.chain.from_iterable([d['exchanges'] for d in cutoff32]) \n",
" if e['type']=='biosphere']"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"Counter({('Carbon dioxide, non-fossil, from calcination',\n",
" '6900ccc1-4b7a-422a-9fad-7c31eb1fc46f'): 68})"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"collections.Counter([(e['name'], e['flow']) for e in biosphere_exchanges32\n",
" if e['flow'] in only32_codes])"
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python [conda env:bw]",
"language": "python",
"name": "conda-env-bw-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment