Created
August 23, 2016 11:04
-
-
Save pp-mo/f021a95d002837709c292273090f4b9f to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "metadata": { | |
| "collapsed": false | |
| }, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/plain": [ | |
| "'\\nCheck save works the same between iris-grib and iris.fileformats.grib(strict),\\nwhen testing all the FF example files from iris-test-data.\\n\\n'" | |
| ] | |
| }, | |
| "execution_count": 1, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "\"\"\"\n", | |
| "Check save works the same between iris-grib and iris.fileformats.grib(strict),\n", | |
| "when testing all the FF example files from iris-test-data.\n", | |
| "\n", | |
| "\"\"\"" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "metadata": { | |
| "collapsed": false | |
| }, | |
| "outputs": [ | |
| { | |
| "name": "stderr", | |
| "output_type": "stream", | |
| "text": [ | |
| "/home/h05/itpp/git/iris/iris_main/lib/iris/fileformats/grib/__init__.py:59: IrisDeprecation: The module iris.fileformats.grib is deprecated since v1.10. Please install the package 'iris_grib' package instead.\n", | |
| " \"The module iris.fileformats.grib is deprecated since v1.10. \"\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "import os\n", | |
| "import os.path\n", | |
| "import shutil\n", | |
| "from shutil import rmtree\n", | |
| "import tempfile\n", | |
| "from tempfile import mkdtemp\n", | |
| "\n", | |
| "import iris\n", | |
| "from iris.fileformats.grib import save_grib2 as old_irisffg_save\n", | |
| "import iris.tests\n", | |
| "from iris.tests import get_data_path as get_testdata_path\n", | |
| "import iris_grib\n", | |
| "from iris_grib import save_grib2 as new_irisgrib_save\n", | |
| "from nose import exc" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "def filesin(dirpath):\n", | |
| " assert os.path.isdir(dirpath)\n", | |
| " for name in os.listdir(dirpath):\n", | |
| " subpath = os.path.join(dirpath, name)\n", | |
| " if os.path.isdir(subpath):\n", | |
| " for result in filesin(subpath):\n", | |
| " yield result\n", | |
| " else:\n", | |
| " yield subpath\n", | |
| "\n", | |
| "def all_testdata_FF_files():\n", | |
| " ff_testfiles_path = get_testdata_path('FF')\n", | |
| " for filepath in filesin(ff_testfiles_path):\n", | |
| " yield filepath\n", | |
| "\n", | |
| "def all_testdata_FF_cubes():\n", | |
| " for i_file, filepath in enumerate(all_testdata_FF_files()):\n", | |
| " try:\n", | |
| " cubes = iris.load(filepath)\n", | |
| " except Exception as e:\n", | |
| " cubes = [iris.cube.Cube([1], long_name=\"FAILED : \" + str(e))]\n", | |
| " for i_cube, cube in enumerate(cubes):\n", | |
| " yield i_file, filepath, i_cube, cube" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 4, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "def save_result(cube, save_call):\n", | |
| " try:\n", | |
| " temp_output_dirpath = mkdtemp()\n", | |
| " temp_output_filepath = os.path.join(temp_output_dirpath, 'temp.grib2')\n", | |
| "\n", | |
| " try:\n", | |
| " save_call(cube, temp_output_filepath)\n", | |
| " exception = None\n", | |
| " except Exception as e:\n", | |
| " exception = e\n", | |
| "\n", | |
| " if exception is None:\n", | |
| " result_brief, exception_type, exception_text = ('ok', None, None)\n", | |
| " else:\n", | |
| " result_brief, exception_type, exception_text = \\\n", | |
| " ('FAIL', type(exception), str(exception))\n", | |
| " finally:\n", | |
| " rmtree(temp_output_dirpath, ignore_errors=True)\n", | |
| "\n", | |
| " return result_brief, exception_type, exception_text" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 5, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "def report_on(cube):\n", | |
| " old_msg, old_type, old_text = save_result(cube, old_irisffg_save)\n", | |
| " new_msg, new_type, new_text = save_result(cube, new_irisgrib_save)\n", | |
| " if old_msg == 'ok' and new_msg == 'ok':\n", | |
| " report = ' BOTH ok.'\n", | |
| " else:\n", | |
| " if old_msg == new_msg:\n", | |
| " report = ' BOTH FAILED'\n", | |
| " if old_type == new_type and old_text == new_text:\n", | |
| " report += ': SAME - {} {}'.format(new_type, new_text)\n", | |
| " else:\n", | |
| " report += ': DIFFERENT..'\n", | |
| " msg = '\\n {} {} {}'\n", | |
| " report += msg.format('old', old_type, old_text)\n", | |
| " report += msg.format('new', new_type, new_text)\n", | |
| " else:\n", | |
| " # One failed\n", | |
| " report = ' DIFFERENT SUCCESSES'\n", | |
| " msg = '\\n {} \"{}\" {} {}'\n", | |
| " report += msg.format('old', old_msg, old_type, old_text)\n", | |
| " report += msg.format('new', new_msg, new_type, new_text)\n", | |
| " return report" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 6, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "def test():\n", | |
| " print 'Iris version : ', iris.__version__\n", | |
| " print 'Old save routine from : ', old_irisffg_save.__module__\n", | |
| " print 'New save routine from : ', new_irisgrib_save.__module__\n", | |
| " print\n", | |
| " for i_file, filepath, i_cube, cube in all_testdata_FF_cubes():\n", | |
| " msg = 'file#{:02d} \"{}\" : cube#{:02d} - {}'\n", | |
| " print msg.format(i_file, filepath.rjust(40), i_cube, cube.summary(shorten=True))\n", | |
| " print report_on(cube)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 8, | |
| "metadata": { | |
| "collapsed": false | |
| }, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "Iris version : 1.10.0-DEV\n", | |
| "Old save routine from : iris.fileformats.grib\n", | |
| "New save routine from : iris_grib\n", | |
| "\n", | |
| "file#00 \"/home/h05/itpp/git/iris-test-data/test_data/FF/ancillary_fixed_length_header\" : cube#00 - FAILED : index out of bounds / (unknown) (-- : 1)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> Did not find one (and only one) x or y coord\n", | |
| "file#01 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field.ieee32\" : cube#00 - air_temperature / (K) (latitude: 73; longitude: 96)\n", | |
| " BOTH ok.\n", | |
| "file#01 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field.ieee32\" : cube#01 - air_temperature / (K) (latitude: 73; longitude: 96)\n", | |
| " BOTH FAILED: SAME - <class 'gribapi.GribInternalError'> Encoding invalid\n", | |
| "file#01 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field.ieee32\" : cube#02 - soil_temperature / (K) (latitude: 73; longitude: 96)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('soil_model_level_number') are not recognised or handled.\n", | |
| "file#01 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field.ieee32\" : cube#03 - surface_altitude / (m) (latitude: 73; longitude: 96)\n", | |
| " BOTH ok.\n", | |
| "file#02 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbc/small_lbc\" : cube#00 - unknown / (unknown) (grid_latitude: 16; grid_longitude: 16)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#02 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbc/small_lbc\" : cube#01 - unknown / (unknown) (time: 2; model_level_number: 4; grid_latitude: 16; grid_longitude: 16)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#02 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbc/small_lbc\" : cube#02 - unknown / (unknown) (time: 2; model_level_number: 4; grid_latitude: 17; grid_longitude: 16)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#02 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbc/small_lbc\" : cube#03 - unknown / (unknown) (time: 2; model_level_number: 5; grid_latitude: 16; grid_longitude: 16)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#02 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbc/small_lbc\" : cube#04 - unknown / (unknown) (time: 2; model_level_number: 4; grid_latitude: 16; grid_longitude: 16)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#02 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbc/small_lbc\" : cube#05 - unknown / (unknown) (time: 2; model_level_number: 5; grid_latitude: 16; grid_longitude: 16)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#02 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbc/small_lbc\" : cube#06 - unknown / (unknown) (time: 2; model_level_number: 5; grid_latitude: 16; grid_longitude: 16)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#02 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbc/small_lbc\" : cube#07 - unknown / (unknown) (time: 2; model_level_number: 5; grid_latitude: 16; grid_longitude: 16)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#02 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbc/small_lbc\" : cube#08 - unknown / (unknown) (time: 2; model_level_number: 5; grid_latitude: 16; grid_longitude: 16)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#02 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbc/small_lbc\" : cube#09 - unknown / (unknown) (time: 2; model_level_number: 5; grid_latitude: 16; grid_longitude: 16)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#03 \"/home/h05/itpp/git/iris-test-data/test_data/FF/ancillary/qrparm.mask\" : cube#00 - land_binary_mask / (1) (grid_latitude: 1016; grid_longitude: 940)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> time coord not found\n", | |
| "file#04 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field.ieee32.gz\" : cube#00 - FAILED : No format specification could be found for the given buffer. File element cache:\n", | |
| " {'FileExtension()': '.gz', 'LeadingLine()': '\\x1f\\x8b\\x08\\x08\\xef\\xe8\\x9dQ\\x00\\x03n48_multi_field.ieee32\\x00\\xec\\xda\\x07xU\\xd5\\xda(\\xea\\xa5\\x08*\\xa2B\\x04\\x04D...', 'MagicNumber(8, None)': '2272919271416307025', 'UriProtocol()': 'file', 'MagicNumber(4, None)': '529205256', 'MagicNumber(100, None)': '\\x1f\\x8b\\x08\\x08\\xef\\xe8\\x9dQ\\x00\\x03n48_multi_field.ieee32\\x00\\xec\\xda\\x07xU\\xd5\\xda(\\xea\\xa5\\x08*\\xa2B\\x04\\x04D...'} / (unknown) (-- : 1)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> Did not find one (and only one) x or y coord\n", | |
| "file#05 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field_table_count\" : cube#00 - air_temperature / (K) (latitude: 73; longitude: 96)\n", | |
| " BOTH ok.\n", | |
| "file#05 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field_table_count\" : cube#01 - air_temperature / (K) (latitude: 73; longitude: 96)\n", | |
| " BOTH FAILED: SAME - <class 'gribapi.GribInternalError'> Encoding invalid\n", | |
| "file#05 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field_table_count\" : cube#02 - soil_temperature / (K) (latitude: 73; longitude: 96)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('soil_model_level_number') are not recognised or handled.\n", | |
| "file#05 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field_table_count\" : cube#03 - surface_altitude / (m) (latitude: 73; longitude: 96)\n", | |
| " BOTH ok.\n", | |
| "file#06 \"/home/h05/itpp/git/iris-test-data/test_data/FF/structured/small\" : cube#00 - air_pressure / (Pa) (time: 2; model_level_number: 3; latitude: 30; longitude: 40)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('level_height' ,'model_level_number') are not recognised or handled.\n", | |
| "file#07 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field\" : cube#00 - air_temperature / (K) (latitude: 73; longitude: 96)\n", | |
| " BOTH ok.\n", | |
| "file#07 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field\" : cube#01 - air_temperature / (K) (latitude: 73; longitude: 96)\n", | |
| " BOTH FAILED: SAME - <class 'gribapi.GribInternalError'> Encoding invalid\n", | |
| "file#07 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field\" : cube#02 - soil_temperature / (K) (latitude: 73; longitude: 96)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> The vertical-axis coordinate(s) ('soil_model_level_number') are not recognised or handled.\n", | |
| "file#07 \"/home/h05/itpp/git/iris-test-data/test_data/FF/n48_multi_field\" : cube#03 - surface_altitude / (m) (latitude: 73; longitude: 96)\n", | |
| " BOTH ok.\n", | |
| "file#08 \"/home/h05/itpp/git/iris-test-data/test_data/FF/lbrel_test_data\" : cube#00 - FAILED : Unsupported header release number: -32768 / (unknown) (-- : 1)\n", | |
| " BOTH FAILED: SAME - <class 'iris.exceptions.TranslationError'> Did not find one (and only one) x or y coord\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "test()\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 2", | |
| "language": "python", | |
| "name": "python2" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 2 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython2", | |
| "version": "2.7.11" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 0 | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| """ | |
| Check save works the same between iris-grib and iris.fileformats.grib(strict), | |
| when testing all the FF example files from iris-test-data. | |
| """ | |
| import os | |
| import os.path | |
| import shutil | |
| from shutil import rmtree | |
| import tempfile | |
| from tempfile import mkdtemp | |
| import iris | |
| from iris.fileformats.grib import save_grib2 as old_irisffg_save | |
| import iris.tests | |
| from iris.tests import get_data_path as get_testdata_path | |
| import iris_grib | |
| from iris_grib import save_grib2 as new_irisgrib_save | |
| from nose import exc | |
| def filesin(dirpath): | |
| assert os.path.isdir(dirpath) | |
| for name in os.listdir(dirpath): | |
| subpath = os.path.join(dirpath, name) | |
| if os.path.isdir(subpath): | |
| for result in filesin(subpath): | |
| yield result | |
| else: | |
| yield subpath | |
| def all_testdata_FF_files(): | |
| ff_testfiles_path = get_testdata_path('FF') | |
| for filepath in filesin(ff_testfiles_path): | |
| yield filepath | |
| def all_testdata_FF_cubes(): | |
| for i_file, filepath in enumerate(all_testdata_FF_files()): | |
| try: | |
| cubes = iris.load(filepath) | |
| except Exception as e: | |
| cubes = [iris.cube.Cube([1], long_name="FAILED : " + str(e))] | |
| for i_cube, cube in enumerate(cubes): | |
| yield i_file, filepath, i_cube, cube | |
| def save_result(cube, save_call): | |
| try: | |
| temp_output_dirpath = mkdtemp() | |
| temp_output_filepath = os.path.join(temp_output_dirpath, 'temp.grib2') | |
| try: | |
| save_call(cube, temp_output_filepath) | |
| exception = None | |
| except Exception as e: | |
| exception = e | |
| if exception is None: | |
| result_brief, exception_type, exception_text = ('ok', None, None) | |
| else: | |
| result_brief, exception_type, exception_text = \ | |
| ('FAIL', type(exception), str(exception)) | |
| finally: | |
| rmtree(temp_output_dirpath, ignore_errors=True) | |
| return result_brief, exception_type, exception_text | |
| def report_on(cube): | |
| old_msg, old_type, old_text = save_result(cube, old_irisffg_save) | |
| new_msg, new_type, new_text = save_result(cube, new_irisgrib_save) | |
| if old_msg == 'ok' and new_msg == 'ok': | |
| report = ' BOTH ok.' | |
| else: | |
| if old_msg == new_msg: | |
| report = ' BOTH FAILED' | |
| if old_type == new_type and old_text == new_text: | |
| report += ': SAME - {} {}'.format(new_type, new_text) | |
| else: | |
| report += ': DIFFERENT..' | |
| msg = '\n {} {} {}' | |
| report += msg.format('old', old_type, old_text) | |
| report += msg.format('new', new_type, new_text) | |
| else: | |
| # One failed | |
| report = ' DIFFERENT SUCCESSES' | |
| msg = '\n {} "{}" {} {}' | |
| report += msg.format('old', old_msg, old_type, old_text) | |
| report += msg.format('new', new_msg, new_type, new_text) | |
| return report | |
| def test(): | |
| print 'Iris version : ', iris.__version__ | |
| print 'Old save routine from : ', old_irisffg_save.__module__ | |
| print 'New save routine from : ', new_irisgrib_save.__module__ | |
| for i_file, filepath, i_cube, cube in all_testdata_FF_cubes(): | |
| msg = 'file#{:02d} "{}" : cube#{:02d} - {}' | |
| print msg.format(i_file, filepath.rjust(40), i_cube, cube.summary(shorten=True)) | |
| print report_on(cube) | |
| if __name__ == '__main__': | |
| test() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment