{ "cells": [ { "attachments": {}, "cell_type": "markdown", "id": "f168804a-2c0d-4dd7-8b1c-5d909c889452", "metadata": {}, "source": [ "# NOGAPS\n", "\n", "This demonstrates how to download historical NOGAPS data archived at [NCEI](https://www.ncei.noaa.gov/products/weather-climate-models/navy-operational-global-atmospheric-prediction?msclkid=ee48a0e7cdb911eca49b9d0ed06548f8) for historical analyses from 1997 to 2008 at 0.5 and 1.0 degree grids. \n", "\n", "> For NOGAPS data from GODAE, use `model='navgem_godae`." ] }, { "cell_type": "code", "execution_count": 1, "id": "b4cba677-7674-47d6-8dde-2d297b138c57", "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "from herbie import Herbie\n", "\n", "from herbie import paint\n", "from herbie.toolbox import EasyMap, pc" ] }, { "cell_type": "code", "execution_count": 4, "id": "6384df86-fc73-4496-a138-5f227535b6d6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "✅ Found ┊ model=nogaps_ncei ┊ \u001b[3mproduct=058_240\u001b[0m ┊ \u001b[38;2;41;130;13m2008-Jul-28 12:00 UTC\u001b[92m F00\u001b[0m ┊ \u001b[38;2;255;153;0m\u001b[3mGRIB2 @ ncei\u001b[0m ┊ \u001b[38;2;255;153;0m\u001b[3mIDX @ ncei\u001b[0m\n" ] } ], "source": [ "H = Herbie(\"2008-07-28 12:00\", model=\"nogaps_ncei\", product=\"058_240\")" ] }, { "cell_type": "code", "execution_count": 5, "id": "122440c6", "metadata": {}, "outputs": [ { "ename": "ParserError", "evalue": "Error tokenizing data. C error: EOF inside string starting at row 74", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mParserError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[5], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# TODO: Herbie cant read this NOGAPS inventory files\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[43mH\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minventory\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mTMP\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/GITHUB/Herbie/herbie/core.py:791\u001b[0m, in \u001b[0;36mHerbie.inventory\u001b[0;34m(self, search, searchString, verbose)\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minventory\u001b[39m(\u001b[38;5;28mself\u001b[39m, search\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m, searchString\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m 765\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 766\u001b[0m \u001b[38;5;124;03m Inspect the GRIB2 file contents by reading the index file.\u001b[39;00m\n\u001b[1;32m 767\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 789\u001b[0m \n\u001b[1;32m 790\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 791\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindex_as_dataframe\u001b[49m\n\u001b[1;32m 793\u001b[0m \u001b[38;5;66;03m# TODO: Remove this eventually\u001b[39;00m\n\u001b[1;32m 794\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m searchString \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", "File \u001b[0;32m~/miniconda3/envs/herbie-dev/lib/python3.12/functools.py:995\u001b[0m, in \u001b[0;36mcached_property.__get__\u001b[0;34m(self, instance, owner)\u001b[0m\n\u001b[1;32m 993\u001b[0m val \u001b[38;5;241m=\u001b[39m cache\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mattrname, _NOT_FOUND)\n\u001b[1;32m 994\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m val \u001b[38;5;129;01mis\u001b[39;00m _NOT_FOUND:\n\u001b[0;32m--> 995\u001b[0m val \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 996\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 997\u001b[0m cache[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mattrname] \u001b[38;5;241m=\u001b[39m val\n", "File \u001b[0;32m~/GITHUB/Herbie/herbie/core.py:643\u001b[0m, in \u001b[0;36mHerbie.index_as_dataframe\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 640\u001b[0m read_this_idx \u001b[38;5;241m=\u001b[39m StringIO(response\u001b[38;5;241m.\u001b[39mtext)\n\u001b[1;32m 641\u001b[0m response\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m--> 643\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 644\u001b[0m \u001b[43m \u001b[49m\u001b[43mread_this_idx\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 645\u001b[0m \u001b[43m \u001b[49m\u001b[43msep\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m:\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 646\u001b[0m \u001b[43m \u001b[49m\u001b[43mnames\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\n\u001b[1;32m 647\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgrib_message\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 648\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstart_byte\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 649\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mreference_time\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 650\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mvariable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 651\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlevel\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 652\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mforecast_time\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 653\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m?\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 654\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m??\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 655\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m???\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 656\u001b[0m \u001b[43m \u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 657\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 659\u001b[0m \u001b[38;5;66;03m# Format the DataFrame\u001b[39;00m\n\u001b[1;32m 660\u001b[0m df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mreference_time\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mto_datetime(\n\u001b[1;32m 661\u001b[0m df\u001b[38;5;241m.\u001b[39mreference_time, \u001b[38;5;28mformat\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124md=\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mY\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mm\u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mH\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 662\u001b[0m )\n", "File \u001b[0;32m~/miniconda3/envs/herbie-dev/lib/python3.12/site-packages/pandas/io/parsers/readers.py:1026\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[1;32m 1013\u001b[0m kwds_defaults \u001b[38;5;241m=\u001b[39m _refine_defaults_read(\n\u001b[1;32m 1014\u001b[0m dialect,\n\u001b[1;32m 1015\u001b[0m delimiter,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1022\u001b[0m dtype_backend\u001b[38;5;241m=\u001b[39mdtype_backend,\n\u001b[1;32m 1023\u001b[0m )\n\u001b[1;32m 1024\u001b[0m kwds\u001b[38;5;241m.\u001b[39mupdate(kwds_defaults)\n\u001b[0;32m-> 1026\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/miniconda3/envs/herbie-dev/lib/python3.12/site-packages/pandas/io/parsers/readers.py:626\u001b[0m, in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 623\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m parser\n\u001b[1;32m 625\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m parser:\n\u001b[0;32m--> 626\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mparser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnrows\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/miniconda3/envs/herbie-dev/lib/python3.12/site-packages/pandas/io/parsers/readers.py:1923\u001b[0m, in \u001b[0;36mTextFileReader.read\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m 1916\u001b[0m nrows \u001b[38;5;241m=\u001b[39m validate_integer(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnrows\u001b[39m\u001b[38;5;124m\"\u001b[39m, nrows)\n\u001b[1;32m 1917\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1918\u001b[0m \u001b[38;5;66;03m# error: \"ParserBase\" has no attribute \"read\"\u001b[39;00m\n\u001b[1;32m 1919\u001b[0m (\n\u001b[1;32m 1920\u001b[0m index,\n\u001b[1;32m 1921\u001b[0m columns,\n\u001b[1;32m 1922\u001b[0m col_dict,\n\u001b[0;32m-> 1923\u001b[0m ) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# type: ignore[attr-defined]\u001b[39;49;00m\n\u001b[1;32m 1924\u001b[0m \u001b[43m \u001b[49m\u001b[43mnrows\u001b[49m\n\u001b[1;32m 1925\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1926\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m 1927\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n", "File \u001b[0;32m~/miniconda3/envs/herbie-dev/lib/python3.12/site-packages/pandas/io/parsers/c_parser_wrapper.py:234\u001b[0m, in \u001b[0;36mCParserWrapper.read\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 233\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlow_memory:\n\u001b[0;32m--> 234\u001b[0m chunks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_reader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_low_memory\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnrows\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;66;03m# destructive to chunks\u001b[39;00m\n\u001b[1;32m 236\u001b[0m data \u001b[38;5;241m=\u001b[39m _concatenate_chunks(chunks)\n", "File \u001b[0;32mparsers.pyx:838\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader.read_low_memory\u001b[0;34m()\u001b[0m\n", "File \u001b[0;32mparsers.pyx:905\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n", "File \u001b[0;32mparsers.pyx:874\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._tokenize_rows\u001b[0;34m()\u001b[0m\n", "File \u001b[0;32mparsers.pyx:891\u001b[0m, in \u001b[0;36mpandas._libs.parsers.TextReader._check_tokenize_status\u001b[0;34m()\u001b[0m\n", "File \u001b[0;32mparsers.pyx:2061\u001b[0m, in \u001b[0;36mpandas._libs.parsers.raise_parser_error\u001b[0;34m()\u001b[0m\n", "\u001b[0;31mParserError\u001b[0m: Error tokenizing data. C error: EOF inside string starting at row 74" ] } ], "source": [ "# TODO: Herbie cant read this NOGAPS inventory files\n", "\n", "H.inventory(\"TMP\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "c8928077", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "✅ Found ┊ model=nogaps_ncei ┊ \u001b[3mproduct=008_240\u001b[0m ┊ \u001b[38;2;41;130;13m2008-Jul-28 12:00 UTC\u001b[92m F00\u001b[0m ┊ \u001b[38;2;255;153;0m\u001b[3mGRIB2 @ ncei\u001b[0m ┊ \u001b[38;2;255;153;0m\u001b[3mIDX @ ncei\u001b[0m\n" ] } ], "source": [ "H = Herbie(\"2008-07-28 12:00\", model=\"nogaps_ncei\", product=\"008_240\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "a4973897", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | grib_message | \n", "start_byte | \n", "end_byte | \n", "range | \n", "reference_time | \n", "valid_time | \n", "variable | \n", "level | \n", "forecast_time | \n", "? | \n", "?? | \n", "??? | \n", "search_this | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "1 | \n", "21 | \n", "228296.0 | \n", "21-228296 | \n", "2008-07-28 12:00:00 | \n", "2008-07-28 12:00:00 | \n", "TMP | \n", "0 mb | \n", "kpds=11,100,0 | \n", "anl | \n", "winds are N/S | \n", "Temp. [K]\\n2:89724:D=2008072812:HGT:0 mb:kpds=... | \n", ":TMP:0 mb:kpds=11,100,0:anl:winds are N/S:Temp... | \n", "
| 2 | \n", "5 | \n", "407705 | \n", "644126.0 | \n", "407705-644126 | \n", "2008-07-28 12:00:00 | \n", "2008-07-28 12:00:00 | \n", "TMP | \n", "1 mb | \n", "kpds=11,100,1 | \n", "anl | \n", "winds are N/S | \n", "Temp. [K]\\n6:505554:D=2008072812:HGT:1 mb:kpds... | \n", ":TMP:1 mb:kpds=11,100,1:anl:winds are N/S:Temp... | \n", "
| 4 | \n", "9 | \n", "823535 | \n", "1068100.0 | \n", "823535-1068100 | \n", "2008-07-28 12:00:00 | \n", "2008-07-28 12:00:00 | \n", "TMP | \n", "2 mb | \n", "kpds=11,100,2 | \n", "anl | \n", "winds are N/S | \n", "Temp. [K]\\n10:929528:D=2008072812:HGT:2 mb:kpd... | \n", ":TMP:2 mb:kpds=11,100,2:anl:winds are N/S:Temp... | \n", "
| 6 | \n", "13 | \n", "1231217 | \n", "1467638.0 | \n", "1231217-1467638 | \n", "2008-07-28 12:00:00 | \n", "2008-07-28 12:00:00 | \n", "TMP | \n", "5 mb | \n", "kpds=11,100,5 | \n", "anl | \n", "winds are N/S | \n", "Temp. [K]\\n14:1337210:D=2008072812:HGT:5 mb:kp... | \n", ":TMP:5 mb:kpds=11,100,5:anl:winds are N/S:Temp... | \n", "
<xarray.Dataset> Size: 526kB\n",
"Dimensions: (latitude: 181, longitude: 360)\n",
"Coordinates:\n",
" time datetime64[ns] 8B 2008-07-28T12:00:00\n",
" step timedelta64[ns] 8B 00:00:00\n",
" isobaricInhPa float64 8B 5.0\n",
" * latitude (latitude) float64 1kB -90.0 -89.0 -88.0 ... 89.0 90.0\n",
" * longitude (longitude) float64 3kB -1.0 0.0 1.0 ... 357.0 358.0\n",
" valid_time datetime64[ns] 8B 2008-07-28T12:00:00\n",
" gribfile_projection object 8B None\n",
"Data variables:\n",
" t (latitude, longitude) float32 261kB 210.5 ... 252.4\n",
" gh (latitude, longitude) float32 261kB 3.048e+04 ... 3....\n",
"Attributes:\n",
" GRIB_edition: 1\n",
" GRIB_centre: fnmo\n",
" GRIB_centreDescription: US Navy - Fleet Numerical Oceanography Center\n",
" GRIB_subCentre: 0\n",
" Conventions: CF-1.7\n",
" institution: US Navy - Fleet Numerical Oceanography Center\n",
" model: nogaps_ncei\n",
" product: 008_240\n",
" description: Navy Operational Global Atmospheric Prediction S...\n",
" remote_grib: https://www.ncei.noaa.gov/data/navy-operational-...\n",
" local_grib: /home/blaylock/data/nogaps_ncei/20080728/subset_...\n",
" search: TMP:5 mb