Skip to content

Commit

Permalink
Update regression testing notebooks
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomvothecoder
tomvothecoder committed Feb 5, 2025
1 parent 83e7f61 commit 9b92f8f
Show file tree
Hide file tree
Showing 2 changed files with 239 additions and 99 deletions.
129 changes: 70 additions & 59 deletions auxiliary_tools/cdat_regression_testing/930-zppy-diffs/v2/regression_nc.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@
{
"data": {
"text/plain": [
"(32, 32)"
"(34, 34)"
]
},
"execution_count": 5,
Expand Down Expand Up @@ -301,29 +301,13 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Comparing:\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-branch-930-zppy-diffs/lat_lon/ERA5/ERA5-OMEGA-850-ANN-global_test.nc\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-main-zppy-diffs/lat_lon/ERA5/ERA5-OMEGA-850-ANN-global_test.nc\n",
" * var_key: OMEGA\n",
" \n",
"Not equal to tolerance rtol=0.0001, atol=0\n",
"\n",
"Mismatched elements: 17 / 64800 (0.0262%)\n",
"Max absolute difference among violations: 2.15263471e-05\n",
"Max relative difference among violations: 0.00060596\n",
" ACTUAL: array([[ nan, nan, nan, ..., nan, nan,\n",
" nan],\n",
" [ nan, nan, nan, ..., nan, nan,...\n",
" DESIRED: array([[ nan, nan, nan, ..., nan, nan,\n",
" nan],\n",
" [ nan, nan, nan, ..., nan, nan,...\n",
"Comparing:\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-branch-930-zppy-diffs/lat_lon/ERA5/ERA5-TREFHT-ANN-land_ref.nc\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-main-zppy-diffs/lat_lon/ERA5/ERA5-TREFHT-ANN-land_ref.nc\n",
Expand All @@ -339,6 +323,22 @@
" -45.43447 ],\n",
" [-45.000128, -44.998984, -44.997885, ..., -45.001944, -45.001349,...\n",
"Comparing:\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-branch-930-zppy-diffs/lat_lon/MERRA2/MERRA2-OMEGA-850-ANN-global_test.nc\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-main-zppy-diffs/lat_lon/MERRA2/MERRA2-OMEGA-850-ANN-global_test.nc\n",
" * var_key: OMEGA\n",
" \n",
"Not equal to tolerance rtol=0.0001, atol=0\n",
"\n",
"Mismatched elements: 17 / 64800 (0.0262%)\n",
"Max absolute difference among violations: 2.15263471e-05\n",
"Max relative difference among violations: 0.00060596\n",
" ACTUAL: array([[ nan, nan, nan, ..., nan, nan,\n",
" nan],\n",
" [ nan, nan, nan, ..., nan, nan,...\n",
" DESIRED: array([[ nan, nan, nan, ..., nan, nan,\n",
" nan],\n",
" [ nan, nan, nan, ..., nan, nan,...\n",
"Comparing:\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-branch-930-zppy-diffs/lat_lon/MERRA2/MERRA2-TAUXY-ANN-ocean_ref.nc\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-main-zppy-diffs/lat_lon/MERRA2/MERRA2-TAUXY-ANN-ocean_ref.nc\n",
" * var_key: TAUXY\n",
Expand Down Expand Up @@ -375,7 +375,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -394,20 +394,20 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" stat_name value pct\n",
"0 matching_files_count 28 0.87500\n",
"1 missing_vars_count 0 0.00000\n",
"2 mismatch_errors_count 3 0.09375\n",
"3 not_equal_errors_count 1 0.03125\n",
"4 key_errors_count 0 0.00000\n",
"5 missing_files_count 0 0.00000\n"
" stat_name value pct\n",
"0 matching_files_count 30 0.882353\n",
"1 missing_vars_count 0 0.000000\n",
"2 mismatch_errors_count 3 0.088235\n",
"3 not_equal_errors_count 1 0.029412\n",
"4 key_errors_count 0 0.000000\n",
"5 missing_files_count 0 0.000000\n"
]
}
],
Expand Down Expand Up @@ -475,7 +475,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 11,
"metadata": {},
"outputs": [
{
Expand All @@ -484,7 +484,7 @@
"[]"
]
},
"execution_count": 10,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -509,7 +509,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 12,
"metadata": {},
"outputs": [
{
Expand All @@ -520,7 +520,7 @@
" '/lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-branch-930-zppy-diffs/lat_lon/MERRA2/MERRA2-TREFHT-ANN-land_ref.nc']"
]
},
"execution_count": 11,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -531,7 +531,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -547,7 +547,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 14,
"metadata": {},
"outputs": [
{
Expand All @@ -556,7 +556,7 @@
"[]"
]
},
"execution_count": 13,
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -576,7 +576,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -646,16 +646,16 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Checking variable OMEGA\n",
"Dev Path: /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-branch-930-zppy-diffs/lat_lon/ERA5/ERA5-OMEGA-850-ANN-global_test.nc\n",
"Main Path: /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-main-zppy-diffs/lat_lon/ERA5/ERA5-OMEGA-850-ANN-global_test.nc\n",
"Dev Path: /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-branch-930-zppy-diffs/lat_lon/MERRA2/MERRA2-OMEGA-850-ANN-global_test.nc\n",
"Main Path: /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-main-zppy-diffs/lat_lon/MERRA2/MERRA2-OMEGA-850-ANN-global_test.nc\n",
"-------------------------------------\n",
"* Min - dev: -248.597013, main: -248.597015\n",
"* Max - dev: 604.741699, main: 604.741211\n",
Expand All @@ -681,7 +681,7 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -712,7 +712,7 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 18,
"metadata": {},
"outputs": [
{
Expand All @@ -721,7 +721,7 @@
"[]"
]
},
"execution_count": 17,
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -739,25 +739,25 @@
"We are good to go here. The differences are expected and explained in the comments below.\n",
"\n",
"```python\n",
" stat_name value pct\n",
"0 matching_files_count 28 0.87500\n",
"1 missing_vars_count 0 0.00000\n",
"2 mismatch_errors_count 3 0.09375\n",
"3 not_equal_errors_count 1 0.03125\n",
"4 key_errors_count 0 0.00000\n",
"5 missing_files_count 0 0.00000\n",
" stat_name value pct\n",
"0 matching_files_count 30 0.882353\n",
"1 missing_vars_count 0 0.000000\n",
"2 mismatch_errors_count 3 0.088235\n",
"3 not_equal_errors_count 1 0.029412\n",
"4 key_errors_count 0 0.000000\n",
"5 missing_files_count 0 0.000000\n",
"```\n",
"\n",
"- 28/32 matching files\n",
"- 3/28 mismatch errors due to ccb regional subsetting differences, these are expected\n",
" - Affects `ERA5-TREFHT`, ``MERRA2-TAUXY`, and `MERRA2-TREFHT`\n",
"- 30/34 matching files\n",
"- 3/34 mismatch errors due to ccb regional subsetting differences, these are expected\n",
" - Affects `ERA5-TREFHT`, `MERRA2-TAUXY`, and `MERRA2-TREFHT`\n",
" - Related to https://github.com/E3SM-Project/e3sm_diags/pull/794\n",
"- 1/32 not equal files -- number of different elements are really small and stats (min, max, mean, sum) are similar (OMEGA below)\n",
"- 1/34 not equal files -- number of different elements are really small and stats (min, max, mean, sum) are similar (MERRA2-OMEGA below)\n",
"\n",
" ```python\n",
" Comparing:\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-branch-930-zppy-diffs/lat_lon/ERA5/ERA5-OMEGA-850-ANN-global_test.nc\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-main-zppy-diffs/lat_lon/ERA5/ERA5-OMEGA-850-ANN-global_test.nc\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-branch-930-zppy-diffs/lat_lon/MERRA2/MERRA2-OMEGA-850-ANN-global_test.nc\n",
" * /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-main-zppy-diffs/lat_lon/MERRA2/MERRA2-OMEGA-850-ANN-global_test.nc\n",
" * var_key: OMEGA\n",
"\n",
" Not equal to tolerance rtol=0.0001, atol=0\n",
Expand All @@ -771,14 +771,25 @@
" DESIRED: array([[ nan, nan, nan, ..., nan, nan,\n",
" nan],\n",
" [ nan, nan, nan, ..., nan, nan,...\n",
"\n",
" ```\n",
"\n",
" ```python\n",
" Checking variable OMEGA\n",
" Dev Path: /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-branch-930-zppy-diffs/lat_lon/MERRA2/MERRA2-OMEGA-850-ANN-global_test.nc\n",
" Main Path: /lcrc/group/e3sm/public_html/cdat-migration-fy24/25-02-04-main-zppy-diffs/lat_lon/MERRA2/MERRA2-OMEGA-850-ANN-global_test.nc\n",
" -------------------------------------\n",
" * Min - dev: -248.597013, main: -248.597015\n",
" * Max - dev: 604.741699, main: 604.741211\n",
" * Mean - dev: 0.446415, main: 0.446415\n",
" * Absolute Mean Diff: 1.3810975812988957e-07\n",
" * Relative Mean Diff: 0.000031%\n",
" * Sum - dev: 26022.445543, main: 26022.439453\n",
" * Absolute Sum Diff: 0.006090177481382852\n",
" * Relative Sum Diff: 0.000023%\n",
" ```\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down
Loading

0 comments on commit 9b92f8f

Please sign in to comment.