plume_mat_fieldset_loader.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load mats into netcdfs from plume tracker\n",
    "\n",
    "if you have a directory of .mat file versions of the HFR data, this notebook converts those into a single NetCDF file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.io\n",
    "import xarray as xr\n",
    "\n",
    "from pyplume.constants import DATA_DIR, FIELD_NETCDF_DIR\n",
    "import pyplume.utils as utils\n",
    "from pyplume.dataloaders import dataset_to_fieldset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "change settings accordingly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_filename = \"Tot_SDLJ_202209.nc\"\n",
    "\n",
    "start_time = np.datetime64(\"2022-09-01T00:00\", \"m\")\n",
    "end_time = np.datetime64(\"2022-09-30T23:00\", \"m\")\n",
    "incr = np.timedelta64(1, \"h\")\n",
    "\n",
    "coord_grid_path = DATA_DIR / \"sdcodargrid_hfr.mat\"\n",
    "\n",
    "mat_tot_dir = DATA_DIR / \"support_data/TrackerOutput/hourly\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_mat_time(time):\n",
    "    date = np.datetime64(\"0\") + np.timedelta64(int(time) - 1, \"D\")\n",
    "    hour = int(round((time - int(time)) * 24))\n",
    "    return date + np.timedelta64(hour, \"h\")\n",
    "\n",
    "\n",
    "def parse_time(time):\n",
    "    d = \"\".join(str(time).split(\"T\")[0].split(\"-\"))\n",
    "    t = \"\".join(str(time).split(\"T\")[1].split(\":\")[:2])\n",
    "    return d, t\n",
    "\n",
    "\n",
    "def dataset_from_mat(grid_mat_path, current_mat_path):\n",
    "    grid_mat = scipy.io.loadmat(grid_mat_path)\n",
    "    current_mat = scipy.io.loadmat(current_mat_path)\n",
    "    if \"totalGrid\" in grid_mat:\n",
    "        coords = grid_mat[\"totalGrid\"]\n",
    "    else:\n",
    "        gx = grid_mat[\"gx\"]\n",
    "        gy = grid_mat[\"gy\"]\n",
    "        coords = np.empty((gx.size, 2), dtype=np.float32)\n",
    "        coords[:, 0] = gx.flatten()\n",
    "        coords[:, 1] = gy.flatten()\n",
    "    u = current_mat[\"U\"]\n",
    "    v = current_mat[\"V\"]\n",
    "    time = np.array([parse_mat_time(current_mat[\"t\"][0, 0])])\n",
    "    lats = np.sort(np.unique(coords.T[1]))\n",
    "    lons = np.sort(np.unique(coords.T[0]))\n",
    "    u_grid = np.zeros((1, len(lats), len(lons)))\n",
    "    v_grid = np.zeros((1, len(lats), len(lons)))\n",
    "    for i in range(len(lons)):\n",
    "        for j in range(len(lats)):\n",
    "            # data from .mat is measured in cm/s\n",
    "            u_grid[0, j, i] = u[0, i * len(lats) + j] / 100\n",
    "            v_grid[0, j, i] = v[0, i * len(lats) + j] / 100\n",
    "    ds = xr.Dataset(\n",
    "        {\n",
    "            \"U\": ([\"time\", \"lat\", \"lon\"], u_grid),\n",
    "            \"V\": ([\"time\", \"lat\", \"lon\"], v_grid),\n",
    "        },\n",
    "        coords={\"time\": time, \"lat\": lats, \"lon\": lons},\n",
    "    )\n",
    "    return ds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "concatenates all the separate .mat files into a single dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ds_total = None\n",
    "current_time = start_time\n",
    "\n",
    "while current_time <= end_time:\n",
    "    date_parsed, time_parsed = parse_time(current_time)\n",
    "    filename = mat_tot_dir / f\"Tot_SDLJ_{date_parsed}_{time_parsed}.mat\"\n",
    "    if filename.is_file():\n",
    "        ds = dataset_from_mat(coord_grid_path, filename)\n",
    "        if ds_total is None:\n",
    "            ds_total = ds\n",
    "        else:\n",
    "            ds_total = xr.concat([ds_total, ds], dim=\"time\")\n",
    "    else:\n",
    "        print(f\"file {filename} not found\")\n",
    "    current_time += incr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# display fieldset to see if data was loaded correctly\n",
    "fs = dataset_to_fieldset(ds_total)\n",
    "fs.U.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_parsed, _ = parse_time(start_time)\n",
    "saveto = FIELD_NETCDF_DIR / f\"oi_fields/{out_filename}\"\n",
    "ds_total.to_netcdf(saveto)\n",
    "print(f\"saved to {saveto}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.10 ('py3-parcels')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "0eba6c4b74a2f8ee4f11000f0a8df2ef42b87fd4ae75b12d527f1b8c08aad6c6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}