Time series plots (#31)

* added: timeseries plots TimeSeriesFigure class

* added: diet_plots plot_bar_meals & plot_lollipop_meals

* improved: docstring

* added: plot_events_fill
renamed, reformatted and moved plot_events -> plot_events_bars

* improved: diet_plots standardised signature

* added: diet_plots.plot_meals_events
fixed: diet_plots.plot_meals_bars
added: label arg

* improved: TimeSeriesFigure adjust limits after each plot

* improved: plot_events_bars inverted y-axis

* renamed: arg color -> hue

* renamed: arg cmap -> palette

* improved: plot_meals_lollipop size='total_g'

* added: config LEGEND_SHIFT

* improved: plot_events_fill accepts multiple axes

* improved: legend position

* improved: re-wrote plot_sleep with new ts funcs

* improved: plot_events_fill behind other plots

* added: plot_diet_cgm_sleep

* rename: diet plots funcs

* improved: plot_diet_cgm_sleep default channels

* improved: plot_meals_hbars with size legend

* improved: plot_diet_cgm_sleep diet / cgm on single subplot

* refactored: second_y arg moved to TimeSeriesFigure.plot

* improved: plot_diet_cgm_sleep calibration

* improved: plot_events_bars accepts hue=None

* improved: plot_diet_cgm_sleep accepts kwargs for each subplot

* improved: new args nut_exclude nut_include

* added: get_events_period, get_sleep_period
improved: plot_diet_cgm_sleep uses get_sleep_period

* chore: PR notes

* renamed: filter_df -> format_timeseries

* renamed: prepare_events -> prep_to_plot_timeseries

* added: TimeSeriesFigure.plot args adjust_time, adjust_by_axis
support for union / intersect of xlims

* fixed: cgm_plots for new example data

* chore: clean cells

* improved: timeseries_plots: set_periodic_ticks arg start_time

* added: timeseries_plots tutorial

* improved: timeseries_plots: text

* improved: timeseries_plots: text

* added: sleep_plots tutorial

* added: diet_plots: tutorial

* improved: sleep_plots: tutorial

* improved: timeseries_plots: title

* improved: diet_plots: food category rename

* added: new example datasets (time series)

* fixed: deploy error

* fixed: deploy error

nbs/00_config.ipynb

@@ -54,6 +54,7 @@
    "outputs": [],
    "source": [
     "#| export\n",
+    "DEFAULT_PALETTE = 'muted'\n",
     "REF_COLOR = \"k\"\n",
     "FEMALE_COLOR = \"C1\"\n",
     "MALE_COLOR = \"C0\"\n",
@@ -62,6 +63,9 @@
     "GLUC_COLOR = \"C0\"\n",
     "FOOD_COLOR = \"C1\"\n",
     "\n",
+    "LEGEND_SHIFT = (1.05, 1.05)\n",
+    "TIME_FORMAT = '%d/%m\\n%H:%M'\n",
+    "\n",
     "DATASETS_PATH = '/home/ec2-user/studies/hpp_datasets/'\n",
     "COHORT = None\n",
     "EVENTS_DATASET = 'events'\n",
@@ -70,8 +74,7 @@
     "BULK_DATA_PATH = {}\n",
     "PREFERRED_LANGUAGE = 'english'\n",
     "\n",
-    "config_found = False\n",
-    "\n"
+    "config_found = False"
    ]
   },
   {

nbs/examples/diet_logging/metadata/diet_logging_data_dictionary.csv

@@ -1,19 +1,13 @@
 tabular_field_name,field_string,description_string,parent_dataframe,relative_location,value_type,units,field_type,array,cohorts,data_type,debut,pandas_dtype,sampling_rate
 collection_timestamp,Collection timestamp,Collection timestamp,,diet_logging/diet_logging.parquet,Time,Time,Data,Single,10K,Time Series,2019-01-29,"datetime64[ns, Asia/Jerusalem]",
-collection_date,Date,Datetime column relecting the time food item was logged,,diet_logging/diet_logging.parquet,Time,Time,Data,Single,10K,Time Series,2019-09-01,datetime64[ns],
 food_id,Food ID,IDs in the diet logging app representing specific food ,,diet_logging/diet_logging.parquet,Categorical (single) ,None,Data,Single,10K,Time Series,2019-09-01,integer,
-logging_day,Logging day per participant,Integer indicating which day of logging period ,,diet_logging/diet_logging.parquet,Integer,None ,Data,Single,10K,Time Series,2019-09-01,float,
-weight,Weight,Weight of food item logged,,diet_logging/diet_logging.parquet,Continuous,g,Data,Single,10K,Time Series,2019-09-01,float,
 short_food_name,Short food name,Classifcation of food item logged into a short food name category,,diet_logging/diet_logging.parquet,Categorical (single) ,None,Data,Single,10K,Time Series,2019-09-01,object,
 food_category,Food category,Classifcation of food item logged into a food category,,diet_logging/diet_logging.parquet,Categorical (single) ,None,Data,Single,10K,Time Series,2019-09-01,object,
-product_name,Product name ,Product name of food logged,,diet_logging/diet_logging.parquet,Categorical (single) ,None,Data,Single,10K,Time Series,2019-09-01,object,
-calories,Calories,Calories of food item logged,,diet_logging/diet_logging.parquet,Continuous,kcal,Data,Single,10K,Time Series,2019-09-01,float,
+weight_g,Weight,Weight of food item logged,,diet_logging/diet_logging.parquet,Continuous,g,Data,Single,10K,Time Series,2019-09-01,float,
+calories_kcal,Calories,Calories of food item logged,,diet_logging/diet_logging.parquet,Continuous,kcal,Data,Single,10K,Time Series,2019-09-01,float,
 carbohydrate_g,Carbohydrate intake per food logged,Carbohydrate intake per food logged,,diet_logging/diet_logging.parquet,Continuous,g,Data,Single,10K,Time Series,2019-09-01,float,
-llipid_g,Fat intake per food logged,Fat intake per food logged,,diet_logging/diet_logging.parquet,Continuous,g,Data,Single,10K,Time Series,2019-09-01,float,
+lipid_g,Fat intake per food logged,Fat intake per food logged,,diet_logging/diet_logging.parquet,Continuous,g,Data,Single,10K,Time Series,2019-09-01,float,
 protein_g,Protein intake per food logged,Protein intake per food logged,,diet_logging/diet_logging.parquet,Continuous,g,Data,Single,10K,Time Series,2019-09-01,float,
-sodium_mg             ,Sodium intake per food logged,Sodium intake per food logged,,diet_logging/diet_logging.parquet,Continuous,mg,Data,Single,10K,Time Series,2019-09-01,float,
+sodium_mg,Sodium intake per food logged,Sodium intake per food logged,,diet_logging/diet_logging.parquet,Continuous,mg,Data,Single,10K,Time Series,2019-09-01,float,
 alcohol_g ,Alcohol intake per food logged,Alcohol intake per food logged,,diet_logging/diet_logging.parquet,Continuous,g,Data,Single,10K,Time Series,2019-09-01,float,
 dietary_fiber_g,Dietary fiber intake per food logged,Dietary fiber intake per food logged,,diet_logging/diet_logging.parquet,Continuous,g,Data,Single,10K,Time Series,2019-09-01,float,
-local_timestamp,Local timestamp,Local timestamp of food logging,,diet_logging/diet_logging.parquet,Time,Time,Data,Single,10K,Time Series,2019-09-01,datetime64[ns],
-eaten_in_restaurant,Eaten at restaurant indication,Indication if food was eatn at home or at a restaurant,,diet_logging/diet_logging.parquet,Boolean,None,Data,Single,10K,Time Series,2019-09-01,bool,
-total_logging_days,Total number of days logged,Total number of days diet was logged per research stage,,diet_logging/diet_logging.parquet,Integer,None,Data,Single,10K,Time Series,2019-09-01,integer,

nbs/examples/sleep/metadata/sleep_data_dictionary.csv

@@ -1,3 +1,5 @@
-tabular_field_name,field_string,description_string,parent_dataframe,relative_location,value_type,units,sampling_rate,item_type,array,cohorts,field_type,debut,pandas_dtype
-ahi,AHI,AHI (Apnea-Hypopnea Index),,sleep/sleep.parquet,Continuous,Events / Hour,,Data,Multiple,10K,Continuous,2020-01-15,float64
-total_sleep_time,Total sleep time,Total sleep time,,sleep/sleep.parquet,Integer,Seconds,,Data,Multiple,10K,Continuous,2020-01-15,float64
+tabular_field_name,field_string,description_string,parent_dataframe,relative_location,units,sampling_rate,array,cohorts,field_type,debut,pandas_dtype
+ahi,AHI,AHI (Apnea-Hypopnea Index),,sleep/sleep.parquet,Events / Hour,,Multiple,10K,Continuous,2020-01-15,float
+total_sleep_time,Total sleep time,Total sleep time,,sleep/sleep.parquet,Seconds,,Multiple,10K,Continuous,2020-01-15,int
+channels_time_series,Channels time series,Sensor and derived channels time series parquet files,,sleep/sleep.parquet,,,Multiple,10K,Time series file (individual),2020-01-15,string
+events_time_series,Events time series,"Events during sleep derived from the raw channels, such as sleep stages, respiratory events, pulse rate events, and others",,sleep/sleep.parquet,,Data,Multiple,10K,Time series file (group),2020-01-15,string

nbs/sidebar.yml

@@ -15,6 +15,7 @@ website:
           - 01_basic_plots.ipynb
           - 02_blandaltman_plots.ipynb
           - 03_age_reference_plots.ipynb
+          - 15_timeseries_plots.ipynb
           - 04_date_plots.ipynb
           - 06_sleep_plots.ipynb
           - 08_cgm_plots.ipynb

pheno_utils/cgm_plots.py

@@ -28,7 +28,7 @@ def __init__(
         cgm_date_col: str = "collection_timestamp",
         gluc_col: str = "glucose",
         diet_date_col: str = "collection_timestamp",
-        diet_text_col: str = "shortname_eng",
+        diet_text_col: str = "short_food_name",
         ax: Optional[plt.Axes] = None,
         smooth: bool = False,
         sleep_tuples: Optional[List[Tuple[pd.Timestamp, pd.Timestamp]]] = None,
@@ -39,15 +39,15 @@ def __init__(
         Args:
             cgm_df (pd.DataFrame): DataFrame containing the glucose measurements.
             diet_df (Optional[pd.DataFrame], optional): DataFrame containing the diet data. Defaults to None.
-            cgm_date_col (str, optional): Name of the date column in cgm_df. Defaults to "Date".
+            cgm_date_col (str, optional): Name of the date column in cgm_df. Defaults to "collection_timestamp".
             gluc_col (str, optional): Name of the glucose column in cgm_df. Defaults to "glucose".
-            diet_date_col (str, optional): Name of the date column in diet_df. Defaults to "Date".
-            diet_text_col (str, optional): Name of the text column in diet_df. Defaults to "shortname_eng".
+            diet_date_col (str, optional): Name of the date column in diet_df. Defaults to "collection_timestamp".
+            diet_text_col (str, optional): Name of the text column in diet_df. Defaults to "short_food_name".
             ax (Optional[plt.Axes], optional): Matplotlib Axes object to plot on. Defaults to None.
             smooth (bool, optional): Apply smoothing to the glucose curve. Defaults to False.
             sleep_tuples (Optional[List[Tuple[pd.Timestamp, pd.Timestamp]]], optional): List of sleep start and end times. Defaults to None.
         """
-        self.cgm_df = cgm_df
+        self.cgm_df = cgm_df.reset_index()
         self.diet_df = diet_df
         self.cgm_date_col = cgm_date_col
         self.gluc_col = gluc_col
@@ -110,7 +110,7 @@ def plot_diet(self) -> None:
         for i, (food_datetime, group) in enumerate(
             self.diet_df.groupby(self.diet_date_col)
         ):
-            food_str = "\n".join(group[self.diet_text_col])
+            food_str = "\n".join(group[self.diet_text_col].dropna())
 
             txt_x = food_datetime - pd.to_timedelta(7.5, "m")
             if i % 2 == 0:

pheno_utils/config.py

@@ -1,11 +1,11 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_config.ipynb.
 
 # %% auto 0
-__all__ = ['REF_COLOR', 'FEMALE_COLOR', 'MALE_COLOR', 'ALL_COLOR', 'GLUC_COLOR', 'FOOD_COLOR', 'DATASETS_PATH', 'COHORT',
-           'EVENTS_DATASET', 'ERROR_ACTION', 'CONFIG_FILES', 'BULK_DATA_PATH', 'PREFERRED_LANGUAGE', 'config_found',
-           'DICT_PROPERTY_PATH', 'DATA_CODING_PATH', 'copy_tre_config', 'get_dictionary_properties_file_path',
-           'get_data_coding_file_path', 'generate_synthetic_data', 'generate_synthetic_data_like',
-           'generate_categorical_synthetic_data']
+__all__ = ['DEFAULT_PALETTE', 'REF_COLOR', 'FEMALE_COLOR', 'MALE_COLOR', 'ALL_COLOR', 'GLUC_COLOR', 'FOOD_COLOR', 'LEGEND_SHIFT',
+           'TIME_FORMAT', 'DATASETS_PATH', 'COHORT', 'EVENTS_DATASET', 'ERROR_ACTION', 'CONFIG_FILES', 'BULK_DATA_PATH',
+           'PREFERRED_LANGUAGE', 'config_found', 'DICT_PROPERTY_PATH', 'DATA_CODING_PATH', 'copy_tre_config',
+           'get_dictionary_properties_file_path', 'get_data_coding_file_path', 'generate_synthetic_data',
+           'generate_synthetic_data_like', 'generate_categorical_synthetic_data']
 
 # %% ../nbs/00_config.ipynb 3
 import os
@@ -16,6 +16,7 @@
 from glob import glob
 
 # %% ../nbs/00_config.ipynb 4
+DEFAULT_PALETTE = 'muted'
 REF_COLOR = "k"
 FEMALE_COLOR = "C1"
 MALE_COLOR = "C0"
@@ -24,6 +25,9 @@
 GLUC_COLOR = "C0"
 FOOD_COLOR = "C1"
 
+LEGEND_SHIFT = (1.05, 1.05)
+TIME_FORMAT = '%d/%m\n%H:%M'
+
 DATASETS_PATH = '/home/ec2-user/studies/hpp_datasets/'
 COHORT = None
 EVENTS_DATASET = 'events'
@@ -34,8 +38,6 @@
 
 config_found = False
 
-
-
 # %% ../nbs/00_config.ipynb 5
 def copy_tre_config():
     default_config_found = False