From cb133de2c790eccf02ae8648e67c3d00c6590b5c Mon Sep 17 00:00:00 2001 From: Haroon Ghori <65193601+hsghori@users.noreply.github.com> Date: Sat, 18 Dec 2021 10:53:46 -0800 Subject: [PATCH] Fix python script and allow autotune prep/core to write to file directly (#1394) --- bin/oref0-autotune-core.js | 16 +++- bin/oref0-autotune-prep.js | 14 ++- bin/oref0-autotune.py | 171 ++++++++++++++++++------------------- requirements.txt | 3 + 4 files changed, 107 insertions(+), 97 deletions(-) create mode 100644 requirements.txt diff --git a/bin/oref0-autotune-core.js b/bin/oref0-autotune-core.js index 439e8ac0d..be51dae7b 100755 --- a/bin/oref0-autotune-core.js +++ b/bin/oref0-autotune-core.js @@ -5,7 +5,7 @@ Uses the output of oref0-autotune-prep.js - Calculates adjustments to basal schedule, ISF, and CSF + Calculates adjustments to basal schedule, ISF, and CSF Released under MIT license. See the accompanying LICENSE.txt file for full terms and conditions @@ -19,13 +19,17 @@ THE SOFTWARE. */ - var autotune = require('../lib/autotune'); var stringify = require('json-stable-stringify'); if (!module.parent) { var argv = require('yargs') - .usage("$0 ") + .usage("$0 [--output-file=]") + .option('output-file', { + alias: 'o', + describe: 'File to write output', + default: null, + }) .demand(3) .strict(true) .help('help'); @@ -65,6 +69,10 @@ if (!module.parent) { }; var autotune_output = autotune(inputs); - console.log(stringify(autotune_output, { space: ' '})); + if (params["output-file"]) { + fs.writeFileSync(params["output-file"], stringify(autotune_output, {space: ' '})); + } else { + console.log(stringify(autotune_output, { space: ' '})); + } } diff --git a/bin/oref0-autotune-prep.js b/bin/oref0-autotune-prep.js index 8977a570a..4c781cb5c 100755 --- a/bin/oref0-autotune-prep.js +++ b/bin/oref0-autotune-prep.js @@ -27,7 +27,7 @@ var moment = require('moment'); if (!module.parent) { var argv = require('yargs') - .usage("$0 [] [--categorize_uam_as_basal] [--tune-insulin-curve]") + .usage("$0 [] [--categorize_uam_as_basal] [--tune-insulin-curve] [--output-file=]") .option('categorize_uam_as_basal', { alias: 'u', boolean: true, @@ -40,6 +40,11 @@ if (!module.parent) { describe: "Tune peak time and end time", default: false }) + .option('output-file', { + alias: 'o', + describe: 'Output file to write output', + default: null, + }) .strict(true) .help('help'); @@ -66,7 +71,6 @@ if (!module.parent) { console.log('{ "error": "Could not parse input data" }'); return console.error("Could not parse input data: ", e); } - var pumpprofile_data = { }; if (typeof pumpprofile_input !== 'undefined') { try { @@ -129,6 +133,10 @@ if (!module.parent) { }; var prepped_glucose = generate(inputs); - console.log(JSON.stringify(prepped_glucose)); + if (params['output-file']) { + fs.writeFileSync(params['output-file'], JSON.stringify(prepped_glucose)) + } else { + console.log(JSON.stringify(prepped_glucose)); + } } diff --git a/bin/oref0-autotune.py b/bin/oref0-autotune.py index c936641c1..a6ec91989 100755 --- a/bin/oref0-autotune.py +++ b/bin/oref0-autotune.py @@ -2,15 +2,15 @@ # Python version of oref0-autotune.sh # Original bash code: scottleibrand, pietergit, beached, danamlewis -# This script sets up an easy test environment for autotune, allowing the user to vary parameters +# This script sets up an easy test environment for autotune, allowing the user to vary parameters # like start/end date and number of runs. # -# Required Inputs: +# Required Inputs: # DIR, (--dir=) # NIGHTSCOUT_HOST, (--ns-host=) # Optional Inputs: -# END_DATE, (--end-date=) +# END_DATE, (--end-date=) # if no end date supplied, assume we want a months worth or until day before current day # NUMBER_OF_RUNS (--runs=) # if no number of runs designated, then default to 5 @@ -25,29 +25,22 @@ import datetime import os, errno import logging +import pytz from subprocess import call import shutil +import six -DIR = '' -NIGHTSCOUT_HOST = '' -START_DATE = datetime.datetime.today() - datetime.timedelta(days=1) -END_DATE = datetime.datetime.today() -NUMBER_OF_RUNS = 1 -EXPORT_EXCEL = None -TERMINAL_LOGGING = True -RECOMMENDS_REPORT = True - def get_input_arguments(): parser = argparse.ArgumentParser(description='Autotune') - + # Required # NOTE: As the code runs right now, this directory needs to exist and as well as the subfolders: autotune, settings parser.add_argument('--dir', '-d', type=str, required=True, - help='(--dir=)') + help='(--dir=)') parser.add_argument('--ns-host', '-n', type=str, @@ -73,56 +66,46 @@ def get_input_arguments(): '-x', type=str, metavar='EXPORT_EXCEL', - help='(--xlsx=)') + help='(--xlsx=)') parser.add_argument('--log', '-l', - type=str, + type=bool, + default=True, metavar='TERMINAL_LOGGING', help='(--log )') - + return parser.parse_args() def assign_args_to_variables(args): # TODO: Input checking. - - global DIR, NIGHTSCOUT_HOST, START_DATE, END_DATE, NUMBER_OF_RUNS, \ - EXPORT_EXCEL, TERMINAL_LOGGING, RECOMMENDS_REPORT - + # On Unix and Windows, return the argument with an initial component of # ~ or ~user replaced by that user's home directory. - DIR = os.path.expanduser(args.dir) - - NIGHTSCOUT_HOST = args.ns_host - - START_DATE = args.start_date - - if args.end_date is not None: - END_DATE = args.end_date - - if args.runs is not None: - NUMBER_OF_RUNS = args.runs - - if args.xlsx is not None: - EXPORT_EXCEL = args.xlsx - - if args.log is not None: - RECOMMENDS_REPORT = args.logs - -def get_nightscout_profile(nightscout_host): + directory = os.path.expanduser(args.dir) + nightscout_host = args.ns_host + start_date = args.start_date + end_date = args.end_date or datetime.datetime.today() + number_of_runs = args.runs or 1 + export_excel = args.xlsx + recommends_report = args.log + + return directory, nightscout_host, start_date, end_date, number_of_runs, export_excel, recommends_report + +def get_nightscout_profile(nightscout_host, directory): #TODO: Add ability to use API secret for Nightscout. res = requests.get(nightscout_host + '/api/v1/profile.json') - with open(os.path.join(autotune_directory, 'nightscout.profile.json'), 'w') as f: # noqa: F821 - f.write(res.text) + with open(os.path.join(directory, 'autotune', 'nightscout.profile.json'), 'w') as f: # noqa: F821 + f.write(six.ensure_str(res.text, encoding='utf-8')) def get_openaps_profile(directory): shutil.copy(os.path.join(directory, 'settings', 'pumpprofile.json'), os.path.join(directory, 'autotune', 'profile.pump.json')) - + # If a previous valid settings/autotune.json exists, use that; otherwise start from settings/profile.json - + # This allows manual users to be able to run autotune by simply creating a settings/pumpprofile.json file. # cp -up settings/pumpprofile.json settings/profile.json shutil.copy(os.path.join(directory, 'settings', 'pumpprofile.json'), os.path.join(directory, 'settings', 'profile.json')) - + # TODO: Get this to work. For now, just copy from settings/profile.json each time. # If a previous valid settings/autotune.json exists, use that; otherwise start from settings/profile.json # cp settings/autotune.json autotune/profile.json && cat autotune/profile.json | json | grep -q start || cp autotune/profile.pump.json autotune/profile.json @@ -130,26 +113,34 @@ def get_openaps_profile(directory): # print create_autotune_json # call(create_autotune_json, shell=True) - # cp settings/autotune.json autotune/profile.json + # cp settings/profile.json settings/autotune.json shutil.copy(os.path.join(directory, 'settings', 'profile.json'), os.path.join(directory, 'settings', 'autotune.json')) - + # cp settings/autotune.json autotune/profile.json shutil.copy(os.path.join(directory, 'settings', 'autotune.json'), os.path.join(directory, 'autotune', 'profile.json')) - + + # cp settings/autotune.json autotune/pumpprofile.json + shutil.copy(os.path.join(directory, 'settings', 'autotune.json'), os.path.join(directory, 'autotune', 'pumpprofile.json')) + #TODO: Do the correct copying here. # cat autotune/profile.json | json | grep -q start || cp autotune/profile.pump.json autotune/profile.json']) def get_nightscout_carb_and_insulin_treatments(nightscout_host, start_date, end_date, directory): logging.info('Grabbing NIGHTSCOUT treatments.json for date range: {0} to {1}'.format(start_date, end_date)) - # TODO: What does 'T20:00-05:00' mean? output_file_name = os.path.join(directory, 'autotune', 'ns-treatments.json') - start_date = start_date.strftime("%Y-%m-%d") + 'T20:00-05:00' - end_date = end_date.strftime("%Y-%m-%d") + 'T20:00-05:00' + + def _normalize_datetime(dt): + dt = dt.replace(hour=20, minute=0, second=0, microsecond=0, tzinfo=None) + dt = pytz.timezone('US/Eastern').localize(dt) + return dt + + start_date = _normalize_datetime(start_date) + end_date = _normalize_datetime(end_date) url='{0}/api/v1/treatments.json?find\[created_at\]\[\$gte\]=`date --date="{1} -4 hours" -Iminutes`&find\[created_at\]\[\$lte\]=`date --date="{2} +1 days" -Iminutes`'.format(nightscout_host, start_date, end_date) #TODO: Add ability to use API secret for Nightscout. res = requests.get(url) with open(output_file_name, 'w') as f: - f.write(res.text.encode('utf-8')) + f.write(six.ensure_str(res.text, 'utf-8')) def get_nightscout_bg_entries(nightscout_host, start_date, end_date, directory): logging.info('Grabbing NIGHTSCOUT enries/sgv.json for date range: {0} to {1}'.format(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d"))) @@ -161,50 +152,50 @@ def get_nightscout_bg_entries(nightscout_host, start_date, end_date, directory): #TODO: Add ability to use API secret for Nightscout. res = requests.get(url) with open(os.path.join(directory, 'autotune', 'ns-entries.{date}.json'.format(date=date.strftime("%Y-%m-%d"))), 'w') as f: - f.write(res.text.encode('utf-8')) + f.write(six.ensure_str(res.text, 'utf-8')) def run_autotune(start_date, end_date, number_of_runs, directory): date_list = [start_date + datetime.timedelta(days=x) for x in range(0, (end_date - start_date).days)] autotune_directory = os.path.join(directory, 'autotune') + FNULL = open(os.devnull, 'w') for run_number in range(1, number_of_runs + 1): for date in date_list: # cp profile.json profile.$run_number.$i.json shutil.copy(os.path.join(autotune_directory, 'profile.json'), os.path.join(autotune_directory, 'profile.{run_number}.{date}.json' .format(run_number=run_number, date=date.strftime("%Y-%m-%d")))) - - # Autotune Prep (required args, ), output prepped glucose + + # Autotune Prep (required args, ), output prepped glucose # data or below # oref0-autotune-prep ns-treatments.json profile.json ns-entries.$DATE.json > autotune.$RUN_NUMBER.$DATE.json ns_treatments = os.path.join(autotune_directory, 'ns-treatments.json') profile = os.path.join(autotune_directory, 'profile.json') + pump_profile = os.path.join(autotune_directory, "pumpprofile.json") ns_entries = os.path.join(autotune_directory, 'ns-entries.{date}.json'.format(date=date.strftime("%Y-%m-%d"))) - autotune_prep = 'oref0-autotune-prep {ns_treatments} {profile} {ns_entries}'.format(ns_treatments=ns_treatments, profile=profile, ns_entries=ns_entries) - - # autotune.$RUN_NUMBER.$DATE.json + + # autotune.$RUN_NUMBER.$DATE.json autotune_run_filename = os.path.join(autotune_directory, 'autotune.{run_number}.{date}.json' .format(run_number=run_number, date=date.strftime("%Y-%m-%d"))) - with open(autotune_run_filename, "w+") as output: - logging.info('Running {script}'.format(script=autotune_prep)) - call(autotune_prep, stdout=output, shell=True) - logging.info('Writing output to {filename}'.format(filename=autotune_run_filename)) - - # Autotune (required args, ), + autotune_prep = 'oref0-autotune-prep {ns_treatments} {profile} {ns_entries} {pump_profile} --output-file {autotune_run_filename}'.format(ns_treatments=ns_treatments, profile=profile, ns_entries=ns_entries, pump_profile=pump_profile, autotune_run_filename=autotune_run_filename) + logging.info('Running {script}'.format(script=autotune_prep)) + call(autotune_prep, stdout=FNULL, shell=True) + logging.info('Writing output to {filename}'.format(filename=autotune_run_filename)) + + # Autotune (required args, ), # output autotuned profile or what will be used as in the next iteration # oref0-autotune-core autotune.$RUN_NUMBER.$DATE.json profile.json profile.pump.json > newprofile.$RUN_NUMBER.$DATE.json - + # oref0-autotune-core autotune.$run_number.$i.json profile.json profile.pump.json > newprofile.$RUN_NUMBER.$DATE.json profile_pump = os.path.join(autotune_directory, 'profile.pump.json') - autotune_core = 'oref0-autotune-core {autotune_run} {profile} {profile_pump}'.format(profile=profile, profile_pump = profile_pump, autotune_run=autotune_run_filename) - + # newprofile.$RUN_NUMBER.$DATE.json newprofile_run_filename = os.path.join(autotune_directory, 'newprofile.{run_number}.{date}.json' .format(run_number=run_number, date=date.strftime("%Y-%m-%d"))) - with open(newprofile_run_filename, "w+") as output: - logging.info('Running {script}'.format(script=autotune_core)) - call(autotune_core, stdout=output, shell=True) - logging.info('Writing output to {filename}'.format(filename=autotune_run_filename)) - + autotune_core = 'oref0-autotune-core {autotune_run} {profile} {profile_pump} --output-file {newprofile_run_filename}'.format(profile=profile, profile_pump = profile_pump, autotune_run=autotune_run_filename, newprofile_run_filename=newprofile_run_filename) + logging.info('Running {script}'.format(script=autotune_core)) + call(autotune_core, stdout=FNULL, shell=True) + logging.info('Writing output to {filename}'.format(filename=newprofile_run_filename)) + # Copy tuned profile produced by autotune to profile.json for use with next day of data # cp newprofile.$RUN_NUMBER.$DATE.json profile.json shutil.copy(os.path.join(autotune_directory, 'newprofile.{run_number}.{date}.json'.format(run_number=run_number, date=date.strftime("%Y-%m-%d"))), @@ -218,13 +209,13 @@ def create_summary_report_and_display_results(output_directory): print() print("Autotune pump profile recommendations:") print("---------------------------------------------------------") - + report_file = os.path.join(output_directory, 'autotune', 'autotune_recommendations.log') autotune_recommends_report = 'oref0-autotune-recommends-report {0}'.format(output_directory) - + call(autotune_recommends_report, shell=True) print("Recommendations Log File: {0}".format(report_file)) - + # Go ahead and echo autotune_recommendations.log to the terminal, minus blank lines # cat $report_file | egrep -v "\| *\| *$" call(['cat {0} | egrep -v "\| *\| *$"'.format(report_file)], shell=True) @@ -234,20 +225,20 @@ def create_summary_report_and_display_results(output_directory): logging.basicConfig(level=logging.DEBUG) # Supress non-essential logs (below WARNING) from requests module. logging.getLogger("requests").setLevel(logging.WARNING) - + args = get_input_arguments() - assign_args_to_variables(args) - + directory, nightscout_host, start_date, end_date, number_of_runs, export_excel, recommends_report = assign_args_to_variables(args) + # TODO: Convert Nightscout profile to OpenAPS profile format. - #get_nightscout_profile(NIGHTSCOUT_HOST) - - get_openaps_profile(DIR) - get_nightscout_carb_and_insulin_treatments(NIGHTSCOUT_HOST, START_DATE, END_DATE, DIR) - get_nightscout_bg_entries(NIGHTSCOUT_HOST, START_DATE, END_DATE, DIR) - run_autotune(START_DATE, END_DATE, NUMBER_OF_RUNS, DIR) - - if EXPORT_EXCEL: - export_to_excel(DIR, EXPORT_EXCEL) - - if RECOMMENDS_REPORT: - create_summary_report_and_display_results(DIR) + #get_nightscout_profile(NIGHTSCOUT_HOST, DIR) + + get_openaps_profile(directory) + get_nightscout_carb_and_insulin_treatments(nightscout_host, start_date, end_date, directory) + get_nightscout_bg_entries(nightscout_host, start_date, end_date, directory) + run_autotune(start_date, end_date, number_of_runs, directory) + + if export_excel: + export_to_excel(directory, export_excel) + + if recommends_report: + create_summary_report_and_display_results(directory) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..b51dedf39 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +requests==2.25.1 +six==1.15.0 +pytz==2021.1