workaround for hanging rig-pump communication (with WW pump) (#445)

* workaround for hanging rig-pump communication (with WW pump)

and improve logging, more robust killing of reset.py if it hangs

* fix scotts review remarks

* fix python closing "

* improve log statement for US pump users

* also create cgm-loop for G4-upload

bin/oref0-radio-reboot.sh

@@ -1,4 +1,13 @@
-radio_errors=`tail /var/log/openaps/pump-loop.log | grep "spidev5.1 already in use"`
+#!/bin/bash
+
+# There are 2 known conditions in which communication between rig and pump is not working and a reboot is required.
+# 1) spidev5.1 already in use, see https://github.com/openaps/oref0/pull/411 (all pumps)
+# 2) continuous 'retry 0' or hanging reset.py, see https://github.com/oskarpearson/mmeowlink/issues/60 (WW-pump users only)
+# If one of those occur we will reboot the rig
+# If it will restore within 5 minutes, then the reboot will be cancelled
+# Note that this is a workaround, until we found the root cause of why the rig pump communication fails
+
+radio_errors=`tail --lines=20 /var/log/openaps/pump-loop.log | egrep "spidev5.1 already in use|retry 0|TimeoutExpired. Killing process"`
 logfile=/var/log/openaps/pump-loop.log
 if [ ! -z "$radio_errors" ]; then
     if [ ! -e /run/nologin ]; then

bin/oref0-setup.sh

@@ -472,6 +472,7 @@ elif [[ ${CGM,,} =~ "shareble" ]]; then
     # comment out existing line if it exists and isn't already commented out
     sed -i"" 's/^screen -S "brcm_patchram_plus" -d -m \/usr\/local\/sbin\/bluetooth_patchram.sh/# &/' /etc/rc.local
 fi
+
 if [[ ${CGM,,} =~ "shareble" || ${CGM,,} =~ "g4-upload" ]]; then
     mkdir -p $directory-cgm-loop
     if ( cd $directory-cgm-loop && git status 2>/dev/null >/dev/null && openaps use -h >/dev/null ); then

bin/oref0_init_pump_comms.py

@@ -3,6 +3,8 @@
 # This script initializes the connection between the openaps environment
 # and the insulin pump
 # Currently supported features:
+# - Call reset function for spi_serial
+# - Initialize WW pumps
 
 import sys
 import logging
@@ -15,6 +17,7 @@ def run_script(args):
         logging.basicConfig(level=logging.ERROR, format='%(asctime)s %(levelname)s %(message)s')
     init_spi_serial()
     init_ww_pump(args)
+    logging.debug("Exit succesfully with exit code 0")
     sys.exit(0)
 
 def init_spi_serial():
@@ -28,12 +31,10 @@ def init_spi_serial():
         s = spi_serial.SpiSerial()
         logging.debug("Issuing spidev serial reset")
         s.reset()
-        logging.debug("Exit succesfully with exit code 0")
-        sys.exit(0)
     except ImportError: # silence import error by default
         logging.debug("spi_serial not installed. Assuming not using spidev")
-    except Exception as ex: 
-        logging.error("Exception in oref0-init-pump-comms spi_serial: %s" % ex)
+    except Exception: 
+        logging.exception("Exception in oref0-init-pump-comms spi_serial")
         sys.exit(1)
 
 def init_ww_pump(args):
@@ -42,27 +43,28 @@ def init_ww_pump(args):
         logging.debug("Import oref0_subg_ww_radio_parameters")
         import oref0_subg_ww_radio_parameters
         oref0_subg_ww_radio_parameters.main(args)
-        logging.debug("Exit succesfully with exit code 0")
-        sys.exit(0)
-    except ImportError: # silence import error by default
-        logging.debug("could not import oref0_subg_ww_radio_parameters")
-        sys.exit(0)
-    except Exception as ex: 
-        logging.error("Exception in oref0-init-pump-comms spi_serial: %s" % ex)
+    except ImportError: 
+        logging.debug("Could not import oref0_subg_ww_radio_parameters. Assuming US pump. This is no error")
+    except Exception:
+        logging.exception("Exception in oref0-init-pump-comms init_ww_pump:")
         sys.exit(1)
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Initializes the connection between the openaps environment and the insulin pump. It can reset_spi_serial and it will initalize the connetion to the World Wide pumps if necessary')
-    # default arguments
-    parser.add_argument('-v', '--verbose', action="store_true", help='increase output verbosity')
-    parser.add_argument('--version', action='version', version='%(prog)s 0.0.1-dev')
-    # arguments required for ww pump
-    parser.add_argument('-d', '--dir', help='openaps dir', default='.')
-    parser.add_argument('-t', '--timeout', type=int, help='timeout value for script', default=30)
-    parser.add_argument('-w', '--wait', type=int, help='wait time after command', default=0.5)
-    parser.add_argument('--pump_ini', help='filename for pump config file', default='pump.ini')
-    parser.add_argument('--ww_ti_usb_reset', type=str, help='call oref0_reset_usb command or not. Use \'yes\' only for TI USB and WW-pump. Default: no' , default='no')
-    parser.add_argument('--rfsypy_rtscts', type=int, help='sets the RFSPY_RTSCTS environment variable (set to 0 for ERF and TI USB)', default=0)
-    args = parser.parse_args()
-    run_script(args)
+    try:
+        parser = argparse.ArgumentParser(description='Initializes the connection between the openaps environment and the insulin pump. It can reset_spi_serial and it will initalize the connetion to the World Wide pumps if necessary')
+        # default arguments
+        parser.add_argument('-v', '--verbose', action="store_true", help='increase output verbosity')
+        parser.add_argument('--version', action='version', version='%(prog)s 0.0.1-dev')
+        # arguments required for ww pump
+        parser.add_argument('-d', '--dir', help='openaps dir', default='.')
+        parser.add_argument('-t', '--timeout', type=int, help='timeout value for script', default=30)
+        parser.add_argument('-w', '--wait', type=int, help='wait time after command', default=0.5)
+        parser.add_argument('--pump_ini', help='filename for pump config file', default='pump.ini')
+        parser.add_argument('--ww_ti_usb_reset', type=str, help='call oref0_reset_usb command or not. Use \'yes\' only for TI USB and WW-pump. Default: no' , default='no')
+        parser.add_argument('--rfsypy_rtscts', type=int, help='sets the RFSPY_RTSCTS environment variable (set to 0 for ERF and TI USB)', default=0)
+        args = parser.parse_args()
+        run_script(args)
+    except Exception:
+        logging.exception("Exception in oref0_init_pump_comms.py")
+

bin/oref0_subg_ww_radio_parameters.py

@@ -12,6 +12,7 @@
 import logging
 import subprocess
 import time
+import signal
 
 PORT_NOT_SET="port not set in pump.ini"
 RADIO_LOCALE_NOT_SET="radio_locale not set in pump.ini"
@@ -52,15 +53,9 @@ def execute(cmd, cmdtimeout, wait):
         return proc.returncode
     except subprocess.TimeoutExpired:
         logging.error("TimeoutExpired. Killing process")
-        outs, errs = proc.communicate()
         if proc.pid:
-            pgrp = os.getpgid(proc.pid)
-            if prgp:
-                logging.debug("Sending SIGINT to process %s" % prgp)
-                os.killpg(pgrp, signal.SIGINT)
-                time.sleep(5) # sleep 5 secons
-                logging.debug("Sending SIGTERM to process %s" % prgp)
-                os.killpg(pgrp, signal.SIGTERM)
+            os.kill(int(proc.pid), signal.SIGKILL)
+        logging.debug("Exit with status code 1")
         sys.exit(1)
 
 def main(args):
@@ -106,8 +101,8 @@ def main(args):
         # step 6: now set the subg ww radio parameters
         exitcode=execute(['oref0-subg-ww-radio-parameters', pump_port], args.timeout, args.wait)
         sys.exit(exitcode) # propagate exit code from oref0-subg-ww-radio-parameters
-    except Exception as ex:
-        logging.error("Exception: %s" % ex)
+    except Exception:
+        logging.exception("Exception in subg_ww_radio_parameters.py")
         sys.exit(1)