more robust COSM submit error handling

vpyshnenko · Mar 15, 2013 · 08f051e · 08f051e
1 parent 19d7dee
commit 08f051e
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 13 deletions.
diff --git a/cosm_submit.py b/cosm_submit.py
@@ -126,6 +126,7 @@ def main():
             temps={}
             volts={}
             n={}
+            normal_exit = True
             for l in f:
                 c = string.strip(l).split(",")
                 w=float(c[0])
@@ -143,23 +144,40 @@ def main():
                     volts[ch]+=string.join([ts,v],",")+ "\r\n"
                     n[ch]+=1
                     if n[ch]==MAX_DATAPOINTS:
-                        if not debug_mode:                        
-                            for ch in n:
-                                cosm.submit_datapoints(feed,ch,key,temps[ch])
-                                # Voltage datastream is 100+temp datasteam
-                                cosm.submit_datapoints(feed,ch+100,key,volts[ch])
-                            write_watermark(WATERMARK_FILE % feed,w)
+                        if not debug_mode:
+                            try:
+                                for ch in n:
+                                    cosm.submit_datapoints(feed,ch,key,temps[ch])
+                                    # Voltage datastream is 100+temp datasteam
+                                    cosm.submit_datapoints(feed,ch+100,key,volts[ch])
+                            except Exception, ex:
+                                log.error("Error sending to COSM: %s" % ex )
+                                normal_exit = False
+                                break
+                            try:
+                                write_watermark(WATERMARK_FILE % feed,w)
+                            except Exception, ex:
+                                log.error("Error writing watermark: %s" % ex )
+                                sys.exit(1)
                         watermark = w
                         temps={}
                         volts={}
                         n={}
-
-            if not debug_mode:                        
+            # End of loop. Send stuff remaining since last sumbit.
+            # However if we exited due to error, do not try to write again.
+            if not debug_mode and normal_exit:
                 for ch in n:
-                    cosm.submit_datapoints(feed,ch,key,temps[ch])
-                    # Voltage datastream is 100+temp datasteam
-                    cosm.submit_datapoints(feed,ch+100,key,volts[ch])
-                write_watermark(WATERMARK_FILE % feed,w)
+                    try:
+                        cosm.submit_datapoints(feed,ch,key,temps[ch])
+                        # Voltage datastream is 100+temp datasteam
+                        cosm.submit_datapoints(feed,ch+100,key,volts[ch])
+                    except Exception, ex:
+                        log.error("Error sending to COSM: %s" % ex )
+                try:
+                    write_watermark(WATERMARK_FILE % feed,w)
+                except Exception, ex:
+                    log.error("Error writing watermark: %s" % ex )
+                    sys.exit(1)
         finally:
             f.close()
 

diff --git a/wu_temp_collector.py b/wu_temp_collector.py
@@ -176,7 +176,7 @@ def main():
                         cosm_report =string.join([ts,str(temp_c)],",") + "\r\n"
                         cosm.submit_datapoints(cosm_feed,cosm_datastream,cosm_key,cosm_report)
                     except Exception, ex:
-                        # Error sending to cosm is non-fatal, but logged anyway
+                        # Error sending to COSM is non-fatal, but logged anyway
                         log.error("Error sending to COSM: %s" % ex )
 
             if sleep_time>0: