in-progress: support BOLUS_BG and CGM features.

BOLUS_BG will add BG readings which are associated with boluses
on the pump into the Nightscout treatment object. It will determine
whether the BG reading was automatically filled via the Dexcom
connection on the pump or was manually entered by seeing if the
BG reading matches the current CGM reading as known to the pump
at that time. Support for this is nearly complete.

CGM will add Dexcom CGM readings from the pump to Nightscout
as SGV (sensor glucose value) entries. This should only be
used in a situation where xDrip is not used and the pump
connection to the CGM will be the only source of CGM
data to Nightscout. This requires additional testing before
it should be considered ready.

Both options are hidden behind the ENABLE_TESTING_MODES=true
environment variable. Run tconnectsync with this environment
variable and BOLUS_BG and CGM will be usable with the
--features argument.

tconnectsync/features.py

@@ -1,8 +1,11 @@
-"""Supported synchronization features."""
+from .secret import ENABLE_TESTING_MODES
 
+"""Supported synchronization features."""
 BASAL = "BASAL"
 BOLUS = "BOLUS"
 IOB = "IOB"
+BOLUS_BG = "BOLUS_BG"
+CGM = "CGM"
 
 DEFAULT_FEATURES = [
     BASAL,
@@ -14,4 +17,12 @@
     BASAL,
     BOLUS,
     IOB
-]
+]
+
+
+# These modes are not yet ready for wide use.
+if ENABLE_TESTING_MODES:
+    ALL_FEATURES += [
+        BOLUS_BG,
+        CGM
+    ]

tconnectsync/nightscout.py

@@ -53,7 +53,19 @@ def last_uploaded_entry(self, eventType):
 			'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
 		})
 		if latest.status_code != 200:
-			raise ApiException(latest.status_code, "Nightscout treatments response: %s" % latest.text)
+			raise ApiException(latest.status_code, "Nightscout last_uploaded_entry response: %s" % latest.text)
+
+		j = latest.json()
+		if j and len(j) > 0:
+			return j[0]
+		return None
+
+	def last_uploaded_bg_entry(self):
+		latest = requests.get(urljoin(self.url, 'api/v1/entries.json?count=1&find[device]=' + urllib.parse.quote(ENTERED_BY) + '&ts=' + str(time.time())), headers={
+			'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
+		})
+		if latest.status_code != 200:
+			raise ApiException(latest.status_code, "Nightscout last_uploaded_bg_entry response: %s" % latest.text)
 
 		j = latest.json()
 		if j and len(j) > 0:

tconnectsync/parser/nightscout.py

@@ -1,8 +1,11 @@
+import arrow
+
 ENTERED_BY = "Pump (tconnectsync)"
 BASAL_EVENTTYPE = "Temp Basal"
 BOLUS_EVENTTYPE = "Combo Bolus"
 IOB_ACTIVITYTYPE = "tconnect_iob"
 
+
 """
 Conversion methods for parsing data into Nightscout objects.
 """
@@ -21,16 +24,30 @@ def basal(value, duration_mins, created_at, reason=""):
             "enteredBy": ENTERED_BY
         }
 
+    # Note that Nightscout is not consistent and uses "Sensor"/"Finger"
+    # for treatment objects, unlike "sgv"/"mbg" for entries
+    SENSOR = "Sensor"
+    FINGER = "Finger"
+
     @staticmethod
-    def bolus(bolus, carbs, created_at, notes=""):
-        return {
+    def bolus(bolus, carbs, created_at, notes="", bg="", bg_type=""):
+        data = {
             "eventType": BOLUS_EVENTTYPE,
 			"created_at": created_at,
 			"carbs": int(carbs),
 			"insulin": float(bolus),
 			"notes": notes,
 			"enteredBy": ENTERED_BY,
         }
+        if bg:
+            if bg_type not in (NightscoutEntry.SENSOR, NightscoutEntry.FINGER):
+                raise InvalidBolusTypeException
+
+            data.update({
+                "glucose": str(bg),
+                "glucoseType": bg_type
+            })
+        return data
 
     @staticmethod
     def iob(iob, created_at):
@@ -39,4 +56,18 @@ def iob(iob, created_at):
             "iob": float(iob),
             "created_at": created_at,
             "enteredBy": ENTERED_BY
-        }
+        }
+
+    @staticmethod
+    def entry(sgv, created_at):
+        return {
+            "type": "sgv",
+            "sgv": int(sgv),
+            "date": int(1000 * arrow.get(created_at).timestamp()),
+            "dateString": arrow.get(created_at).strftime('%Y-%m-%dT%H:%M:%S%z'),
+            "device": ENTERED_BY,
+            # delta, direction are undefined
+        }
+
+class InvalidBolusTypeException(RuntimeError):
+    pass

tconnectsync/parser/tconnect.py

@@ -100,8 +100,17 @@ def is_complete(s):
             "insulin": data["InsulinDelivered"],
             "requested_insulin": data["ActualTotalBolusRequested"],
             "carbs": data["CarbSize"],
+            "bg": data["BG"], # Note: can be empty string for automatic Control-IQ boluses
             "user_override": data["UserOverride"],
             "extended_bolus": "1" if extended_bolus else "",
             "bolex_completion_time": TConnectEntry._datetime_parse(data["BolexCompletionDateTime"]).format() if complete and extended_bolus else None,
             "bolex_start_time": TConnectEntry._datetime_parse(data["BolexStartDateTime"]).format() if complete and extended_bolus else None,
+        }
+
+    @staticmethod
+    def parse_reading_entry(data):
+        return {
+            "time": TConnectEntry._datetime_parse(data["EventDateTime"]).format(),
+            "bg": data["Readings (CGM / BGM)"],
+            "type": data["Description"]
         }

tconnectsync/process.py

@@ -18,8 +18,12 @@
     process_iob_events,
     ns_write_iob_events
 )
+from .sync.cgm import (
+    process_cgm_events,
+    ns_write_cgm_events
+)
 from .parser.tconnect import TConnectEntry
-from .features import BASAL, BOLUS, IOB, DEFAULT_FEATURES
+from .features import BASAL, BOLUS, IOB, BOLUS_BG, CGM, DEFAULT_FEATURES
 
 logger = logging.getLogger(__name__)
 
@@ -60,6 +64,15 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend, feat
 
     added = 0
 
+    cgmData = None
+    if CGM in features or BOLUS_BG in features:
+        logger.debug("Processing CGM events")
+        cgmData = process_cgm_events(readingData)
+
+    if CGM in features:
+        logger.debug("Writing CGM events")
+        added += ns_write_cgm_events(nightscout, cgmData, pretend)
+
     if BASAL in features:
         basalEvents = process_ciq_basal_events(ciqTherapyTimelineData)
         if csvBasalData:
@@ -72,7 +85,7 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend, feat
 
     if BOLUS in features:
         bolusEvents = process_bolus_events(bolusData)
-        added += ns_write_bolus_events(nightscout, bolusEvents, pretend=pretend)
+        added += ns_write_bolus_events(nightscout, bolusEvents, pretend=pretend, include_bg=(BOLUS_BG in features))
 
     if IOB in features:
         iobEvents = process_iob_events(iobData)

tconnectsync/secret.py

@@ -36,11 +36,7 @@ def get_bool(name, default):
 AUTOUPDATE_FAILURE_MINUTES = get_number('AUTOUPDATE_FAILURE_MINUTES', '180') # 3 hours
 AUTOUPDATE_RESTART_ON_FAILURE = get_bool('AUTOUPDATE_RESTART_ON_FAILURE', 'false')
 
-_config = ['TCONNECT_EMAIL', 'TCONNECT_PASSWORD', 'PUMP_SERIAL_NUMBER',
-          'NS_URL', 'NS_SECRET', 'TIMEZONE_NAME',
-          'AUTOUPDATE_DEFAULT_SLEEP_SECONDS', 'AUTOUPDATE_MAX_SLEEP_SECONDS',
-          'AUTOUPDATE_USE_FIXED_SLEEP', 'AUTOUPDATE_FAILURE_MINUTES',
-          'AUTOUPDATE_RESTART_ON_FAILURE']
+ENABLE_TESTING_MODES = get_bool('ENABLE_TESTING_MODES', 'false')
 
 if __name__ == '__main__':
     for k in locals():

tconnectsync/sync/bolus.py

@@ -1,6 +1,8 @@
 import arrow
 import logging
 
+from tconnectsync.sync.cgm import find_event_at
+
 from ..parser.nightscout import (
     BOLUS_EVENTTYPE,
     NightscoutEntry
@@ -12,7 +14,7 @@
 """
 Given bolus data input from the therapy timeline CSV, converts it into a digestable format.
 """
-def process_bolus_events(bolusdata):
+def process_bolus_events(bolusdata, cgmEvents=None):
     bolusEvents = []
 
     for b in bolusdata:
@@ -24,16 +26,37 @@ def process_bolus_events(bolusdata):
             else:
                 logger.warning("Skipping non-completed bolus data (was a bolus in progress?): %s parsed: %s" % (b, parsed))
                 continue
+
+        if parsed["bg"] and cgmEvents:
+            requested_at = parsed["request_time"] if not parsed["extended_bolus"] else parsed["bolex_start_time"]
+            parsed["bg_type"] = guess_bolus_bg_type(parsed["bg"], requested_at, cgmEvents)
+
         bolusEvents.append(parsed)
 
-    bolusEvents.sort(key=lambda event: arrow.get(event["completion_time"] if not event["extended_bolus"] else event["bolex_start_time"]))
+    bolusEvents.sort(key=lambda event: arrow.get(event["request_time"] if not event["extended_bolus"] else event["bolex_start_time"]))
 
     return bolusEvents
 
+"""
+Determine whether the given BG specified in the bolus is identical to the
+most recent CGM reading at that time. If it is, return SENSOR.
+Otherwise, return FINGER.
+"""
+def guess_bolus_bg_type(bg, created_at, cgmEvents):
+    if not cgmEvents:
+        return NightscoutEntry.FINGER
+
+    event = find_event_at(cgmEvents, created_at)
+    if event and str(event["bg"]) == str(bg):
+        return NightscoutEntry.SENSOR
+
+    return NightscoutEntry.FINGER
+
+
 """
 Given processed bolus data, adds bolus events to Nightscout.
 """
-def ns_write_bolus_events(nightscout, bolusEvents, pretend=False):
+def ns_write_bolus_events(nightscout, bolusEvents, pretend=False, include_bg=False, reading_events=None):
     logger.debug("ns_write_bolus_events: querying for last uploaded entry")
     last_upload = nightscout.last_uploaded_entry(BOLUS_EVENTTYPE)
     last_upload_time = None
@@ -49,12 +72,22 @@ def ns_write_bolus_events(nightscout, bolusEvents, pretend=False):
                 logger.info("Skipping basal event before last upload time: %s" % event)
             continue
 
-        entry = NightscoutEntry.bolus(
-            bolus=event["insulin"],
-            carbs=event["carbs"],
-            created_at=created_at,
-            notes="{}{}{}".format(event["description"], " (Override)" if event["user_override"] == "1" else "", " (Extended)" if event["extended_bolus"] == "1" else "")
-        )
+        if include_bg and event["bg"]:
+            entry = NightscoutEntry.bolus(
+                bolus=event["insulin"],
+                carbs=event["carbs"],
+                created_at=created_at,
+                notes="{}{}{}".format(event["description"], " (Override)" if event["user_override"] == "1" else "", " (Extended)" if event["extended_bolus"] == "1" else ""),
+                bg=event["bg"],
+                bg_type=event["bg_type"]
+            )
+        else:
+            entry = NightscoutEntry.bolus(
+                bolus=event["insulin"],
+                carbs=event["carbs"],
+                created_at=created_at,
+                notes="{}{}{}".format(event["description"], " (Override)" if event["user_override"] == "1" else "", " (Extended)" if event["extended_bolus"] == "1" else "")
+            )
 
         add_count += 1
 

tconnectsync/sync/cgm.py

@@ -0,0 +1,69 @@
+import json
+import arrow
+import logging
+
+from ..parser.tconnect import TConnectEntry
+from ..parser.nightscout import NightscoutEntry
+
+logger = logging.getLogger(__name__)
+
+def process_cgm_events(readingData):
+    data = []
+    for r in readingData:
+        data.append(TConnectEntry.parse_reading_entry(r))
+
+    return data
+
+"""
+Given reading data and a time, finds the BG reading event which would have
+been the current one at that time. e.g., it looks before the given time,
+not after.
+This is a heuristic for checking whether the BG component of a bolus was
+manually entered or inferred based on the pump's CGM.
+"""
+def find_event_at(cgmEvents, find_time):
+    find_t = arrow.get(find_time)
+    events = list(map(lambda x: (arrow.get(x["time"]), x), cgmEvents))
+    events.sort()
+
+    closestReading = None
+    for t, r in events:
+        if t > find_t:
+            break
+        closestReading = r
+
+
+    return closestReading
+
+
+"""
+Given processed CGM data, adds reading entries to Nightscout.
+"""
+def ns_write_cgm_events(nightscout, cgmEvents, pretend=False):
+    logger.debug("ns_write_cgm_events: querying for last uploaded entry")
+    last_upload = nightscout.last_uploaded_bg_entry()
+    last_upload_time = None
+    if last_upload:
+        last_upload_time = arrow.get(last_upload["dateString"])
+    logger.info("Last Nightscout CGM upload: %s" % last_upload_time)
+
+    add_count = 0
+    for event in cgmEvents:
+        created_at = event["time"]
+        if last_upload_time and arrow.get(created_at) <= last_upload_time:
+            if pretend:
+                logger.info("Skipping CGM event before last upload time: %s" % event)
+            continue
+
+        entry = NightscoutEntry.entry(
+            sgv=event["bg"],
+            created_at=created_at
+        )
+
+        add_count += 1
+
+        logger.info("  Processing cgm reading: %s entry: %s" % (event, entry))
+        if not pretend:
+            nightscout.upload_entry(entry, entity='entries')
+
+    return add_count