Merge pull request #1 from commaai/devel

updata from oirgin

.gitignore

@@ -1,6 +1,7 @@
 .DS_Store
 .tags
 .ipynb_checkpoints
+.idea
 model2.png
 
 *.d

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "opendbc"]
 	path = opendbc
 	url = https://github.com/commaai/opendbc.git
+[submodule "pyextra"]
+	path = pyextra
+	url = https://github.com/commaai/openpilot-pyextra.git

Dockerfile.openpilot

@@ -1,15 +1,9 @@
-FROM ubuntu:14.04
+FROM ubuntu:16.04
 ENV PYTHONUNBUFFERED 1
 
-RUN apt-get update && apt-get install -y build-essential screen wget bzip2 git libglib2.0-0
+RUN apt-get update && apt-get install -y build-essential clang vim screen wget bzip2 git libglib2.0-0 python-pip capnproto libcapnp-dev libzmq5-dev libffi-dev
 
-RUN wget -nv -O /tmp/anaconda.sh https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh && \
-  /bin/bash /tmp/anaconda.sh -b -p /opt/conda && \
-  rm /tmp/anaconda.sh
-
-ENV PATH /opt/conda/bin:$PATH
-
-RUN conda install numpy==1.11.2 && conda install scipy==0.18.1
+RUN pip install numpy==1.11.2 scipy==0.18.1 matplotlib
 
 COPY requirements_openpilot.txt /tmp/
 RUN pip install -r /tmp/requirements_openpilot.txt

README.md

@@ -101,6 +101,8 @@ Licensing
 
 openpilot is released under the MIT license.
 
+Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
+
 **THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
 YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
 NO WARRANTY EXPRESSED OR IMPLIED.**

RELEASES.md

@@ -1,3 +1,17 @@
+Version 0.3.5 (2017-07-30)
+==========================
+ * Fix bug where new devices would not begin calibration
+ * Minor robustness improvements
+
+Version 0.3.4 (2017-07-28)
+==========================
+ * Improved model trained on more data
+ * Much improved controls tuning
+ * Performance improvements
+ * Bugfixes and improvements to calibration
+ * Driving log can play back video
+ * Acura only: system now stays engaged below 25mph as long as brakes are applied
+
 Version 0.3.3  (2017-06-28)
 ===========================
   * Improved model trained on more data

cereal/__init__.py

@@ -1,8 +1,20 @@
 import os
 import capnp
-capnp.remove_import_hook()
 
 CEREAL_PATH = os.path.dirname(os.path.abspath(__file__))
-log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
-car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
+capnp.remove_import_hook()
+
+if os.getenv("NEWCAPNP"):
+  import tempfile
+  import pyximport
 
+  importers = pyximport.install(build_dir=os.path.join(tempfile.gettempdir(), ".pyxbld"))
+  try:
+    import cereal.gen.cython.log_capnp_cython as log
+    import cereal.gen.cython.car_capnp_cython as car
+  finally:
+    pyximport.uninstall(*importers)
+    del importers
+else:
+  log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
+  car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))

cereal/build_from_src.mk

@@ -1,14 +1,21 @@
 SRCS := log.capnp car.capnp
 
-GENS := gen/c/car.capnp.c gen/c/log.capnp.c gen/c/c++.capnp.h gen/c/java.capnp.h \
-	      gen/cpp/car.capnp.c++ gen/cpp/log.capnp.c++
+GENS := gen/cpp/car.capnp.c++ gen/cpp/log.capnp.c++
+
 
-# Dont build java on the phone...
 UNAME_M := $(shell uname -m)
+
+# only generate C++ for docker tests
+ifneq ($(OPTEST),1)
+	GENS += gen/c/car.capnp.c gen/c/log.capnp.c gen/c/c++.capnp.h gen/c/java.capnp.h
+
+# Dont build java on the phone...
 ifeq ($(UNAME_M),x86_64)
 	GENS += gen/java/Car.java gen/java/Log.java
 endif
 
+endif
+
 .PHONY: all
 all: $(GENS)
 

cereal/car.capnp

@@ -50,6 +50,7 @@ struct CarState {
   struct CruiseState {
     enabled @0: Bool;
     speed @1: Float32;
+    available @2: Bool;
   }
 
   enum Error {
@@ -63,7 +64,7 @@ struct CarState {
     seatbeltNotLatched @6;
     espDisabled @7;
     wrongCarMode @8;
-    steerTemporarilyUnavailable @9;
+    steerTempUnavailable @9;
     reverseGear @10;
     # ...
   }

cereal/log.capnp

@@ -140,12 +140,16 @@ struct SensorEventData {
   sensor @1 :Int32;
   type @2 :Int32;
   timestamp @3 :Int64;
+  uncalibratedDEPRECATED @10 :Bool;
+
   union {
     acceleration @4 :SensorVec;
     magnetic @5 :SensorVec;
     orientation @6 :SensorVec;
     gyro @7 :SensorVec;
     pressure @9 :SensorVec;
+    magneticUncalibrated @11 :SensorVec;
+    gyroUncalibrated @12 :SensorVec;
   }
   source @8 :SensorSource;
 
@@ -182,14 +186,26 @@ struct GpsLocationData {
   # Represents heading in degrees.
   bearing @5 :Float32;
 
-  # Represents expected accuracy in meters.
+  # Represents expected accuracy in meters. (presumably 1 sigma?)
   accuracy @6 :Float32;
 
   # Timestamp for the location fix.
   # Milliseconds since January 1, 1970.
   timestamp @7 :Int64;
 
   source @8 :SensorSource;
+
+  # Represents NED velocity in m/s.
+  vNED @9 :List(Float32);
+
+  # Represents expected vertical accuracy in meters. (presumably 1 sigma?)
+  verticalAccuracy @10 :Float32;
+
+  # Represents bearing accuracy in degrees. (presumably 1 sigma?)
+  bearingAccuracy @11 :Float32;
+
+  # Represents velocity accuracy in m/s. (presumably 1 sigma?)
+  speedAccuracy @12 :Float32;
 
   enum SensorSource {
     android @0;
@@ -198,14 +214,15 @@ struct GpsLocationData {
     velodyne @3;  # Velodyne IMU
     fusion @4;
     external @5;
+    ublox @6;
   }
 }
 
 struct CanData {
   address @0 :UInt32;
   busTime @1 :UInt16;
   dat     @2 :Data;
-  src     @3 :Int8;
+  src     @3 :UInt8;
 }
 
 struct ThermalData {
@@ -243,7 +260,8 @@ struct LiveUI {
 struct Live20Data {
   canMonoTimes @10 :List(UInt64);
   mdMonoTime @6 :UInt64;
-  ftMonoTime @7 :UInt64;
+  ftMonoTimeDEPRECATED @7 :UInt64;
+  l100MonoTime @11 :UInt64;
 
   # all deprecated
   warpMatrixDEPRECATED @0 :List(Float32);
@@ -296,10 +314,11 @@ struct LiveTracks {
 }
 
 struct Live100Data {
-  canMonoTime @16 :UInt64;
+  canMonoTimeDEPRECATED @16 :UInt64;
   canMonoTimes @21 :List(UInt64);
   l20MonoTimeDEPRECATED @17 :UInt64;
   mdMonoTimeDEPRECATED @18 :UInt64;
+  planMonoTime @28 :UInt64;
 
   vEgo @0 :Float32;
   aEgoDEPRECATED @1 :Float32;
@@ -393,7 +412,8 @@ struct EncodeIndex {
     bigBoxLossless @0;   # rcamera.mkv
     fullHEVC @1;         # fcamera.hevc
     bigBoxHEVC @2;       # bcamera.hevc
-    chffrAndroidH264 @3; # camera
+    chffrAndroidH264 @3; # acamera
+    fullLosslessClip @4; # prcamera.mkv
   }
 }
 
@@ -413,6 +433,9 @@ struct LogRotate {
 }
 
 struct Plan {
+  mdMonoTime @9 :UInt64;
+  l20MonoTime @10 :UInt64;
+
   # lateral, 3rd order polynomial
   lateralValid @0: Bool;
   dPoly @1 :List(Float32);
@@ -424,6 +447,7 @@ struct Plan {
   aTargetMax @5 :Float32;
   jerkFactor @6 :Float32;
   hasLead @7 :Bool;
+  fcw @8 :Bool;
 }
 
 struct LiveLocationData {
@@ -909,6 +933,114 @@ struct ProcLog {
 
 }
 
+struct UbloxGnss {
+  union {
+    measurementReport @0 :MeasurementReport;
+    ephemeris @1 :Ephemeris;
+  }
+
+  struct MeasurementReport {
+    #received time of week in gps time in seconds and gps week
+    rcvTow @0 :Float64;
+    gpsWeek @1 :UInt16;
+    # leap seconds in seconds
+    leapSeconds @2 :UInt16;
+    # receiver status
+    receiverStatus @3 :ReceiverStatus;
+    # num of measurements to follow
+    numMeas @4 :UInt8;
+    measurements @5 :List(Measurement);
+
+    struct ReceiverStatus {
+      # leap seconds have been determined 
+      leapSecValid @0 :Bool;
+      # Clock reset applied
+      clkReset @1 : Bool;
+    }
+
+    struct Measurement {
+      svId @0 :UInt8;
+      trackingStatus @1 :TrackingStatus;
+      # pseudorange in meters
+      pseudorange @2 :Float64;
+      # carrier phase measurement in cycles
+      carrierCycles @3 :Float64;
+      # doppler measurement in Hz
+      doppler @4 :Float32;
+      # GNSS id, 0 is gps
+      gnssId @5 :UInt8;
+      glonassFrequencyIndex @6 :UInt8;
+      # carrier phase locktime counter in ms
+      locktime @7 :UInt16;
+      # Carrier-to-noise density ratio (signal strength) in dBHz
+      cno @8 : UInt8;
+      # pseudorange standard deviation in meters
+      pseudorangeStdev @9 :Float32;
+      # carrier phase standard deviation in cycles
+      carrierPhaseStdev @10 :Float32;
+      # doppler standard deviation in Hz
+      dopplerStdev @11 :Float32;
+
+      struct TrackingStatus {
+        # pseudorange valid
+        pseudorangeValid @0 :Bool;
+        # carrier phase valid
+        carrierPhaseValid @1 :Bool;
+        # half cycle valid
+        halfCycleValid @2 :Bool;
+        # half sycle subtracted from phase
+        halfCycleSubtracted @3 :Bool;
+      }
+    }
+  }
+
+  struct Ephemeris {
+    # This is according to the rinex (2?) format
+    svId @0 :UInt16;
+    year @1 :UInt16;
+    month @2 :UInt16;
+    day @3 :UInt16;
+    hour @4 :UInt16;
+    minute @5 :UInt16;
+    second @6 :Float32;
+    af0 @7 :Float64;
+    af1 @8 :Float64;
+    af2 @9 :Float64;
+
+    iode @10 :Float64;
+    crs @11 :Float64;
+    deltaN @12 :Float64;
+    m0 @13 :Float64;
+
+    cuc @14 :Float64;
+    ecc @15 :Float64;
+    cus @16 :Float64;
+    a @17 :Float64; # note that this is not the root!!
+
+    toe @18 :Float64;
+    cic @19 :Float64;
+    omega0 @20 :Float64;
+    cis @21 :Float64;
+
+    i0 @22 :Float64;
+    crc @23 :Float64;
+    omega @24 :Float64;
+    omegaDot @25 :Float64;
+
+    iDot @26 :Float64;
+    codesL2 @27 :Float64;
+    gpsWeek @28 :Float64;
+    l2 @29 :Float64;
+
+    svAcc @30 :Float64;
+    svHealth @31 :Float64;
+    tgd @32 :Float64;
+    iodc @33 :Float64;
+
+    transmissionTime @34 :Float64;
+    fitInterval @35 :Float64;
+  }
+}
 struct Event {
   # in nanoseconds?
   logMonoTime @0 :UInt64;
@@ -947,5 +1079,6 @@ struct Event {
     qcomGnss @31 :QcomGnss;
     lidarPts @32 :LidarPts;
     procLog @33 :ProcLog;
+    ubloxGnss @34 :UbloxGnss;
   }
 }

common/dbc.py

@@ -1,6 +1,7 @@
 import re
-import bitstring
+import os
 import struct
+import bitstring
 from collections import namedtuple
 
 def int_or_float(s):
@@ -16,6 +17,7 @@ def int_or_float(s):
 
 class dbc(object):
   def __init__(self, fn):
+    self.name, _ = os.path.splitext(os.path.basename(fn))
     with open(fn) as f:
       self.txt = f.read().split("\n")
     self._warned_addresses = set()
@@ -32,11 +34,8 @@ def __init__(self, fn):
     # signals is a list of DBCSignal in order of increasing start_bit.
     self.msgs = {}
 
-    self.bits = []
-    for i in range(0, 64, 8):
-      for j in range(7, -1, -1):
-        self.bits.append(i+j)
-    self.bits_index = dict(zip(self.bits, range(64)))
+    # lookup to bit reverse each byte
+    self.bits_index = [(i & ~0b111) + ((-i-1) & 0b111) for i in xrange(64)]
 
     for l in self.txt:
       l = l.strip()