merge master

Dockerfile

@@ -44,11 +44,6 @@ ENV PYTHONPATH=/project
 RUN git config --global --add safe.directory '*'
 
 WORKDIR /project
-RUN git clone https://github.com/commaai/cereal.git /project/cereal && \
-    cd /project/cereal && \
-    git checkout 861144c136c91f70dcbc652c2ffe99f57440ad47 && \
-    rm -rf .git && \
-    scons -j$(nproc) --minimal
 
 COPY SConstruct .
 COPY ./site_scons /project/site_scons

SConstruct

@@ -6,8 +6,6 @@ import numpy as np
 zmq = 'zmq'
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 
-cereal_dir = Dir('.')
-
 python_path = sysconfig.get_paths()['include']
 cpppath = [
   '#',
@@ -56,11 +54,6 @@ env = Environment(
 common = ''
 Export('env', 'zmq', 'arch', 'common')
 
-cereal = [File('#cereal/libcereal.a')]
-messaging = [File('#cereal/libmessaging.a')]
-Export('cereal', 'messaging')
-
-
 envCython = env.Clone()
 envCython["CPPPATH"] += [np.get_include()]
 envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
@@ -80,6 +73,4 @@ envCython["LIBS"] = python_libs
 
 Export('envCython')
 
-
-SConscript(['cereal/SConscript'])
 SConscript(['opendbc/can/SConscript'])

can/SConscript

@@ -1,12 +1,12 @@
-Import('env', 'envCython', 'cereal', 'common', 'arch')
+Import('env', 'envCython', 'common', 'arch')
 
 import os
 
 envDBC = env.Clone()
 dbc_file_path = '-DDBC_FILE_PATH=\'"%s"\'' % (envDBC.Dir("..").abspath)
 envDBC['CXXFLAGS'] += [dbc_file_path]
 src = ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]
-libs = [common, "capnp", "kj", "zmq"]
+libs = [common, "zmq"]
 
 # shared library for openpilot
 LINKFLAGS = envDBC["LINKFLAGS"]

can/common.h

@@ -7,13 +7,6 @@
 #include <unordered_map>
 #include <vector>
 
-#include <capnp/dynamic.h>
-#include <capnp/serialize.h>
-
-#ifndef DYNAMIC_CAPNP
-#include "cereal/gen/cpp/log.capnp.h"
-#endif
-
 #include "opendbc/can/logger.h"
 #include "opendbc/can/common_dbc.h"
 
@@ -35,6 +28,17 @@ unsigned int xor_checksum(uint32_t address, const Signal &sig, const std::vector
 unsigned int hkg_can_fd_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d);
 unsigned int pedal_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d);
 
+struct CanFrame {
+  long src;
+  uint32_t address;
+  std::vector<uint8_t> dat;
+};
+
+struct CanData {
+  uint64_t nanos;
+  std::vector<CanFrame> frames;
+};
+
 class MessageState {
 public:
   std::string name;
@@ -60,11 +64,8 @@ class MessageState {
 class CANParser {
 private:
   const int bus;
-  kj::Array<capnp::word> aligned_buf;
-
   const DBC *dbc = NULL;
   std::unordered_map<uint32_t, MessageState> message_states;
-  kj::ArrayPtr<capnp::word> getAlignedData(const std::string &data);
 
 public:
   bool can_valid = false;
@@ -78,12 +79,11 @@ class CANParser {
   CANParser(int abus, const std::string& dbc_name,
             const std::vector<std::pair<uint32_t, int>> &messages);
   CANParser(int abus, const std::string& dbc_name, bool ignore_checksum, bool ignore_counter);
-  #ifndef DYNAMIC_CAPNP
   MessageState *messageState(uint32_t address) { return &message_states.at(address); }
-  std::set<uint32_t> update_strings(const std::vector<std::string> &data, bool sendcan);
-  void UpdateCans(uint64_t nanos, const capnp::List<cereal::CanData>::Reader& cans, std::set<uint32_t> updated_addresses);
-  #endif
-  void UpdateCans(uint64_t nanos, const capnp::DynamicStruct::Reader& cans);
+  std::set<uint32_t> update(const std::vector<CanData> &can_data, bool sendcan);
+
+protected:
+  void UpdateCans(const CanData &can, std::set<uint32_t> &updated_addresses);
   void UpdateValid(uint64_t nanos);
 };
 

can/common.pxd

@@ -1,7 +1,7 @@
 # distutils: language = c++
 # cython: language_level=3
 
-from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t
+from libc.stdint cimport uint8_t, uint32_t, uint64_t
 from libcpp cimport bool
 from libcpp.map cimport map
 from libcpp.pair cimport pair
@@ -70,12 +70,21 @@ cdef extern from "common.h":
   cdef cppclass MessageState:
     map[string, SignalValue] values
 
+  cdef struct CanFrame:
+    long src
+    uint32_t address
+    vector[uint8_t] dat
+
+  cdef struct CanData:
+    uint64_t nanos
+    vector[CanFrame] frames
+
   cdef cppclass CANParser:
     bool can_valid
     bool bus_timeout
     CANParser(int, string, vector[pair[uint32_t, int]]) except +
     MessageState *messageState(uint32_t address)
-    set[uint32_t] update_strings(vector[string]&, bool) except +
+    set[uint32_t] update(vector[CanData]&, bool) except +
 
   cdef cppclass CANPacker:
    CANPacker(string)

can/parser.cc

@@ -89,7 +89,7 @@ bool MessageState::update_counter_generic(int64_t v, int cnt_size) {
 
 
 CANParser::CANParser(int abus, const std::string& dbc_name, const std::vector<std::pair<uint32_t, int>> &messages)
-  : bus(abus), aligned_buf(kj::heapArray<capnp::word>(1024)) {
+  : bus(abus) {
   dbc = dbc_lookup(dbc_name);
   assert(dbc);
 
@@ -153,61 +153,50 @@ CANParser::CANParser(int abus, const std::string& dbc_name, bool ignore_checksum
   }
 }
 
-#ifndef DYNAMIC_CAPNP
- kj::ArrayPtr<capnp::word> CANParser::getAlignedData(const std::string &data) {
-  const size_t buf_size = (data.length() / sizeof(capnp::word)) + 1;
-  if (aligned_buf.size() < buf_size) {
-    aligned_buf = kj::heapArray<capnp::word>(buf_size);
-  }
-  memcpy(aligned_buf.begin(), data.data(), data.length());
-  return aligned_buf.slice(0, buf_size);
-}
-
-std::set<uint32_t> CANParser::update_strings(const std::vector<std::string> &data, bool sendcan) {
-  // Clear all_values
+std::set<uint32_t> CANParser::update(const std::vector<CanData> &can_data, bool sendcan) {
+    // Clear all_values
   for (auto &state : message_states) {
     for (auto &value : state.second.values) {
       value.second.all_values.clear();
     }
   }
 
   std::set<uint32_t> updated_addresses;
-  for (const auto &d : data) {
-    capnp::FlatArrayMessageReader cmsg(getAlignedData(d));
-    cereal::Event::Reader event = cmsg.getRoot<cereal::Event>();
-
+  uint64_t current_nanos = 0;
+  for (const auto &c : can_data) {
     if (first_nanos == 0) {
-      first_nanos = event.getLogMonoTime();
+      first_nanos = c.nanos;
+    }
+    if (current_nanos == 0) {
+      current_nanos = c.nanos;
     }
-    last_nanos = event.getLogMonoTime();
+    last_nanos = c.nanos;
 
-    auto cans = sendcan ? event.getSendcan() : event.getCan();
-    UpdateCans(last_nanos, cans, updated_addresses);
+    UpdateCans(c, updated_addresses);
     UpdateValid(last_nanos);
   }
   return updated_addresses;
 }
 
-void CANParser::UpdateCans(uint64_t nanos, const capnp::List<cereal::CanData>::Reader& cans, std::set<uint32_t> updated_addresses) {
+void CANParser::UpdateCans(const CanData &can, std::set<uint32_t> &updated_addresses) {
+  //DEBUG("got %zu messages\n", can.frames.size());
+
   bool bus_empty = true;
 
-  // parse the messages
-  for (const auto cmsg : cans) {
-    if (cmsg.getSrc() != bus) {
+  for (const auto &frame : can.frames) {
+    if (frame.src != bus) {
       // DEBUG("skip %d: wrong bus\n", cmsg.getAddress());
       continue;
     }
     bus_empty = false;
 
-    auto state_it = message_states.find(cmsg.getAddress());
+    auto state_it = message_states.find(frame.address);
     if (state_it == message_states.end()) {
       // DEBUG("skip %d: not specified\n", cmsg.getAddress());
       continue;
     }
-
-    auto dat = cmsg.getDat();
-    if (dat.size() > 64) {
-      DEBUG("got message longer than 64 bytes: 0x%X %zu\n", cmsg.getAddress(), dat.size());
+    if (frame.dat.size() > 64) {
+      DEBUG("got message longer than 64 bytes: 0x%X %zu\n", frame.address, frame.dat.size());
       continue;
     }
 
@@ -217,48 +206,20 @@ void CANParser::UpdateCans(uint64_t nanos, const capnp::List<cereal::CanData>::R
     //  continue;
     //}
 
-    // TODO: can remove when we ignore unexpected can msg lengths
-    // make sure the data_size is not less than state_it->second.size
-    size_t data_size = std::max<size_t>(dat.size(), state_it->second.size);
-    std::vector<uint8_t> data(data_size, 0);
-    memcpy(data.data(), dat.begin(), dat.size());
-    if (state_it->second.parse(nanos, data)) {
+    if (state_it->second.parse(can.nanos, frame.dat)) {
       updated_addresses.insert(state_it->first);
     }
   }
 
   // update bus timeout
   if (!bus_empty) {
-    last_nonempty_nanos = nanos;
+    last_nonempty_nanos = can.nanos;
   }
-  bus_timeout = (nanos - last_nonempty_nanos) > bus_timeout_threshold;
-}
-#endif
-
-void CANParser::UpdateCans(uint64_t nanos, const capnp::DynamicStruct::Reader& cmsg) {
-  // assume message struct is `cereal::CanData` and parse
-  assert(cmsg.has("address") && cmsg.has("src") && cmsg.has("dat"));
-
-  if (cmsg.get("src").as<uint8_t>() != bus) {
-    DEBUG("skip %d: wrong bus\n", cmsg.get("address").as<uint32_t>());
-    return;
-  }
-
-  auto state_it = message_states.find(cmsg.get("address").as<uint32_t>());
-  if (state_it == message_states.end()) {
-    DEBUG("skip %d: not specified\n", cmsg.get("address").as<uint32_t>());
-    return;
-  }
-
-  auto dat = cmsg.get("dat").as<capnp::Data>();
-  if (dat.size() > 64) return; // shouldn't ever happen
-  std::vector<uint8_t> data(dat.size(), 0);
-  memcpy(data.data(), dat.begin(), dat.size());
-  state_it->second.parse(nanos, data);
+  bus_timeout = (can.nanos - last_nonempty_nanos) > bus_timeout_threshold;
 }
 
 void CANParser::UpdateValid(uint64_t nanos) {
-  const bool show_missing = (last_nanos - first_nanos) > 8e9;
+  const bool show_missing = (nanos - first_nanos) > 8e9;
 
   bool _valid = true;
   bool _counters_valid = true;

can/parser_pyx.pyx

@@ -8,7 +8,7 @@ from libc.stdint cimport uint32_t
 
 from .common cimport CANParser as cpp_CANParser
 from .common cimport MessageState as cpp_MessageState
-from .common cimport dbc_lookup, DBC
+from .common cimport dbc_lookup, DBC, CanData, CanFrame
 
 import numbers
 from collections import defaultdict
@@ -102,7 +102,32 @@ cdef class CANParser:
       del self.can
 
   def update_strings(self, strings, sendcan=False):
-    return self.can.update_strings(strings, sendcan)
+    # input format:
+    # [nanos, [[address, data, src], ...]]
+    # [[nanos, [[address, data, src], ...], ...]]
+
+    cdef CanFrame* frame
+    cdef CanData* can_data
+    cdef vector[CanData] can_data_array
+
+    try:
+      if len(strings) and not isinstance(strings[0], (list, tuple)):
+        strings = [strings]
+
+      can_data_array.reserve(len(strings))
+      for s in strings:
+        can_data = &(can_data_array.emplace_back())
+        can_data.nanos = s[0]
+        can_data.frames.reserve(len(s[1]))
+        for f in s[1]:
+          frame = &(can_data.frames.emplace_back())
+          frame.address = f[0]
+          frame.dat = f[1]
+          frame.src = f[2]
+    except TypeError:
+      raise RuntimeError("invalid parameter")
+
+    return self.can.update(can_data_array, sendcan)
 
   @property
   def can_valid(self):

can/tests/test_checksums.py

@@ -1,6 +1,5 @@
 from opendbc.can.parser import CANParser
 from opendbc.can.packer import CANPacker
-from opendbc.can.tests.test_packer_parser import can_list_to_can_capnp
 
 
 class TestCanChecksums:
@@ -28,8 +27,7 @@ def test_honda_checksum(self):
         packer.make_can_msg("LKAS_HUD", 0, values),
         packer.make_can_msg("LKAS_HUD_A", 0, values),
       ]
-      can_strings = [can_list_to_can_capnp(msgs), ]
-      parser.update_strings(can_strings)
+      parser.update_strings([0, msgs])
 
       assert parser.vl['LKAS_HUD']['CHECKSUM'] == std
       assert parser.vl['LKAS_HUD_A']['CHECKSUM'] == ext