Custom allocator / heap profiler (#350)

* custom allocators layout (WIP) * custom allocator proof-of-concept with Request and Operation * fix Allocator tag handling * keep tag in dynamic memory * use allocators for Configs * revert allocator in Operation interface * use allocators in more Core modules * add memory tag to Request factory * simple heap profiler implementation * add vector with custom allocator type * use allocators in Model and Time * facilitate memory tag concatenation * use allocators for Variables * add type MemJsonDoc and factory method * use custom allocators for Operation classes * find tags for untagged memory blocks * capture maximum heap usage per memory tag * cover remaining allocations and debug * add missing custom alloc statements * fix compilation, remove Debug.h include * fix compilation on Arduino * fix compilation warnings / errors * fix UBSan reported error * add function to reset maximum heap usage * update Memory interface and function names * update changelog * fix building on ESP * fix building on ESP8266 * clean up custom allocator changes
matth-x · Aug 13, 2024 · 94e0918 · 94e0918
1 parent d6840ef
commit 94e0918
Show file tree

Hide file tree

Showing 188 changed files with 1,964 additions and 981 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,7 @@
 - Support for `parentIdTag` ([#344](https://github.com/matth-x/MicroOcpp/pull/344))
 - Input validation for unsigned int Configs ([#344](https://github.com/matth-x/MicroOcpp/pull/344))
 - Support for TransactionMessageAttempts/-RetryInterval ([#345](https://github.com/matth-x/MicroOcpp/pull/345))
+- Heap profiler and custom allocator support ([#350](https://github.com/matth-x/MicroOcpp/pull/350))
 
 ### Removed
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -15,6 +15,7 @@ set(MO_SRC
     src/MicroOcpp/Core/FilesystemAdapter.cpp
     src/MicroOcpp/Core/FilesystemUtils.cpp
     src/MicroOcpp/Core/FtpMbedTLS.cpp
+    src/MicroOcpp/Core/Memory.cpp
     src/MicroOcpp/Core/RequestQueue.cpp
     src/MicroOcpp/Core/Context.cpp
     src/MicroOcpp/Core/Operation.cpp
@@ -169,7 +170,7 @@ if (MO_BUILD_UNIT_MBEDTLS)
 endif()
 
 target_include_directories(mo_unit_tests PUBLIC
-    "./tests/catch2"
+    "./tests"
     "./tests/helpers"
     "./src"
 )
@@ -190,6 +191,10 @@ target_compile_definitions(mo_unit_tests PUBLIC
     MO_ENABLE_CERT_MGMT=1
     MO_ENABLE_CONNECTOR_LOCK=1
     MO_REPORT_NOERROR=1
+    MO_OVERRIDE_ALLOCATION=1
+    MO_ENABLE_HEAP_PROFILER=1
+    MO_HEAP_PROFILER_EXTERNAL_CONTROL=1
+    CATCH_CONFIG_EXTERNAL_INTERFACES
 )
 
 target_compile_options(mo_unit_tests PUBLIC

diff --git a/src/MicroOcpp.cpp b/src/MicroOcpp.cpp
@@ -56,6 +56,12 @@ std::shared_ptr<FilesystemAdapter> filesystem;
 } //end namespace MicroOcpp::Facade
 } //end namespace MicroOcpp
 
+#if MO_ENABLE_HEAP_PROFILER
+#ifndef MO_HEAP_PROFILER_EXTERNAL_CONTROL
+#define MO_HEAP_PROFILER_EXTERNAL_CONTROL 0 //enable if you want to manually reset the heap profiler (e.g. for keeping stats over multiple MO lifecycles)
+#endif
+#endif
+
 using namespace MicroOcpp;
 using namespace MicroOcpp::Facade;
 using namespace MicroOcpp::Ocpp16;
@@ -79,7 +85,7 @@ void mocpp_initialize(const char *backendUrl, const char *chargeBoxId, const cha
     /*
      * parse backendUrl so that it suits the links2004/arduinoWebSockets interface
      */
-    std::string url = backendUrl;
+    auto url = makeString("MicroOcpp.cpp", backendUrl);
 
     //tolower protocol specifier
     for (auto c = url.begin(); *c != ':' && c != url.end(); c++) {
@@ -97,16 +103,16 @@ void mocpp_initialize(const char *backendUrl, const char *chargeBoxId, const cha
     }
 
     //parse host, port
-    std::string host_port_path = url.substr(url.find_first_of("://") + strlen("://"));
-    std::string host_port = host_port_path.substr(0, host_port_path.find_first_of('/'));
-    std::string path = host_port_path.substr(host_port.length());
-    std::string host = host_port.substr(0, host_port.find_first_of(':'));
+    auto host_port_path = url.substr(url.find_first_of("://") + strlen("://"));
+    auto host_port = host_port_path.substr(0, host_port_path.find_first_of('/'));
+    auto path = host_port_path.substr(host_port.length());
+    auto host = host_port.substr(0, host_port.find_first_of(':'));
     if (host.empty()) {
         MO_DBG_ERR("could not parse host: %s", url.c_str());
         return;
     }
     uint16_t port = 0;
-    std::string port_str = host_port.substr(host.length());
+    auto port_str = host_port.substr(host.length());
     if (port_str.empty()) {
         port = isTLS ? 443U : 80U;
     } else {
@@ -287,7 +293,7 @@ void mocpp_initialize(Connection& connection, const char *bootNotificationCreden
         new BootService(*context, filesystem)));
     model.setConnectorsCommon(std::unique_ptr<ConnectorsCommon>(
         new ConnectorsCommon(*context, MO_NUMCONNECTORS, filesystem)));
-    std::vector<std::unique_ptr<Connector>> connectors;
+    auto connectors = makeVector<std::unique_ptr<Connector>>("v16.ConnectorBase.Connector");
     for (unsigned int connectorId = 0; connectorId < MO_NUMCONNECTORS; connectorId++) {
         connectors.emplace_back(new Connector(*context, filesystem, connectorId));
     }
@@ -404,6 +410,10 @@ void mocpp_deinitialize() {
 
     configuration_deinit();
 
+#if !MO_HEAP_PROFILER_EXTERNAL_CONTROL
+    MO_MEM_DEINIT();
+#endif
+
     MO_DBG_DEBUG("deinitialized OCPP\n");
 }
 
@@ -470,8 +480,8 @@ bool endTransaction(const char *idTag, const char *reason, unsigned int connecto
             {
                 // We have a parent ID tag, so we need to check if this new card also has one
                 auto authorize = makeRequest(new Ocpp16::Authorize(context->getModel(), idTag));
-                std::string idTag_capture = idTag;
-                std::string reason_capture = reason ? reason : "";
+                auto idTag_capture = makeString("MicroOcpp.cpp", idTag);
+                auto reason_capture = makeString("MicroOcpp.cpp", reason ? reason : "");
                 authorize->setOnReceiveConfListener([idTag_capture, reason_capture, connectorId, tx] (JsonObject response) {
                     JsonObject idTagInfo = response["idTagInfo"];
 
@@ -1087,7 +1097,7 @@ void setOnSendConf(const char *operationType, OnSendConfListener onSendConf) {
 }
 
 void sendRequest(const char *operationType,
-            std::function<std::unique_ptr<DynamicJsonDocument> ()> fn_createReq,
+            std::function<std::unique_ptr<JsonDoc> ()> fn_createReq,
             std::function<void (JsonObject)> fn_processConf) {
 
     if (!context) {
@@ -1105,7 +1115,7 @@ void sendRequest(const char *operationType,
 
 void setRequestHandler(const char *operationType,
             std::function<void (JsonObject)> fn_processReq,
-            std::function<std::unique_ptr<DynamicJsonDocument> ()> fn_createConf) {
+            std::function<std::unique_ptr<JsonDoc> ()> fn_createConf) {
 
     if (!context) {
         MO_DBG_ERR("OCPP uninitialized"); //need to call mocpp_initialize before
@@ -1116,7 +1126,7 @@ void setRequestHandler(const char *operationType,
         return;
     }
 
-    std::string captureOpType = operationType;
+    auto captureOpType = makeString("MicroOcpp.cpp", operationType);
 
     context->getOperationRegistry().registerOperation(operationType, [captureOpType, fn_processReq, fn_createConf] () {
         return new CustomOperation(captureOpType.c_str(), fn_processReq, fn_createConf);

diff --git a/src/MicroOcpp.h b/src/MicroOcpp.h
@@ -13,6 +13,7 @@
 #include <MicroOcpp/Core/FilesystemAdapter.h>
 #include <MicroOcpp/Core/RequestCallbacks.h>
 #include <MicroOcpp/Core/Connection.h>
+#include <MicroOcpp/Core/Memory.h>
 #include <MicroOcpp/Model/Metering/SampledValue.h>
 #include <MicroOcpp/Model/Transactions/Transaction.h>
 #include <MicroOcpp/Model/ConnectorBase/Notification.h>
@@ -436,11 +437,11 @@ void setOnSendConf(const char *operationType, OnSendConfListener onSendConf);
  * 
  * Use case 1, extend the library by sending additional operations. E.g. DataTransfer:
  * 
- * sendRequest("DataTransfer", [] () -> std::unique_ptr<DynamicJsonDocument> {
+ * sendRequest("DataTransfer", [] () -> std::unique_ptr<MicroOcpp::JsonDoc> {
  *     //will be called to create the request once this operation is being sent out
  *     size_t capacity = JSON_OBJECT_SIZE(3) +
  *                       JSON_OBJECT_SIZE(2); //for calculating the required capacity, see https://arduinojson.org/v6/assistant/
- *     auto res = std::unique_ptr<DynamicJsonDocument>(new DynamicJsonDocument(capacity)); 
+ *     auto res = std::unique_ptr<MicroOcpp::JsonDoc>(new MicroOcpp::JsonDoc(capacity)); 
  *     JsonObject request = *res;
  *     request["vendorId"] = "My company Ltd.";
  *     request["messageId"] = "TargetValues";
@@ -457,10 +458,10 @@ void setOnSendConf(const char *operationType, OnSendConfListener onSendConf);
  * 
  * Use case 2, bypass the business logic of this library for custom behavior. E.g. StartTransaction:
  * 
- * sendRequest("StartTransaction", [] () -> std::unique_ptr<DynamicJsonDocument> {
+ * sendRequest("StartTransaction", [] () -> std::unique_ptr<MicroOcpp::JsonDoc> {
  *     //will be called to create the request once this operation is being sent out
  *     size_t capacity = JSON_OBJECT_SIZE(4); //for calculating the required capacity, see https://arduinojson.org/v6/assistant/
- *     auto res = std::unique_ptr<DynamicJsonDocument>(new DynamicJsonDocument(capacity)); 
+ *     auto res = std::unique_ptr<MicroOcpp::JsonDoc>(new MicroOcpp::JsonDoc(capacity)); 
  *     JsonObject request = res->to<JsonObject>();
  *     request["connectorId"] = 1;
  *     request["idTag"] = "A9C3CE1D7B71EA";
@@ -477,7 +478,7 @@ void setOnSendConf(const char *operationType, OnSendConfListener onSendConf);
  * its own.
  */
 void sendRequest(const char *operationType,
-            std::function<std::unique_ptr<DynamicJsonDocument> ()> fn_createReq,
+            std::function<std::unique_ptr<MicroOcpp::JsonDoc> ()> fn_createReq,
             std::function<void (JsonObject)> fn_processConf);
 
 /*
@@ -495,11 +496,11 @@ void sendRequest(const char *operationType,
  *     const char *messageId = request["messageId"];
  *     int battery_capacity = request["data"]["battery_capacity"];
  *     int battery_soc = request["data"]["battery_soc"];
- * }, [] () -> std::unique_ptr<DynamicJsonDocument> {
+ * }, [] () -> std::unique_ptr<MicroOcpp::JsonDoc> {
  *     //will be called  to create the response once this operation is being sent out
  *     size_t capacity = JSON_OBJECT_SIZE(2) +
  *                       JSON_OBJECT_SIZE(1); //for calculating the required capacity, see https://arduinojson.org/v6/assistant/
- *     auto res = std::unique_ptr<DynamicJsonDocument>(new DynamicJsonDocument(capacity)); 
+ *     auto res = std::unique_ptr<MicroOcpp::JsonDoc>(new MicroOcpp::JsonDoc(capacity)); 
  *     JsonObject response = res->to<JsonObject>();
  *     response["status"] = "Accepted";
  *     response["data"]["max_energy"] = 59;
@@ -508,7 +509,7 @@ void sendRequest(const char *operationType,
  */
 void setRequestHandler(const char *operationType,
             std::function<void (JsonObject)> fn_processReq,
-            std::function<std::unique_ptr<DynamicJsonDocument> ()> fn_createConf);
+            std::function<std::unique_ptr<MicroOcpp::JsonDoc> ()> fn_createConf);
 
 /*
  * Send OCPP operations manually not bypassing the internal business logic

diff --git a/src/MicroOcpp/Core/Configuration.cpp b/src/MicroOcpp/Core/Configuration.cpp
@@ -4,10 +4,10 @@
 
 #include <MicroOcpp/Core/Configuration.h>
 #include <MicroOcpp/Core/ConfigurationContainerFlash.h>
+#include <MicroOcpp/Core/Memory.h>
 #include <MicroOcpp/Debug.h>
 
 #include <string.h>
-#include <vector>
 #include <algorithm>
 #include <ArduinoJson.h>
 
@@ -24,8 +24,8 @@ struct Validator {
 namespace ConfigurationLocal {
 
 std::shared_ptr<FilesystemAdapter> filesystem;
-std::vector<std::shared_ptr<ConfigurationContainer>> configurationContainers;
-std::vector<Validator> validators;
+auto configurationContainers = makeVector<std::shared_ptr<ConfigurationContainer>>("v16.Configuration.Containers");
+auto validators = makeVector<Validator>("v16.Configuration.Validators");
 
 }
 
@@ -50,8 +50,8 @@ void addConfigurationContainer(std::shared_ptr<ConfigurationContainer> container
 }
 
 std::shared_ptr<ConfigurationContainer> getContainer(const char *filename) {
-    std::vector<std::shared_ptr<ConfigurationContainer>>::iterator container = std::find_if(configurationContainers.begin(), configurationContainers.end(),
-        [filename](std::shared_ptr<ConfigurationContainer> &elem) {
+    auto container = std::find_if(configurationContainers.begin(), configurationContainers.end(),
+        [filename](decltype(configurationContainers)::value_type &elem) {
             return !strcmp(elem->getFilename(), filename);
         });
 
@@ -192,8 +192,8 @@ Configuration *getConfigurationPublic(const char *key) {
     return nullptr;
 }
 
-std::vector<ConfigurationContainer*> getConfigurationContainersPublic() {
-    std::vector<ConfigurationContainer*> res;
+Vector<ConfigurationContainer*> getConfigurationContainersPublic() {
+    auto res = makeVector<ConfigurationContainer*>("v16.Configuration.Containers");
 
     for (auto& container : configurationContainers) {
         if (container->isAccessible()) {
@@ -210,8 +210,8 @@ bool configuration_init(std::shared_ptr<FilesystemAdapter> _filesystem) {
 }
 
 void configuration_deinit() {
-    configurationContainers.clear();
-    validators.clear();
+    makeVector<decltype(configurationContainers)::value_type>("v16.Configuration.Containers").swap(configurationContainers); //release allocated memory (see https://cplusplus.com/reference/vector/vector/clear/)
+    makeVector<decltype(validators)::value_type>("v16.Configuration.Validators").swap(validators);
     filesystem.reset();
 }
 

diff --git a/src/MicroOcpp/Core/Configuration.h b/src/MicroOcpp/Core/Configuration.h
@@ -8,9 +8,9 @@
 #include <MicroOcpp/Core/ConfigurationKeyValue.h>
 #include <MicroOcpp/Core/ConfigurationContainer.h>
 #include <MicroOcpp/Core/FilesystemAdapter.h>
+#include <MicroOcpp/Core/Memory.h>
 
 #include <memory>
-#include <vector>
 
 #define CONFIGURATION_FN (MO_FILENAME_PREFIX "ocpp-config.jsn")
 #define CONFIGURATION_VOLATILE "/volatile"
@@ -27,7 +27,7 @@ void registerConfigurationValidator(const char *key, std::function<bool(const ch
 void addConfigurationContainer(std::shared_ptr<ConfigurationContainer> container);
 
 Configuration *getConfigurationPublic(const char *key);
-std::vector<ConfigurationContainer*> getConfigurationContainersPublic();
+Vector<ConfigurationContainer*> getConfigurationContainersPublic();
 
 bool configuration_init(std::shared_ptr<FilesystemAdapter> filesytem);
 void configuration_deinit();

diff --git a/src/MicroOcpp/Core/ConfigurationContainer.cpp b/src/MicroOcpp/Core/ConfigurationContainer.cpp
@@ -12,7 +12,8 @@ ConfigurationContainer::~ConfigurationContainer() {
 
 }
 
-ConfigurationContainerVolatile::ConfigurationContainerVolatile(const char *filename, bool accessible) : ConfigurationContainer(filename, accessible) {
+ConfigurationContainerVolatile::ConfigurationContainerVolatile(const char *filename, bool accessible) :
+        ConfigurationContainer(filename, accessible), MemoryManaged("v16.Configuration.ContainerVoltaile.", filename), configurations(makeVector<std::shared_ptr<Configuration>>(getMemoryTag())) {
 
 }
 
@@ -25,7 +26,7 @@ bool ConfigurationContainerVolatile::save() {
 }
 
 std::shared_ptr<Configuration> ConfigurationContainerVolatile::createConfiguration(TConfig type, const char *key) {
-    std::shared_ptr<Configuration> res = makeConfiguration(type, key);
+    auto res = std::shared_ptr<Configuration>(makeConfiguration(type, key).release(), std::default_delete<Configuration>(), makeAllocator<Configuration>("v16.Configuration.", key));
     if (!res) {
         //allocation failure - OOM
         MO_DBG_ERR("OOM");

diff --git a/src/MicroOcpp/Core/ConfigurationContainer.h b/src/MicroOcpp/Core/ConfigurationContainer.h
@@ -5,10 +5,10 @@
 #ifndef MO_CONFIGURATIONCONTAINER_H
 #define MO_CONFIGURATIONCONTAINER_H
 
-#include <vector>
 #include <memory>
 
 #include <MicroOcpp/Core/ConfigurationKeyValue.h>
+#include <MicroOcpp/Core/Memory.h>
 
 namespace MicroOcpp {
 
@@ -37,9 +37,9 @@ class ConfigurationContainer {
     virtual void loadStaticKey(Configuration& config, const char *key) { } //possible optimization: can replace internal key with passed static key
 };
 
-class ConfigurationContainerVolatile : public ConfigurationContainer {
+class ConfigurationContainerVolatile : public ConfigurationContainer, public MemoryManaged {
 private:
-    std::vector<std::shared_ptr<Configuration>> configurations;
+    Vector<std::shared_ptr<Configuration>> configurations;
 public:
     ConfigurationContainerVolatile(const char *filename, bool accessible);