From a6a65deb436aa0f8d4e554fd6b071824a0467315 Mon Sep 17 00:00:00 2001 From: Nikolaos Ftylitakis Date: Thu, 4 May 2023 19:06:33 +0300 Subject: [PATCH 01/10] before applying any configuration ,process values to ensure proper format --- .../lib/utils/configuration_handler.py | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/insighioNode/lib/utils/configuration_handler.py b/insighioNode/lib/utils/configuration_handler.py index fc79021..c0569f2 100644 --- a/insighioNode/lib/utils/configuration_handler.py +++ b/insighioNode/lib/utils/configuration_handler.py @@ -105,6 +105,16 @@ NoneType = type(None) +def fixValue(val): + if val == "true": + return "True" + elif val == "false": + return "False" + elif val == "": + return "None" + else: + return val + def get_config_values(fillWithUndefinedIfNotExists=True, prepareForInternalUse=False): configKeyValues = dict() @@ -170,13 +180,7 @@ def get_config_values(fillWithUndefinedIfNotExists=True, prepareForInternalUse=F tmpDict = dict() for key in configKeyValues.keys(): newKey = key.replace("_", "-") - newValue = configKeyValues[key] - if newValue == "true": - newValue = "True" - elif newValue == "false": - newValue = "False" - elif newValue == "": - newValue = "None" + newValue = fixValue(configKeyValues[key]) tmpDict[newKey] = newValue return tmpDict else: @@ -247,10 +251,7 @@ def stringParamsToDict(configurationParameters): for keyValueStr in keyValueStrings: keyValue = keyValueStr.split("=") if len(keyValue) == 2: - if keyValue[1] == "true": - keyValue[1] = "True" - elif keyValue[1] == "false": - keyValue[1] = "False" + keyValue[1] = fixValue(keyValue[1]) keyValueDict[keyValue[0]] = keyValue[1] logging.debug("key value added [{}] -> {}".format(keyValue[0], keyValue[1])) else: @@ -265,12 +266,14 @@ def apply_configuration(keyValuePairDictionary): # fix naming of keys to use '-' instead of '_' # unfortunatelly it is technical burden from old implementations, to be fixed in future release for param in keyValuePairDictionary: + val = keyValuePairDictionary[param] + key = param if '_' in param: - new_key = param.replace("_", '-') - print("changing key [{}] to [{}]".format(param, new_key)) - val = keyValuePairDictionary[param] + key = param.replace("_", '-') + print("changing key [{}] to [{}]".format(param, key)) del keyValuePairDictionary[param] - keyValuePairDictionary[new_key] = val + + keyValuePairDictionary[key] = fixValue(val) gc.collect() From b2ce9c69ab654a7908dd53287f63471024149590 Mon Sep 17 00:00:00 2001 From: Nikolaos Ftylitakis Date: Fri, 5 May 2023 09:51:34 +0300 Subject: [PATCH 02/10] do not use argument cfg in get_measurements, use global cfg --- insighioNode/apps/demo_console/scenario_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/insighioNode/apps/demo_console/scenario_utils.py b/insighioNode/apps/demo_console/scenario_utils.py index 5cacc6c..584013f 100644 --- a/insighioNode/apps/demo_console/scenario_utils.py +++ b/insighioNode/apps/demo_console/scenario_utils.py @@ -71,20 +71,20 @@ def watchdog_reset(): # functions -def get_measurements(cfg): +def get_measurements(cfg_dummy=None): measurements = {} try: - if cfg._MEAS_BATTERY_STAT_ENABLE: + if get_config("_MEAS_BATTERY_STAT_ENABLE"): vbatt = read_battery_voltage() set_value_int(measurements, "vbatt", vbatt, SenmlSecondaryUnits.SENML_SEC_UNIT_MILLIVOLT) - if cfg._MEAS_BOARD_STAT_ENABLE: + if get_config("_MEAS_BOARD_STAT_ENABLE"): (mem_alloc, mem_free) = device_info.get_heap_memory() set_value(measurements, "reset_cause", device_info.get_reset_cause()) set_value(measurements, "mem_alloc", mem_alloc, SenmlUnits.SENML_UNIT_BYTE) set_value(measurements, "mem_free", mem_free, SenmlUnits.SENML_UNIT_BYTE) - if cfg._MEAS_TEMP_UNIT_IS_CELSIUS: + if get_config("_MEAS_TEMP_UNIT_IS_CELSIUS"): set_value_float(measurements, "cpu_temp", device_info.get_cpu_temp(), SenmlUnits.SENML_UNIT_DEGREES_CELSIUS) else: set_value_float(measurements, "cpu_temp", device_info.get_cpu_temp(False), SenmlSecondaryUnits.SENML_SEC_UNIT_FAHRENHEIT) @@ -100,7 +100,7 @@ def get_measurements(cfg): # read internal temperature and humidity try: - if cfg._MEAS_BOARD_SENSE_ENABLE: + if get_config("_MEAS_BOARD_SENSE_ENABLE"): if cfg._UC_INTERNAL_TEMP_HUM_SENSOR == cfg._CONST_SENSOR_SI7021: from sensors import si7021 as sens elif cfg._UC_INTERNAL_TEMP_HUM_SENSOR == cfg._UC_INTERNAL_TEMP_HUM_SENSOR: @@ -118,7 +118,7 @@ def get_measurements(cfg): else: #if shield_name == get_config("_CONST_SHIELD_DIG_ANALOG"): default_board_measurements(measurements) - if hasattr(cfg, '_MEAS_KEYVALUE') and cfg._MEAS_KEYVALUE: + if get_config("_MEAS_KEYVALUE"): add_explicit_key_values(measurements) except Exception as e: From 8c7ab63b852509d13eae542a5fb245f0afc830de Mon Sep 17 00:00:00 2001 From: Nikolaos Ftylitakis Date: Fri, 5 May 2023 12:04:26 +0300 Subject: [PATCH 03/10] add missing record for network --- insighioNode/lib/utils/configuration_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/insighioNode/lib/utils/configuration_handler.py b/insighioNode/lib/utils/configuration_handler.py index c0569f2..5f6fc9c 100644 --- a/insighioNode/lib/utils/configuration_handler.py +++ b/insighioNode/lib/utils/configuration_handler.py @@ -89,6 +89,7 @@ "_UC_IO_SCALE_OFFSET": "meas_scale_offset", "_UC_IO_SCALE_SCALE": "meas_scale_scale", "protocol": "protocol", +"network": "network", "_ALWAYS_ON_CONNECTION": "always_on_connection", "_FORCE_ALWAYS_ON_CONNECTION": "force_always_on_connection", "_ALWAYS_ON_PERIOD": "always_on_period", From 5392ffc36723142d9989e72873eb4d62787ce9f7 Mon Sep 17 00:00:00 2001 From: Nikolaos Ftylitakis Date: Fri, 5 May 2023 12:22:32 +0300 Subject: [PATCH 04/10] add custom version stats if found --- insighioNode/apps/demo_console/scenario.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/insighioNode/apps/demo_console/scenario.py b/insighioNode/apps/demo_console/scenario.py index ce4f819..8ba819d 100644 --- a/insighioNode/apps/demo_console/scenario.py +++ b/insighioNode/apps/demo_console/scenario.py @@ -264,13 +264,18 @@ def executeConnectAndUpload(cfg, measurements, is_first_run, always_on): logging.exception(e, "Exception during disconenction:") return message_sent +def get_var_from_module(module, key): + return getattr(module, key) if hasattr(module, key) else None + def executeDeviceStatisticsUpload(cfg, network): stats = {} try: import srcInfo - stats["sw_branch"] = srcInfo.branch - stats["sw_commit"] = srcInfo.commit + stats["sw_branch"] = get_var_from_module(srcInfo, "branch") + stats["sw_commit"] = get_var_from_module(srcInfo, "commit") + stats["sw_custom_branch"] = get_var_from_module(srcInfo, "custom_branch") + stats["sw_custom_commit"] = get_var_from_module(srcInfo, "custom_commit") except: logging.error("no srcInfo file found") From 62c747e2665a98d7beb79bf3a042142bae02b4da Mon Sep 17 00:00:00 2001 From: Nikolaos Ftylitakis Date: Fri, 5 May 2023 12:22:47 +0300 Subject: [PATCH 05/10] set timeout to 3 minues for COPS=0 call --- insighioNode/lib/networking/modem/modem_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/insighioNode/lib/networking/modem/modem_base.py b/insighioNode/lib/networking/modem/modem_base.py index 5e046be..23a2593 100644 --- a/insighioNode/lib/networking/modem/modem_base.py +++ b/insighioNode/lib/networking/modem/modem_base.py @@ -95,7 +95,7 @@ def init(self, ip_version, apn, technology): if technology.lower() == "nbiot": self.send_at_cmd("AT+COPS=1,2,20201,9") else: - self.send_at_cmd("AT+COPS=0") + self.send_at_cmd("AT+COPS=0", 180000) # disable command echo self.send_at_cmd('ATE0') From 1d48506216a619e457589942c87ade7aaf0b8035 Mon Sep 17 00:00:00 2001 From: Nikolaos Ftylitakis Date: Wed, 10 May 2023 17:49:22 +0300 Subject: [PATCH 06/10] fix storing measurement if failed connection (opt) --- insighioNode/apps/demo_console/scenario.py | 12 ++---------- .../apps/demo_console/scenario_utils.py | 19 +++++++++++++------ insighioNode/www/step-5-measurements.html | 2 +- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/insighioNode/apps/demo_console/scenario.py b/insighioNode/apps/demo_console/scenario.py index 8ba819d..007888b 100644 --- a/insighioNode/apps/demo_console/scenario.py +++ b/insighioNode/apps/demo_console/scenario.py @@ -40,14 +40,6 @@ def execute(): executeDeviceDeinitialization() executeTimingConfiguration() -def storeMeasurement(measurements, force_store): - if cfg._BATCH_UPLOAD_MESSAGE_BUFFER is None and not force_store: - logging.error("Batch upload not activated, ignoring") - return False - - message_buffer.timestamp_measurements(measurements) - return message_buffer.store_measurement(measurements) - def executeDeviceInitialization(): # The demo_config.py is autogenerated by webui # it is the concatenation of the template configurations in template folder. @@ -121,7 +113,7 @@ def executeMeasureAndUploadLoop(): # if the RTC is OK, timestamp message if buffered_upload_enabled and not execute_connection_procedure: - measurementStored = storeMeasurement(measurements, False) + measurementStored = scenario_utils.storeMeasurement(measurements, False) # if always on, do run connect and upload # else run connection only if measurement was not stored execute_connection_procedure = True if always_on else not measurementStored @@ -252,7 +244,7 @@ def executeConnectAndUpload(cfg, measurements, is_first_run, always_on): if not message_sent: logging.info("Message transmission failed, storing for later") if scenario_utils.get_config("_STORE_MEASUREMENT_IF_FAILED_CONNECTION"): - storeMeasurement(measurements, True) + scenario_utils.storeMeasurement(measurements, True) # disconnect from network if not always_on or not is_connected: diff --git a/insighioNode/apps/demo_console/scenario_utils.py b/insighioNode/apps/demo_console/scenario_utils.py index 584013f..334f547 100644 --- a/insighioNode/apps/demo_console/scenario_utils.py +++ b/insighioNode/apps/demo_console/scenario_utils.py @@ -84,10 +84,13 @@ def get_measurements(cfg_dummy=None): set_value(measurements, "reset_cause", device_info.get_reset_cause()) set_value(measurements, "mem_alloc", mem_alloc, SenmlUnits.SENML_UNIT_BYTE) set_value(measurements, "mem_free", mem_free, SenmlUnits.SENML_UNIT_BYTE) - if get_config("_MEAS_TEMP_UNIT_IS_CELSIUS"): - set_value_float(measurements, "cpu_temp", device_info.get_cpu_temp(), SenmlUnits.SENML_UNIT_DEGREES_CELSIUS) - else: - set_value_float(measurements, "cpu_temp", device_info.get_cpu_temp(False), SenmlSecondaryUnits.SENML_SEC_UNIT_FAHRENHEIT) + try: + if get_config("_MEAS_TEMP_UNIT_IS_CELSIUS"): + set_value_float(measurements, "cpu_temp", device_info.get_cpu_temp(), SenmlUnits.SENML_UNIT_DEGREES_CELSIUS) + else: + set_value_float(measurements, "cpu_temp", device_info.get_cpu_temp(False), SenmlSecondaryUnits.SENML_SEC_UNIT_FAHRENHEIT) + except: + logging.error("Error getting cpu_temp.") except Exception as e: logging.exception(e, "unable to measure board sensors") @@ -318,9 +321,13 @@ def execute_transformation(measurements, name, raw_value, transformator): logging.exception(e, "transformator name:{}, raw_value:{}, code:{}".format(name, raw_value, transformator)) pass -def storeMeasurement(measurements, force_store): +def storeMeasurement(measurements, force_store=False): + if get_config("_BATCH_UPLOAD_MESSAGE_BUFFER") is None and not force_store: + logging.error("Batch upload not activated, ignoring") + return False + message_buffer.timestamp_measurements(measurements) - return message_buffer.store_measurement(measurements) + return message_buffer.store_measurement(measurements, force_store) def read_accelerometer(): logging.info("starting XL thread") diff --git a/insighioNode/www/step-5-measurements.html b/insighioNode/www/step-5-measurements.html index ce1e326..a400495 100644 --- a/insighioNode/www/step-5-measurements.html +++ b/insighioNode/www/step-5-measurements.html @@ -495,7 +495,7 @@ showElement('input-s1-sensor-a-d-p2-trans-div', adp2 === "analog") showElement('input-s1-sensor-a-d-p3-trans-div', adp3 === "analog") - //setElemValueBool('input-store-meas-if-failed-conn', data.store_meas_if_failed_conn, false) + setElemValueBool('input-store-meas-if-failed-conn', data.store_meas_if_failed_conn, false) fillKeyValuePairsFromDictionary(data.meas_keyvalue) From f9b08805fef0e208155ec6d3468421a7859ac3c2 Mon Sep 17 00:00:00 2001 From: Nikolaos Ftylitakis Date: Wed, 10 May 2023 17:52:14 +0300 Subject: [PATCH 07/10] set COPS timeout to 180s when locking to NBIoT --- insighioNode/lib/networking/modem/modem_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/insighioNode/lib/networking/modem/modem_base.py b/insighioNode/lib/networking/modem/modem_base.py index 23a2593..6ec1562 100644 --- a/insighioNode/lib/networking/modem/modem_base.py +++ b/insighioNode/lib/networking/modem/modem_base.py @@ -93,7 +93,7 @@ def init(self, ip_version, apn, technology): self.set_technology(technology) if technology.lower() == "nbiot": - self.send_at_cmd("AT+COPS=1,2,20201,9") + self.send_at_cmd("AT+COPS=1,2,20201,9", 180000) else: self.send_at_cmd("AT+COPS=0", 180000) From 74c5a1589a851cd72a34bb4eae608c9c80abe8b9 Mon Sep 17 00:00:00 2001 From: Nikolaos Ftylitakis Date: Wed, 10 May 2023 18:42:55 +0300 Subject: [PATCH 08/10] minor optimization in command execution when setting technology --- insighioNode/lib/networking/modem/modem_bg600.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/insighioNode/lib/networking/modem/modem_bg600.py b/insighioNode/lib/networking/modem/modem_bg600.py index 09fa628..a2edca1 100644 --- a/insighioNode/lib/networking/modem/modem_bg600.py +++ b/insighioNode/lib/networking/modem/modem_bg600.py @@ -39,7 +39,8 @@ def set_technology(self, technology): else: self.send_at_cmd('AT+QCFG="nwscanseq",00,0') self.send_at_cmd('AT+QCFG="nwscanmode",0,0') - self.send_at_cmd('AT+CFUN=1,1', 30000, "APP RDY") + self.send_at_cmd('AT+CFUN=1,1', 15000, "APP RDY") + self.send_at_cmd('ATE0') utime.sleep_ms(1000) def prioritizeWWAN(self): From cc480bcc9522c15298c17d67b94294ffdae99e48 Mon Sep 17 00:00:00 2001 From: Nikolaos Ftylitakis Date: Thu, 11 May 2023 14:57:49 +0300 Subject: [PATCH 09/10] fix parsing of RSSI result --- insighioNode/lib/networking/modem/modem_base.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/insighioNode/lib/networking/modem/modem_base.py b/insighioNode/lib/networking/modem/modem_base.py index 6ec1562..bdb5bbd 100644 --- a/insighioNode/lib/networking/modem/modem_base.py +++ b/insighioNode/lib/networking/modem/modem_base.py @@ -220,12 +220,20 @@ def disconnect(self): def get_rssi(self): if self.ppp is None: + regex_rssi = r"\+CSQ:\s*(\d+),\d+" self.rssi = -141 (status, lines) = self.send_at_cmd('AT+CSQ') if status and len(lines) > 0: - rssi_tmp = int(lines[0].split(',')[0].split(' ')[-1]) - if(rssi_tmp >= 0 and rssi_tmp <= 31): - self.rssi = -113 + rssi_tmp * 2 + for line in lines: + match_res = ure.search(regex_rssi, line) + if match_res is not None: + try: + rssi_tmp = int(match_res.group(1)) + if(rssi_tmp >= 0 and rssi_tmp <= 31): + self.rssi = -113 + rssi_tmp * 2 + except: + pass + break return self.rssi def get_extended_signal_quality(self): From 34912d9098f49da0030d5a38cf65a09cd997598e Mon Sep 17 00:00:00 2001 From: Nikolaos Ftylitakis Date: Thu, 11 May 2023 14:59:21 +0300 Subject: [PATCH 10/10] auto-select GPS technologies based on MCC --- insighioNode/lib/networking/modem/modem_bg600.py | 1 + 1 file changed, 1 insertion(+) diff --git a/insighioNode/lib/networking/modem/modem_bg600.py b/insighioNode/lib/networking/modem/modem_bg600.py index a2edca1..8a82325 100644 --- a/insighioNode/lib/networking/modem/modem_bg600.py +++ b/insighioNode/lib/networking/modem/modem_bg600.py @@ -99,6 +99,7 @@ def set_gps_state(self, poweron=True): command = "" if poweron: command = 'AT+QGPS=1' + self.send_at_cmd('AT+QGPSCFG="gnssconfig",5') else: command = 'AT+QGPSEND'