Memfault Firmware SDK 0.42.0 (Build 1778)

CHANGES.md

@@ -1,3 +1,30 @@
+### Changes between Memfault SDK 0.42.0 and 0.41.2 - Mar 22, 2023
+
+#### :chart_with_upwards_trend: Improvements
+
+- Zephyr:
+
+  - Add option to capture full thread stacks for classifying stack overflows and
+    determining stack high watermarks. This feature is enabled by setting
+    `CONFIG_MEMFAULT_COREDUMP_FULL_THREAD_STACKS=y`
+  - Remove usage of the `zephyr.h` header in preparation for Zephyr v3.4.0
+
+- `memfault_gdb.py`:
+  - Add support for exporting data from GCC 12 compiled symbol files
+  - Add arguments to override device serial ID, software type, software version,
+    and hardware revision
+
+#### :boom: Breaking Changes
+
+- Metrics:
+  - Integer type metrics (signed/unsigned) will reset to NULL when not set
+    during a heartbeat interval. This NULL value will be discarded by Memfault
+    when received. The previous behavior was to reset to 0 which makes
+    discarding values difficult as 0 is a valid value for these types. For more
+    info please see the
+    [Metrics](https://docs.memfault.com/docs/mcu/metrics-api#setting-metric-values)
+    docs.
+
 ### Changes between Memfault SDK 0.41.2 and SDK 0.41.1 - Mar 10, 2023
 
 #### :chart_with_upwards_trend: Improvements

VERSION

@@ -1,2 +1,2 @@
-BUILD ID: 1696
-GIT COMMIT: e44b190c9
+BUILD ID: 1778
+GIT COMMIT: 539b8ab42

components/include/memfault/core/compiler_gcc.h

@@ -26,7 +26,11 @@ extern "C" {
 #if defined(__clang__)
 #define MEMFAULT_NO_OPT __attribute__((optnone))
 #else
-#define MEMFAULT_NO_OPT __attribute__((optimize("O0")))
+// GCC 12 changed to no longer enable the var-tracking debug option when
+// compiling without optimizations. Explicitly enable it so we always get a nice
+// variable loading experience in GDB, even when the function prologue hasn't
+// run yet. This is required when using the memfault_gdb.py hook to upload data.
+#define MEMFAULT_NO_OPT __attribute__((optimize("O0", "var-tracking", "var-tracking-assignments")))
 #endif
 
 #define MEMFAULT_ALIGNED(x) __attribute__((aligned(x)))

components/include/memfault/core/math.h

@@ -18,6 +18,7 @@ extern "C" {
 #define MEMFAULT_MAX(a , b)  (((a) > (b)) ? (a) : (b))
 #define MEMFAULT_FLOOR(a, align) (((a) / (align)) * (align))
 #define MEMFAULT_ABS(a) (((a) < 0) ? -(a) : (a))
+#define MEMFAULT_CEIL_DIV(x, y) (((x) + (y)-1) / (y))
 
 #ifdef __cplusplus
 }

components/include/memfault/metrics/metrics.h

@@ -116,7 +116,10 @@ typedef struct MemfaultMetricsBootInfo {
 } sMemfaultMetricBootInfo;
 
 //! Initializes the metric events API.
-//! All heartbeat values will be initialized to 0.
+//! All heartbeat values will be initialized to their reset values.
+//! Integer types will be reset to unset/null.
+//! Timer metrics will be reset to 0.
+//! String metrics will be reset to empty strings.
 //! @param storage_impl The storage location to serialize metrics out to
 //! @param boot_info Info added to metrics to facilitate computing aggregate statistics in
 //!  the Memfault cloud
@@ -126,6 +129,9 @@ int memfault_metrics_boot(const sMemfaultEventStorageImpl *storage_impl,
                           const sMemfaultMetricBootInfo *boot_info);
 
 //! Set the value of a signed integer metric.
+//!
+//! Integer metrics that are unset during a heartbeat interval
+//! are sent as null and dropped when received.
 //! @param key The key of the metric. @see MEMFAULT_METRICS_KEY
 //! @param value The new value to set for the metric
 //! @return 0 on success, else error code
@@ -162,7 +168,7 @@ int memfault_metrics_heartbeat_timer_stop(MemfaultMetricId key);
 //! @param key The key of the metric. @see MEMFAULT_METRICS_KEY
 //! @param inc The amount to increment the metric by
 //! @return 0 on success, else error code
-//! @note The metric must be of type kMemfaultMetricType_Counter
+//! @note The metric must be of type kMemfaultMetricType_Unsigned or kMemfaultMetricType_Signed
 int memfault_metrics_heartbeat_add(MemfaultMetricId key, int32_t amount);
 
 //! For debugging purposes: prints the current heartbeat values using

components/include/memfault/metrics/utils.h

@@ -11,6 +11,8 @@
 //! @note A user of the Memfault SDK should _never_ call any
 //! of these routines directly
 
+#include <stdbool.h>
+
 #include "memfault/metrics/metrics.h"
 
 #ifdef __cplusplus
@@ -26,6 +28,7 @@ union MemfaultMetricValue {
 typedef struct {
   MemfaultMetricId key;
   eMemfaultMetricType type;
+  bool is_set;
   union MemfaultMetricValue val;
 } sMemfaultMetricInfo;
 

components/include/memfault/util/cbor.h

@@ -155,6 +155,11 @@ bool memfault_cbor_encode_uint64_as_double(sMemfaultCborEncoder *encoder, uint64
 //! @return true on success, false otherwise
 bool memfault_cbor_encode_long_signed_integer(sMemfaultCborEncoder *encoder, int64_t value);
 
+//! Encode a CBOR null value
+//!
+//! @param encoder The encoder context to use
+//! @return true on success, false otherwise
+bool memfault_cbor_encode_null(sMemfaultCborEncoder *encoder);
 
 //! NOTE: For internal use only, included in the header so it's easy for a caller to statically
 //! allocate the structure

components/include/memfault/version.h

@@ -19,7 +19,7 @@ typedef struct {
   uint8_t patch;
 } sMfltSdkVersion;
 
-#define MEMFAULT_SDK_VERSION   { .major = 0, .minor = 41, .patch = 2 }
+#define MEMFAULT_SDK_VERSION   { .major = 0, .minor = 42, .patch = 0 }
 
 #ifdef __cplusplus
 }

components/metrics/src/memfault_metrics.c

@@ -38,6 +38,7 @@ MEMFAULT_DISABLE_WARNING("-Wunused-macros")
 #define MEMFAULT_METRICS_TYPE_NO_CHANGE (-4)
 #define MEMFAULT_METRICS_STORAGE_TOO_SMALL (-5)
 #define MEMFAULT_METRICS_TIMER_BOOT_FAILED (-6)
+#define MEMFAULT_METRICS_VALUE_NOT_SET (-7)
 
 typedef struct MemfaultMetricKVPair {
   MemfaultMetricId key;
@@ -90,7 +91,7 @@ MEMFAULT_STATIC_ASSERT(MEMFAULT_ARRAY_SIZE(s_memfault_heartbeat_keys) != 0,
 
 #define MEMFAULT_METRICS_TIMER_VAL_MAX 0x80000000
 typedef struct MemfaultMetricValueMetadata {
-  bool is_running:1;
+  bool is_running : 1;
   // We'll use 32 bits since the rollover time is ~25 days which is much much greater than a
   // reasonable heartbeat interval. This let's us track whether or not the timer is running in the
   // top bit
@@ -100,6 +101,7 @@ typedef struct MemfaultMetricValueMetadata {
 typedef struct MemfaultMetricValueInfo {
   union MemfaultMetricValue *valuep;
   sMemfaultMetricValueMetadata *meta_datap;
+  bool is_set;
 } sMemfaultMetricValueInfo;
 
 
@@ -180,6 +182,21 @@ static union MemfaultMetricValue s_memfault_heartbeat_values[] = {
   #undef MEMFAULT_METRICS_STRING_KEY_DEFINE
 };
 
+// Value Set flag data structures and definitions
+// MEMFAULT_IS_SET_FLAGS_PER_BYTE must be a power of 2
+// MEMFAULT_IS_SET_FLAGS_DIVIDER must be equal to log2(MEMFAULT_IS_SET_FLAGS_PER_BYTE)
+#define MEMFAULT_IS_SET_FLAGS_PER_BYTE 8
+#define MEMFAULT_IS_SET_FLAGS_DIVIDER 3
+
+// Create a byte array to contain an is-set flag for each entry in s_memfault_heartbeat_values
+static uint8_t s_memfault_heatbeat_value_is_set_flags[MEMFAULT_CEIL_DIV(
+  MEMFAULT_ARRAY_SIZE(s_memfault_heartbeat_values), MEMFAULT_IS_SET_FLAGS_PER_BYTE)];
+
+MEMFAULT_STATIC_ASSERT(
+  MEMFAULT_ARRAY_SIZE(s_memfault_heatbeat_value_is_set_flags) >=
+    (MEMFAULT_ARRAY_SIZE(s_memfault_heartbeat_values) / MEMFAULT_IS_SET_FLAGS_PER_BYTE),
+  "Mismatch between s_memfault_heatbeat_value_is_set_flags and s_memfault_heartbeat_values");
+
 // String value lookup table. Const- the pointers do not change at runtime, so
 // this table can be stored in ROM and save a little RAM.
 #define MEMFAULT_METRICS_KEY_DEFINE(key_name, value_type)
@@ -250,6 +267,12 @@ static const int s_metric_timer_metadata_mapping[] = {
   #undef MEMFAULT_METRICS_STRING_KEY_DEFINE
 };
 
+// Helper macros to convert between the various metrics indices
+#define MEMFAULT_METRICS_ID_TO_KEY(id) ((size_t)(id)._impl)
+#define MEMFAULT_METRICS_KEY_TO_KV_INDEX(key) (s_memfault_heartbeat_key_to_valueindex[(key)])
+#define MEMFAULT_METRICS_ID_TO_KV_INDEX(id) \
+  (MEMFAULT_METRICS_KEY_TO_KV_INDEX(MEMFAULT_METRICS_ID_TO_KEY(id)))
+
 static struct {
   const sMemfaultEventStorageImpl *storage_impl;
 } s_memfault_metrics_ctx;
@@ -280,16 +303,36 @@ static sMemfaultMetricValueMetadata *prv_find_timer_metadatap(eMfltMetricsIndex
   return &s_memfault_heartbeat_timer_values_metadata[timer_index];
 }
 
-static eMemfaultMetricType prv_find_value_for_key(MemfaultMetricId key,
+//! Helper function to read/write is_set bits for the provided metric
+//!
+//! @param id Metric ID to select corresponding is_set field
+//! @param write Boolean to control whether to write 1 to is_set
+//! @return Returns the value of metric's is_set field. The updated value is returned if write =
+//! true
+static bool prv_read_write_is_value_set(MemfaultMetricId id, bool write) {
+  // Shift the kv index by MEMFAULT_IS_SET_FLAGS_DIVIDER to select byte within
+  // s_memfault_heartbeat_value_is_set_flags
+  size_t byte_index = MEMFAULT_METRICS_ID_TO_KV_INDEX(id) >> MEMFAULT_IS_SET_FLAGS_DIVIDER;
+  // Modulo the kv index by MEMFAULT_IS_SET_FLAGS_PER_BYTE to get bit of the selected byte
+  size_t bit_index = MEMFAULT_METRICS_ID_TO_KV_INDEX(id) % MEMFAULT_IS_SET_FLAGS_PER_BYTE;
+
+  if (write) {
+    s_memfault_heatbeat_value_is_set_flags[byte_index] |= (1 << bit_index);
+  }
+
+  return (s_memfault_heatbeat_value_is_set_flags[byte_index] >> bit_index) & 0x01;
+}
+
+static eMemfaultMetricType prv_find_value_for_key(MemfaultMetricId id,
                                                   sMemfaultMetricValueInfo *value_info_out) {
-  const size_t idx = (size_t)key._impl;
+  const size_t idx = MEMFAULT_METRICS_ID_TO_KEY(id);
   if (idx >= MEMFAULT_ARRAY_SIZE(s_memfault_heartbeat_keys)) {
     *value_info_out = (sMemfaultMetricValueInfo){0};
     return kMemfaultMetricType_NumTypes;
   }
 
   // get the index for the value matching this key.
-  eMfltMetricKeyToValueIndex key_index = s_memfault_heartbeat_key_to_valueindex[idx];
+  eMfltMetricKeyToValueIndex key_index = MEMFAULT_METRICS_KEY_TO_KV_INDEX(idx);
   // for scalar types, this will be the returned value pointer. non-scalars
   // will be handled in the switch below
   union MemfaultMetricValue *value_ptr = &s_memfault_heartbeat_values[key_index];
@@ -321,6 +364,10 @@ static eMemfaultMetricType prv_find_value_for_key(MemfaultMetricId key,
     .meta_datap = prv_find_timer_metadatap((eMfltMetricsIndex)idx),
   };
 
+  if (key_type == kMemfaultMetricType_Unsigned || key_type == kMemfaultMetricType_Signed) {
+    value_info_out->is_set = prv_read_write_is_value_set(id, false);
+  }
+
   return key_type;
 }
 
@@ -374,6 +421,7 @@ static int prv_find_and_set_value_for_key(
   }
 
   *value_info.valuep = *new_value;
+  prv_read_write_is_value_set(key, true);
   return 0;
 }
 
@@ -516,6 +564,7 @@ static bool prv_tally_and_update_timer_cb(MEMFAULT_UNUSED void *ctx,
 static void prv_reset_metrics(void) {
   // reset all scalar metric values
   memset(s_memfault_heartbeat_values, 0, sizeof(s_memfault_heartbeat_values));
+  memset(s_memfault_heatbeat_value_is_set_flags, 0, sizeof(s_memfault_heatbeat_value_is_set_flags));
 
   // reset all string metric values. -1 to skip the last, stub entry in the
   // table
@@ -525,6 +574,13 @@ static void prv_reset_metrics(void) {
       ((char *)s_memfault_heartbeat_string_values[i].ptr)[0] = 0;
     }
   }
+
+  // Set MemfaultSdkMetric_UnexpectedRebootDidOccur to 0 at the end of each reset.
+  // This must be explicitly set because we use 0 to track number of operational hours
+  // Without setting to 0, this defaults to null and will not be counted in the sum
+  // of total operational hours
+  memfault_metrics_heartbeat_set_unsigned(
+    MEMFAULT_METRICS_KEY(MemfaultSdkMetric_UnexpectedRebootDidOccur), 0);
 }
 
 static void prv_heartbeat_timer_update(void) {
@@ -608,6 +664,11 @@ static int prv_find_key_of_type(MemfaultMetricId key, eMemfaultMetricType expect
   if (type != expected_type) {
     return MEMFAULT_METRICS_TYPE_INCOMPATIBLE;
   }
+  if ((type == kMemfaultMetricType_Signed || type == kMemfaultMetricType_Unsigned) &&
+      !(value_info.is_set)) {
+    return MEMFAULT_METRICS_VALUE_NOT_SET;
+  }
+
   *value_out = value_info.valuep;
   return 0;
 }
@@ -707,7 +768,8 @@ static bool prv_metrics_heartbeat_iterate_cb(void *ctx,
   sMemfaultMetricInfo info = {
     .key = key_info->key,
     .type = key_info->type,
-    .val = *value_info->valuep
+    .val = *value_info->valuep,
+    .is_set = value_info->is_set,
   };
   return ctx_info->user_cb(ctx_info->user_ctx, &info);
 }
@@ -748,12 +810,22 @@ static bool prv_heartbeat_debug_print(MEMFAULT_UNUSED void *ctx,
   const char *key_name = s_idx_to_metric_name[key->_impl];
 
   switch (metric_info->type) {
-    case kMemfaultMetricType_Unsigned:
     case kMemfaultMetricType_Timer:
       MEMFAULT_LOG_DEBUG("  %s: %" PRIu32, key_name, value->u32);
       break;
+    case kMemfaultMetricType_Unsigned:
+      if (metric_info->is_set) {
+        MEMFAULT_LOG_DEBUG("  %s: %" PRIu32, key_name, value->u32);
+      } else {
+        MEMFAULT_LOG_DEBUG("  %s: null", key_name);
+      }
+      break;
     case kMemfaultMetricType_Signed:
-      MEMFAULT_LOG_DEBUG("  %s: %" PRIi32, key_name, value->i32);
+      if (metric_info->is_set) {
+        MEMFAULT_LOG_DEBUG("  %s: %" PRIi32, key_name, value->i32);
+      } else {
+        MEMFAULT_LOG_DEBUG("  %s: null", key_name);
+      }
       break;
     case kMemfaultMetricType_String:
       MEMFAULT_LOG_DEBUG("  %s: \"%s\"", key_name, (const char *)value->ptr);

components/metrics/src/memfault_metrics_serializer.c

@@ -30,21 +30,41 @@ typedef struct {
   bool encode_success;
 } sMemfaultSerializerState;
 
+static bool prv_metric_heartbeat_write_integer(sMemfaultSerializerState *state,
+                                               sMemfaultCborEncoder *encoder,
+                                               const sMemfaultMetricInfo *metric_info) {
+  if (!metric_info->is_set && !state->compute_worst_case_size) {
+    return memfault_cbor_encode_null(encoder);
+  }
+
+  if (metric_info->type == kMemfaultMetricType_Unsigned) {
+    const uint32_t value = state->compute_worst_case_size ? UINT32_MAX : metric_info->val.u32;
+    return memfault_cbor_encode_unsigned_integer(encoder, value);
+  }
+
+  if (metric_info->type == kMemfaultMetricType_Signed) {
+    const int32_t value = state->compute_worst_case_size ? INT32_MIN : metric_info->val.i32;
+    return memfault_cbor_encode_signed_integer(encoder, value);
+  }
+
+  // Should be unreachable
+  return false;
+}
+
 static bool prv_metric_heartbeat_writer(void *ctx, const sMemfaultMetricInfo *metric_info) {
   sMemfaultSerializerState *state = (sMemfaultSerializerState *)ctx;
   sMemfaultCborEncoder *encoder = &state->encoder;
 
   // encode the value
   switch (metric_info->type) {
-    case kMemfaultMetricType_Timer:
-    case kMemfaultMetricType_Unsigned: {
+    case kMemfaultMetricType_Timer: {
       const uint32_t value = state->compute_worst_case_size ? UINT32_MAX : metric_info->val.u32;
       state->encode_success = memfault_cbor_encode_unsigned_integer(encoder, value);
       break;
     }
+    case kMemfaultMetricType_Unsigned:
     case kMemfaultMetricType_Signed: {
-      const int32_t value = state->compute_worst_case_size ? INT32_MIN : metric_info->val.i32;
-      state->encode_success = memfault_cbor_encode_signed_integer(encoder, value);
+      state->encode_success = prv_metric_heartbeat_write_integer(state, encoder, metric_info);
       break;
     }
     case kMemfaultMetricType_String: {

components/util/src/memfault_minimal_cbor.c

@@ -22,10 +22,23 @@ typedef enum CborMajorType {
   kCborMajorType_SimpleType = 7,
 } eCborMajorType;
 
+// Definitions for Additional Info for Simple Values can be found here:
+// https://www.rfc-editor.org/rfc/rfc7049#section-2.3
+typedef enum CborAddInfoSimpleVals {
+  kCborAddInfoSimpleVals_False = 20,
+  kCborAddInfoSimpleVals_True = 21,
+  kCborAddInfoSimpleVals_Null = 22,
+  kCborAddInfoSimpleVals_Undefined = 23,
+} eCborAddInfoSimpleVals;
+
 // A CBOR payload is composed of a stream of "data items"
 // The main type of each "data item" is known as the CBOR Major Type
 // and populates the upper 3 bits of the first byte of each "data item"
-#define CBOR_SERIALIZE_MAJOR_TYPE(mt) ((uint8_t) ((mt & 0x7) << 5))
+#define CBOR_SERIALIZE_MAJOR_TYPE(mt) ((uint8_t)((mt & 0x7) << 5))
+
+// CBOR NULL is formed by using a major type of SimpleValue and the assigned value of null
+#define CBOR_NULL \
+  (CBOR_SERIALIZE_MAJOR_TYPE(kCborMajorType_SimpleType) | kCborAddInfoSimpleVals_Null)
 
 void memfault_cbor_encoder_init(sMemfaultCborEncoder *encoder, MemfaultCborWriteCallback write_cb,
                                 void *write_cb_ctx, size_t buf_len) {
@@ -184,3 +197,8 @@ bool memfault_cbor_encode_array_begin(
     sMemfaultCborEncoder *encoder, size_t num_elements) {
   return prv_encode_unsigned_integer(encoder, kCborMajorType_Array, num_elements);
 }
+
+bool memfault_cbor_encode_null(sMemfaultCborEncoder *encoder) {
+  uint8_t tmp_buf[] = {CBOR_NULL};
+  return prv_add_to_result_buffer(encoder, tmp_buf, sizeof(tmp_buf));
+}