ENT-14329: cf-monitord stores only in-use measurement slots

cf-check/dump.c

@@ -162,18 +162,20 @@ static void print_struct_lock_data(
 static void print_struct_averages(
     const MDB_val value, const bool strip_strings, const char *tskey_filename)
 {
-    assert(sizeof(Averages) == value.mv_size);
-    if (sizeof(Averages) != value.mv_size)
+    // Records hold only the in-use slots (AveragesUsedSize), so a short record
+    // is normal. Copy into a zeroed struct so unstored slots read as zero.
+    assert(value.mv_size <= sizeof(Averages));
+    if (value.mv_size > sizeof(Averages))
     {
-        // Fall back to simple printing in release builds:
+        // Larger than the struct: fall back to simple printing.
         print_json_string(value.mv_data, value.mv_size, strip_strings);
     }
     else
     {
         // TODO: clean up Averages
         char **obnames = NULL;
-        Averages averages;
-        memcpy(&averages, value.mv_data, sizeof(averages));
+        Averages averages = {0};
+        memcpy(&averages, value.mv_data, value.mv_size);
         const time_t last_seen = averages.last_seen;
 
         obnames = GetObservableNames(tskey_filename);
@@ -288,18 +290,7 @@ static void print_struct_or_string(
 {
     if (structs)
     {
-        if ((StringContains(file, "cf_observations.lmdb")
-             || StringContains(file, "history.lmdb"))
-            && StringEqual(key.mv_data, "version"))
-        {
-            // After the CF_OBSERVABLES migration (dbm_migration_observations.c)
-            // these DBs hold a "version" bookkeeping key whose value is a short
-            // version string, not an Averages struct. Print it as a string;
-            // otherwise it would reach print_struct_averages() and trip its
-            // struct-size assertion.
-            print_json_string(value.mv_data, value.mv_size, strip_strings);
-        }
-        else if (StringContains(file, "cf_lastseen.lmdb")
+        if (StringContains(file, "cf_lastseen.lmdb")
             && StringStartsWith(key.mv_data, "q"))
         {
             print_struct_lastseen_quality(value, strip_strings);

cf-check/observables.c

@@ -5,7 +5,7 @@
 #include <file_lib.h>    // FILE_SEPARATOR
 #include <alloc.h>       // xstrdup()
 #include <db_structs.h>  // observable_strings
-#include <string_lib.h>  // StringEqual
+#include <string_lib.h>  // StringEqual(), StringCopy()
 #include <observables.h>
 
 /**
@@ -100,16 +100,19 @@ char **GetObservableNames(const char *ts_key_path)
             if ((fields != 2) && (fields != 6))
             {
                 Log(LOG_LEVEL_ERR, "Wrong line format in ts_key: %s", line);
+                /* sscanf() may have left name untouched. */
+                StringCopy("spare", name, sizeof(name));
             }
 
             if (StringEqual(name, "spare"))
             {
-                temp[i] = xstrdup(name);
+                /* Numbered, not a shared "spare": these become JSON keys. */
+                snprintf(buf, CF_MAXVARSIZE, "spare[%d]", i);
+                temp[i] = xstrdup(buf);
             }
             else
             {
-                snprintf(buf, CF_MAXVARSIZE, "spare[%d]", i);
-                temp[i] = xstrdup(buf);
+                temp[i] = xstrdup(name);
             }
         }
         fclose(f);

cf-monitord/env_monitor.c

@@ -50,6 +50,7 @@
 #include <probes.h>                      /* MonOtherInit,MonOtherGatherData */
 #include <history.h>                     /* HistoryUpdate */
 #include <monitoring.h>                  /* GetObservable */
+#include <monitoring_read.h>             /* AveragesUsedSize */
 #include <cleanup.h>
 
 
@@ -773,7 +774,7 @@ static void UpdateAverages(EvalContext *ctx, char *timekey, const Averages *cons
 
     Log(LOG_LEVEL_INFO, "Updated averages at '%s'", timekey);
 
-    WriteDB(dbp, timekey, newvals, sizeof(Averages));
+    WriteDB(dbp, timekey, newvals, AveragesUsedSize());
     WriteDB(dbp, "DATABASE_AGE", &AGE, sizeof(double));
 
     CloseDB(dbp);

cf-monitord/history.c

@@ -26,6 +26,7 @@
 #include <history.h>
 
 #include <monitoring.h>                                      /* MakeTimekey */
+#include <monitoring_read.h>                                 /* AveragesUsedSize */
 #include <actuator.h>
 #include <promises.h>
 #include <ornaments.h>
@@ -71,7 +72,7 @@ static void PutRecordForTime(CF_DB *db, time_t time, const Averages *values)
 
     MakeTimekey(time, timekey);
 
-    WriteDB(db, timekey, values, sizeof(Averages));
+    WriteDB(db, timekey, values, AveragesUsedSize());
 }
 
 static void Nova_SaveFilePosition(const char *handle, const char *name, long fileptr)

cf-monitord/monitoring.c

@@ -93,14 +93,14 @@ void NovaNamedEvent(const char *eventname, double value)
 
 static void Nova_DumpSlots(void)
 {
-#define MAX_KEY_FILE_SIZE 16384  /* usually around 4000, cannot grow much */
-
     char filename[CF_BUFSIZE];
     int i;
 
     snprintf(filename, CF_BUFSIZE - 1, "%s%cts_key", GetStateDir(), FILE_SEPARATOR);
 
-    char file_contents_new[MAX_KEY_FILE_SIZE] = {0};
+    /* Line length depends on the name, description and units a measurement
+     * promise gives the slot, so there is no useful bound to size a buffer to. */
+    Writer *contents = StringWriter();
 
     for (i = 0; i < CF_OBSERVABLES; i++)
     {
@@ -120,12 +120,15 @@ static void Nova_DumpSlots(void)
             snprintf(line, sizeof(line), "%d,spare,unused\n", i);
         }
 
-        strlcat(file_contents_new, line, sizeof(file_contents_new));
+        WriterWrite(contents, line);
     }
 
+    char *file_contents_new = StringWriterClose(contents);
+
     bool contents_changed = true;
 
-    Writer *w = FileRead(filename, MAX_KEY_FILE_SIZE, NULL);
+    /* One byte more than we generated: a longer file differs anyway. */
+    Writer *w = FileRead(filename, strlen(file_contents_new) + 1, NULL);
     if (w)
     {
         if(strcmp(StringWriterData(w), file_contents_new) == 0)
@@ -145,6 +148,8 @@ static void Nova_DumpSlots(void)
                 GetErrorStr());
         }
     }
+
+    free(file_contents_new);
 }
 
 void GetObservable(int i, char *name, size_t name_size, char *desc, size_t desc_size)
@@ -165,8 +170,9 @@ void GetObservable(int i, char *name, size_t name_size, char *desc, size_t desc_
         }
         else
         {
-            strncpy(name, OBSERVABLES[i][0], name_size - 1);
-            strncpy(desc, OBSERVABLES[i][1], desc_size - 1);
+            /* OBSERVABLES has no rows at or above ob_spare. */
+            strncpy(name, "spare", name_size - 1);
+            strncpy(desc, "unused", desc_size - 1);
         }
     }
 }

libpromises/Makefile.am

@@ -84,7 +84,6 @@ libpromises_la_SOURCES = \
 	dbm_api.c dbm_api.h dbm_api_types.h dbm_priv.h \
 	dbm_migration.c dbm_migration.h \
 	dbm_migration_lastseen.c \
-	dbm_migration_observations.c \
 	dbm_lmdb.c \
 	dbm_quick.c \
 	dbm_tokyocab.c \