ENT-8193: Added validfiledata policy function

libenv/sysinfo.c

@@ -1188,7 +1188,7 @@ static void OSReleaseParse(EvalContext *ctx, const char *file_path)
 {
     JsonElement *os_release_json = JsonReadDataFile("system info discovery",
                                                     file_path, DATAFILETYPE_ENV,
-                                                    100 * 1024);
+                                                    100 * 1024, true);
     if (os_release_json != NULL)
     {
         char *tags;

libpromises/evalfunction.c

@@ -7800,7 +7800,7 @@ static FnCallResult ReadDataGeneric(const char *const fname,
     assert(fname != NULL);
     assert(input_path != NULL);
 
-    JsonElement *json = JsonReadDataFile(fname, input_path, requested_mode, size_max);
+    JsonElement *json = JsonReadDataFile(fname, input_path, requested_mode, size_max, true);
     if (json == NULL)
     {
         return FnFailure();
@@ -7809,6 +7809,23 @@ static FnCallResult ReadDataGeneric(const char *const fname,
     return FnReturnContainerNoCopy(json);
 }
 
+static DataFileType ParseRequestedMode(const char *fname, const char *input_path, const char *const mode_string)
+{
+    DataFileType requested_mode = DATAFILETYPE_UNKNOWN;
+    if (StringEqual("auto", mode_string))
+    {
+        requested_mode = GetDataFileTypeFromSuffix(input_path);
+        Log(LOG_LEVEL_VERBOSE,
+            "%s: automatically selected data type %s from filename %s",
+            fname, DataFileTypeToString(requested_mode), input_path);
+    }
+    else
+    {
+        requested_mode = GetDataFileTypeFromString(mode_string);
+    }
+    return requested_mode;
+}
+
 static FnCallResult FnCallReadData(ARG_UNUSED EvalContext *ctx,
                                    ARG_UNUSED const Policy *policy,
                                    const FnCall *fp,
@@ -7823,20 +7840,32 @@ static FnCallResult FnCallReadData(ARG_UNUSED EvalContext *ctx,
 
     const char *input_path = RlistScalarValue(args);
     const char *const mode_string = RlistScalarValue(args->next);
-    DataFileType requested_mode = DATAFILETYPE_UNKNOWN;
-    if (StringEqual("auto", mode_string))
-    {
-        requested_mode = GetDataFileTypeFromSuffix(input_path);
-        Log(LOG_LEVEL_VERBOSE,
-            "%s: automatically selected data type %s from filename %s",
-            fp->name, DataFileTypeToString(requested_mode), input_path);
-    }
-    else
+    DataFileType requested_mode = ParseRequestedMode(fp->name, input_path, mode_string);
+
+    return ReadDataGeneric(fp->name, input_path, CF_INFINITY, requested_mode);
+}
+
+static FnCallResult FnCallValidFileData(ARG_UNUSED EvalContext *ctx,
+                                   ARG_UNUSED const Policy *policy,
+                                   const FnCall *fp,
+                                   const Rlist *args)
+{
+    assert(fp != NULL);
+    if (args == NULL)
     {
-        requested_mode = GetDataFileTypeFromString(mode_string);
+        Log(LOG_LEVEL_ERR, "Function '%s' requires at least one argument", fp->name);
+        return FnFailure();
     }
 
-    return ReadDataGeneric(fp->name, input_path, CF_INFINITY, requested_mode);
+    const char *input_path = RlistScalarValue(args);
+    const char *const mode_string = RlistScalarValue(args->next);
+    DataFileType requested_mode = ParseRequestedMode(fp->name, input_path, mode_string);
+
+    JsonElement *json = JsonReadDataFile(fp->name, input_path, requested_mode, CF_INFINITY, false);
+    bool is_valid = (json != NULL);
+    JsonDestroy(json);
+
+    return FnReturnContext(is_valid);
 }
 
 static FnCallResult ReadGenericDataType(const FnCall *fp,
@@ -10319,7 +10348,7 @@ void ModuleProtocol(EvalContext *ctx, const char *command, const char *line, int
                 Log(LOG_LEVEL_DEBUG, "Module protocol parsing %s file '%s'",
                     DataFileTypeToString(requested_mode), content);
 
-                JsonElement *json = JsonReadDataFile("module file protocol", content, requested_mode, size_max);
+                JsonElement *json = JsonReadDataFile("module file protocol", content, requested_mode, size_max, true);
                 if (json != NULL)
                 {
                     Buffer *tagbuf = StringSetToBuffer(tags, ',');
@@ -11899,6 +11928,8 @@ const FnCallType CF_FNCALL_TYPES[] =
                   FNCALL_OPTION_NONE, FNCALL_CATEGORY_IO, SYNTAX_STATUS_NORMAL, DEFAULT_ARGC),
     FnCallTypeNew("readtcp", CF_DATA_TYPE_STRING, READTCP_ARGS, &FnCallReadTcp, "Connect to tcp port, send string and assign result to variable",
                   FNCALL_OPTION_CACHED, FNCALL_CATEGORY_COMM, SYNTAX_STATUS_NORMAL, DEFAULT_ARGC),
+    FnCallTypeNew("validfiledata", CF_DATA_TYPE_CONTEXT, READDATA_ARGS, &FnCallValidFileData, "Validate a YAML, JSON, CSV, etc.",
+                  FNCALL_OPTION_NONE, FNCALL_CATEGORY_IO, SYNTAX_STATUS_NORMAL, DEFAULT_ARGC),
 
     // reg functions for regex
     FnCallTypeNew("regarray", CF_DATA_TYPE_CONTEXT, REGARRAY_ARGS, &FnCallRegList, "True if the regular expression in arg1 matches any item in the list or array or data container arg2",

tests/acceptance/01_vars/02_functions/validfiledata.cf

@@ -0,0 +1,94 @@
+body common control
+{
+  inputs => { "../../default.sub.cf" };
+  bundlesequence => { init, test, check };
+  version => "1.0";
+}
+
+bundle agent generate_files(template, data)
+{
+  vars:
+    "content" string => string_mustache("$(template[mustache])", @(data));
+
+  files:
+    "/tmp/config.$(template[format])"
+      create => "true",
+      content => "$(content)";
+
+    "/tmp/config.x.$(template[format])"
+      create => "true",
+      content => "-$(content)", # Add some text that makes the json and yaml invalid
+      if => strcmp("$(template[create_invalid])", "yes"); # csv and env are invalid ONLY when the file doesn't exist
+}
+
+bundle agent init
+{
+  vars:
+    "config_data"
+      data => parsejson(
+        '{"hello" : "world", "bye" : "everyone", "pizza" : "hamburger"}'
+      );
+@if feature(yaml)
+    "templates"
+      data => parsejson(
+        '[
+          {"format": "json", "mustache": "{{%-top-}}", "create_invalid" : "yes"},
+          {"format": "yaml", "mustache": "---\\n{{#-top-}}\\n{{@}}: \\\"{{.}}\\\"\\n{{/-top-}}", "create_invalid" : "yes"},
+          {"format": "csv", "mustache": "Key,Value\\r\\n{{#-top-}}\\r\\n{{@}},\\\"{{.}}\\\"\\r\\n{{/-top-}}", "create_invalid" : "no"},
+          {"format": "env", "mustache": "{{#-top-}}\\n{{{@}}}={{{.}}}\\n{{/-top-}}", "create_invalid" : "no"}
+        ]'
+      );
+@else
+    "templates"
+      data => parsejson(
+        '[
+          {"format": "json", "mustache": "{{%-top-}}", "create_invalid" : "yes"},
+          {"format": "csv", "mustache": "Key,Value\\r\\n{{#-top-}}\\r\\n{{@}},\\\"{{.}}\\\"\\r\\n{{/-top-}}", "create_invalid" : "no"},
+          {"format": "env", "mustache": "{{#-top-}}\\n{{{@}}}={{{.}}}\\n{{/-top-}}", "create_invalid" : "no"}
+        ]'
+      );
+@endif
+    "template_idx" slist => getindices(templates);
+
+  methods:
+    "generate"
+      usebundle => generate_files(
+        "@(templates[$(template_idx)])", @(config_data)
+      );
+}
+
+bundle agent test
+{
+  vars:
+    "formats"
+      slist => maparray("$(init.templates[$(this.k)][format])", init.templates);
+
+    "valid_classes" slist => maplist("valid_$(this)", formats);
+    "invalid_classes" slist => maplist("invalid_$(this)", formats);
+
+  classes:
+    "valid_$(formats)"
+      expression => validfiledata("/tmp/config.$(formats)", "auto"),
+      scope => "namespace";
+
+    "invalid_$(formats)"
+      expression => validfiledata("/tmp/config.x.$(formats)", "auto"),
+      scope => "namespace";
+}
+
+bundle agent check
+{
+  classes:
+    "valid" and => { @(test.valid_classes) };
+    "invalid" or => { @(test.invalid_classes) };
+
+  files:
+    "/tmp/config.x.$(test.formats)" delete => tidy;
+
+  reports:
+    valid.!invalid::
+      "$(this.promise_filename) Pass";
+
+    !valid|invalid::
+      "$(this.promise_filename) FAIL";
+}