Fixing TOML ArrayTable parsing issues #1758

Author: mikefarah

examples/data1.yaml

@@ -1,8 +1,3 @@
-# 001
----
-abc:  # 001
-- 1   # one
-- 2   # two
-
----
-def   # 002
+[[fruits]]
+[[fruits.varieties]]  # nested array of tables
+name = "red delicious

examples/sample.toml

@@ -1,26 +1,6 @@
+[[fruits]]
 
+[animals]
 
-# This is a TOML document
-
-title = "TOML Example"
-
-[owner]
-name = "Tom Preston-Werner"
-dob = 1979-05-27T07:32:00-08:00
-
-[database]
-enabled = true
-ports = [ 8000, 8001, 8002 ]
-data = [ ["delta", "phi"], [3.14] ]
-temp_targets = { cpu = 79.5, case = 72.0 }
-
-[servers]
-
-[servers.alpha]
-ip = "10.0.0.1"
-role = "frontend"
-
-[servers.beta]
-ip = "10.0.0.2"
-role = "backend"
-
+[[fruits.varieties]]  # nested array of tables
+name = "red delicious"

pkg/yqlib/decoder_toml.go

@@ -324,32 +324,58 @@ func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode
 }
 
 func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {
-	log.Debug("Entering processArrayTable")
+	log.Debug("c")
 	fullPath := dec.getFullPath(currentNode.Child())
 	log.Debug("Fullpath: %v", fullPath)
 
+	c := Context{}
+	c = c.SingleChildContext(dec.rootMap)
+
+	pathToCheck := fullPath
+	if len(fullPath) >= 1 {
+		pathToCheck = fullPath[:len(fullPath)-1]
+	}
+
+	// if fullPath points to an array of maps rather than a map
+	// then it should set this element into the _last_ element of that array.
+	// Because TOML. So we'll inject the last index into the path.
+	readOp := createTraversalTree(pathToCheck, traversePreferences{DontAutoCreate: true}, false)
+
+	resultContext, err := dec.d.GetMatchingNodes(c, readOp)
+	if err != nil {
+		return false, err
+	}
+	if resultContext.MatchingNodes.Len() >= 1 {
+		match := resultContext.MatchingNodes.Front().Value.(*CandidateNode)
+		// path refers to an array, we need to add this to the last element in the array
+		if match.Kind == SequenceNode {
+			fullPath = append(pathToCheck, len(match.Content)-1, fullPath[len(fullPath)-1])
+			log.Debugf("Adding to end of %v array, using path: %v", pathToCheck, fullPath)
+		}
+	}
+
 	// need to use the array append exp to add another entry to
 	// this array: fullpath += [ thing ]
-
 	hasValue := dec.parser.NextExpression()
-	if !hasValue {
-		return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
-	}
 
 	tableNodeValue := &CandidateNode{
 		Kind: MappingNode,
 		Tag:  "!!map",
 	}
-
-	tableValue := dec.parser.Expression()
-	runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
-	log.Debugf("table node err: %w", err)
-	if err != nil && !errors.Is(err, io.EOF) {
-		return false, err
+	runAgainstCurrentExp := false
+	// if the next value is a ArrayTable or Table, then its not part of this declaration (not a key value pair)
+	// so lets leave that expression for the next round of parsing
+	if hasValue && (dec.parser.Expression().Kind == toml.ArrayTable || dec.parser.Expression().Kind == toml.Table) {
+		runAgainstCurrentExp = true
+	} else if hasValue {
+		// otherwise, if there is a value, it must be some key value pairs of the
+		// first object in the array!
+		tableValue := dec.parser.Expression()
+		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
+		if err != nil && !errors.Is(err, io.EOF) {
+			return false, err
+		}
 	}
-	c := Context{}
-
-	c = c.SingleChildContext(dec.rootMap)
 
 	// += function
 	err = dec.arrayAppend(c, fullPath, tableNodeValue)

pkg/yqlib/doc/usage/toml.md

@@ -104,6 +104,27 @@ owner:
       suburb: nice
 ```
 
+## Parse: Array of Array Table
+Given a sample.toml file of:
+```toml
+
+[[fruits]]
+name = "apple"
+[[fruits.varieties]]  # nested array of tables
+name = "red delicious"
+```
+then
+```bash
+yq -oy '.' sample.toml
+```
+will output
+```yaml
+fruits:
+  - name: apple
+    varieties:
+      - name: red delicious
+```
+
 ## Parse: Empty Table
 Given a sample.toml file of:
 ```toml

pkg/yqlib/toml_test.go

@@ -37,6 +37,64 @@ owner:
     age: 36
 `
 
+var doubleArrayTable = `
+[[fruits]]
+name = "apple"
+[[fruits.varieties]]  # nested array of tables
+name = "red delicious"`
+
+var doubleArrayTableExpected = `fruits:
+  - name: apple
+    varieties:
+      - name: red delicious
+`
+
+var doubleArrayTableMultipleEntries = `
+[[fruits]]
+name = "banana"
+[[fruits]]
+name = "apple"
+[[fruits.varieties]]  # nested array of tables
+name = "red delicious"`
+
+var doubleArrayTableMultipleEntriesExpected = `fruits:
+  - name: banana
+  - name: apple
+    varieties:
+      - name: red delicious
+`
+
+var doubleArrayTableNothingAbove = `
+[[fruits.varieties]]  # nested array of tables
+name = "red delicious"`
+
+var doubleArrayTableNothingAboveExpected = `fruits:
+  varieties:
+    - name: red delicious
+`
+
+var doubleArrayTableEmptyAbove = `
+[[fruits]]
+[[fruits.varieties]]  # nested array of tables
+name = "red delicious"`
+
+var doubleArrayTableEmptyAboveExpected = `fruits:
+  - varieties:
+      - name: red delicious
+`
+
+var emptyArrayTableThenTable = `
+[[fruits]]
+[animals]
+[[fruits.varieties]]  # nested array of tables
+name = "red delicious"`
+
+var emptyArrayTableThenTableExpected = `fruits:
+  - varieties:
+      - name: red delicious
+animals: {}
+`
+
 var sampleArrayTable = `
 [owner.contact]
 name = "Tom Preston-Werner"
@@ -249,6 +307,40 @@ var tomlScenarios = []formatScenario{
 		expected:     sampleArrayTableExpected,
 		scenarioType: "decode",
 	},
+	{
+		description:  "Parse: Array of Array Table",
+		input:        doubleArrayTable,
+		expected:     doubleArrayTableExpected,
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "Parse: Array of Array Table; nothing above",
+		input:        doubleArrayTableNothingAbove,
+		expected:     doubleArrayTableNothingAboveExpected,
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "Parse: Array of Array Table; empty above",
+		input:        doubleArrayTableEmptyAbove,
+		expected:     doubleArrayTableEmptyAboveExpected,
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "Parse: Array of Array Table; multiple entries",
+		input:        doubleArrayTableMultipleEntries,
+		expected:     doubleArrayTableMultipleEntriesExpected,
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "Parse: Array of Array Table; then table; then array table",
+		input:        emptyArrayTableThenTable,
+		expected:     emptyArrayTableThenTableExpected,
+		scenarioType: "decode",
+	},
 	{
 		description:  "Parse: Empty Table",
 		input:        emptyTable,