From 474af40d2406dd7e116b17423762257706bef342 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Thu, 18 Dec 2025 10:58:14 +0100 Subject: [PATCH 1/4] Simple crash_tests for yaml_edit A set of YAML files that we will load and then use yaml_edit to perform arbitrary edits, attempting to trigger internal errors. This works because `YamlEditor._performEdit` will perform the semantic mutation on the existing YamlNode tree, and then modify the YAML string re-parse it and compare it to the semantically expected YamlNode tree. This ensures that `YamlEditor` always produces valid YAML, and that the YAML has the expected semantics. The internal check in `YamlEditor` does not ensure that comments are preserved or that formatting is reasonable. But it does ensure that semantics are as expected, and that the YAML produced is valid. These tests simply exercises these mechancs. I've already added quite a few files that we need to immediately skip because we cannot perform certain mutations on them correctly. Is is particularly likely when comments are inserted in weird locations. How many issues we actually have is hard to quantify, it's quite likely that many of the bugs are the same. --- .../yaml_edit/test/crash_test/crash_test.dart | 190 ++++++++++++++++++ .../crash_test/testdata/block_strings.yaml | 41 ++++ .../test/crash_test/testdata/complex.yaml | 93 +++++++++ .../test/crash_test/testdata/json.yaml | 12 ++ .../crash_test/testdata/json_comments.yaml | 13 ++ .../crash_test/testdata/mangled_json.yaml | 15 ++ .../test/crash_test/testdata/simple.yaml | 8 + .../crash_test/testdata/simple_comments.yaml | 9 + 8 files changed, 381 insertions(+) create mode 100644 pkgs/yaml_edit/test/crash_test/crash_test.dart create mode 100644 pkgs/yaml_edit/test/crash_test/testdata/block_strings.yaml create mode 100644 pkgs/yaml_edit/test/crash_test/testdata/complex.yaml create mode 100644 pkgs/yaml_edit/test/crash_test/testdata/json.yaml create mode 100644 pkgs/yaml_edit/test/crash_test/testdata/json_comments.yaml create mode 100644 pkgs/yaml_edit/test/crash_test/testdata/mangled_json.yaml create mode 100644 pkgs/yaml_edit/test/crash_test/testdata/simple.yaml create mode 100644 pkgs/yaml_edit/test/crash_test/testdata/simple_comments.yaml diff --git a/pkgs/yaml_edit/test/crash_test/crash_test.dart b/pkgs/yaml_edit/test/crash_test/crash_test.dart new file mode 100644 index 000000000..a9e1da27a --- /dev/null +++ b/pkgs/yaml_edit/test/crash_test/crash_test.dart @@ -0,0 +1,190 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +@TestOn('vm') +library; + +import 'dart:io'; +import 'dart:isolate'; + +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; +import 'package:yaml_edit/yaml_edit.dart'; + +final _scalarStyles = [ + ScalarStyle.ANY, + ScalarStyle.PLAIN, + ScalarStyle.LITERAL, + ScalarStyle.FOLDED, + ScalarStyle.SINGLE_QUOTED, + ScalarStyle.DOUBLE_QUOTED, +]; + +/// Files with tests that are broken, so we have to skip them +final _skippedFiles = [ + 'mangled_json.yaml', + 'block_strings.yaml', + 'complex.yaml', + 'simple_comments.yaml', +]; + +/// The crash tests will attempt to enumerate all JSON paths in each input +/// document and then proceed to make arbitrary mutations trying to see if +/// [YamlEditor] will crash. Arbitrary mutations include: +/// - Remove each JSON path +/// - Prepend and append to each list and map. +/// - Set each string to 'hello world' +/// - Set all numbers to 42 +/// +/// Input documents are loaded from: test/crash_test/testdata/*.yaml +Future main() async { + final packageUri = await Isolate.resolvePackageUri( + Uri.parse('package:yaml_edit/yaml_edit.dart')); + + final testdataUri = packageUri!.resolve('../test/crash_test/testdata/'); + final testFiles = Directory.fromUri(testdataUri) + .listSync() + .whereType() + .where((f) => f.path.endsWith('.yaml')) + .toList(); + + for (final f in testFiles) { + final fileName = f.uri.pathSegments.last; + final input = f.readAsStringSync(); + final root = YamlEditor(input); + + test('$fileName is valid YAML', () { + loadYamlNode(input); + }); + + if (_skippedFiles.contains(fileName)) { + test( + 'crash_test.dart for $fileName', + () {}, + skip: 'Known failures in "$fileName"', + ); + continue; + } + + for (final (path, node) in _allJsonPaths(root.parseAt([]))) { + _testJsonPath(fileName, input, path, node); + } + } +} + +void _testJsonPath( + String fileName, + String input, + Iterable path, + YamlNode node, +) { + final editorName = 'YamlEditor($fileName)'; + + // Try to remove the node + test('$editorName.remove($path)', () { + final editor = YamlEditor(input); + editor.remove(path); + }); + + // Try to update path to a string + test('$editorName.update($path, \'updated string\')', () { + final editor = YamlEditor(input); + editor.update(path, 'updated string'); + }); + + // Try to update path to an integer + test('$editorName.update($path, 42)', () { + final editor = YamlEditor(input); + editor.update(path, 42); + }); + + // Try to set a multi-line string for each style + for (final style in _scalarStyles) { + test('$editorName.update($path, \'foo\\nbar\') as $style', () { + final editor = YamlEditor(input); + editor.update(path, YamlScalar.wrap('foo\nbar', style: style)); + }); + } + + // If it's a list, we try to insert into the list for each index + if (node is YamlList) { + for (var i = 0; i < node.length + 1; i++) { + test('$editorName.insertIntoList($path, $i, 42)', () { + final editor = YamlEditor(input); + editor.insertIntoList(path, i, 42); + }); + + test('$editorName.insertIntoList($path, $i, \'new string\')', () { + final editor = YamlEditor(input); + editor.insertIntoList(path, i, 'new string'); + }); + + for (final style in _scalarStyles) { + test('$editorName.insertIntoList($path, $i, \'foo\\nbar\') as $style', + () { + try { + final editor = YamlEditor(input); + editor.insertIntoList( + path, + i, + YamlScalar.wrap( + 'foo\nbar', + style: style, + )); + } catch (e) { + print(e.runtimeType); + rethrow; + } + }); + } + } + } + + // If it's a map, we try to insert a new key (if the new-key name isn't used) + if (node is YamlMap && !node.containsKey('new-key')) { + final newPath = [...path, 'new-key']; + + test('$editorName.update($newPath, 42)', () { + final editor = YamlEditor(input); + editor.update(newPath, 42); + }); + + test('$editorName.update($newPath, \'new string\')', () { + final editor = YamlEditor(input); + editor.update(newPath, 'new string'); + }); + + for (final style in _scalarStyles) { + test('$editorName.update($newPath, \'foo\\nbar\') as $style', () { + final editor = YamlEditor(input); + editor.update( + newPath, + YamlScalar.wrap( + 'foo\nbar', + style: style, + )); + }); + } + } +} + +Iterable<(Iterable, YamlNode)> _allJsonPaths( + YamlNode node, [ + Iterable parents = const [], +]) sync* { + yield (parents, node); + + if (node is YamlMap) { + for (final entry in node.nodes.entries) { + final key = entry.key as YamlNode; + final value = entry.value; + yield* _allJsonPaths(value, [...parents, key.value]); + } + } else if (node is YamlList) { + for (var i = 0; i < node.nodes.length; i++) { + final value = node.nodes[i]; + yield* _allJsonPaths(value, [...parents, i]); + } + } +} diff --git a/pkgs/yaml_edit/test/crash_test/testdata/block_strings.yaml b/pkgs/yaml_edit/test/crash_test/testdata/block_strings.yaml new file mode 100644 index 000000000..a73dee8f7 --- /dev/null +++ b/pkgs/yaml_edit/test/crash_test/testdata/block_strings.yaml @@ -0,0 +1,41 @@ +block-strings: # annoying comment (maybe) + - folded: + - clip: > + first line + + skipped line + + +# This is a comment + # this too! + # And this one + - strip: >+ # We can have comments here + first line + + skipped line + + + - keep: >- + first line + + skipped line + # comment why not! + # and again + - literal: + - clip: | + first line + + skipped line + + + - strip: |+ + first line + + skipped line + + + - keep: |- + first line + + skipped line + diff --git a/pkgs/yaml_edit/test/crash_test/testdata/complex.yaml b/pkgs/yaml_edit/test/crash_test/testdata/complex.yaml new file mode 100644 index 000000000..bb9a067f6 --- /dev/null +++ b/pkgs/yaml_edit/test/crash_test/testdata/complex.yaml @@ -0,0 +1,93 @@ + # Start comment +? string + # why not here too! +: hello world +# top-level comment + # comment again +? block-strings # annoying comment (maybe) +: - folded: + - clip: > + first line + + skipped line + + +# This is a comment + # this too! + # And this one + - strip: >+ # We can have comments here + first line + + skipped line + + + - keep: >- + first line + + skipped line + # comment why not! + # and again + - literal: + - clip: | + first line + + skipped line + + + - strip: |+ + first line + + skipped line + + + - keep: |- + first line + + skipped line + + +? key +: value # Note: this works + # This too ? +? map +: k1: 1 + k2: 2 + k3: 3 +? list +: - 1 + - 2 + - 3 +? inlineMap +: {k1: 1, k2: 2, k3: 3} +? inlineList +: [1, 2, 3] +? complex-object +: foo: 42 + bar: + - 'test test' + - | + hello world + - "test string" + - { + a: 1, + b: + 'hello world' + } +? json-with-comments +: { + "key": "value", + 'string with newline': 'hello + world' + , 32 : 42 +, list : +[ # We can make multi-line strings inline! + 'foo + + +bar' # comment before comma +, -32.4 + # Comment again. +, # Comment on the comma! +# trailing comma, why not +] +} diff --git a/pkgs/yaml_edit/test/crash_test/testdata/json.yaml b/pkgs/yaml_edit/test/crash_test/testdata/json.yaml new file mode 100644 index 000000000..062da25d1 --- /dev/null +++ b/pkgs/yaml_edit/test/crash_test/testdata/json.yaml @@ -0,0 +1,12 @@ +{ + "string": "hello world", + "map": { + "foo": 42, + "bar": "baz" + }, + "list": [ + 1, + 2, + 3 + ] +} diff --git a/pkgs/yaml_edit/test/crash_test/testdata/json_comments.yaml b/pkgs/yaml_edit/test/crash_test/testdata/json_comments.yaml new file mode 100644 index 000000000..d0e8b1aca --- /dev/null +++ b/pkgs/yaml_edit/test/crash_test/testdata/json_comments.yaml @@ -0,0 +1,13 @@ +# Initial comment +{ + "string": "hello world", # Great message + "map": { + "foo": 42, # Fantastic number + "bar": "baz" + }, + "list": [ + 1, # This is a good list + 2, + 3 + ] +} diff --git a/pkgs/yaml_edit/test/crash_test/testdata/mangled_json.yaml b/pkgs/yaml_edit/test/crash_test/testdata/mangled_json.yaml new file mode 100644 index 000000000..b761fd80a --- /dev/null +++ b/pkgs/yaml_edit/test/crash_test/testdata/mangled_json.yaml @@ -0,0 +1,15 @@ +{ + "string": +"hello world", + "map": { + "foo": +42 +,"bar": + "baz" + } + ,"list": [ +1,2 +, + 3 +,] +} diff --git a/pkgs/yaml_edit/test/crash_test/testdata/simple.yaml b/pkgs/yaml_edit/test/crash_test/testdata/simple.yaml new file mode 100644 index 000000000..96d77ae10 --- /dev/null +++ b/pkgs/yaml_edit/test/crash_test/testdata/simple.yaml @@ -0,0 +1,8 @@ +string: 'hello world' +map: + foo: 42 + bar: 'baz' +list: + - 1 + - 2 + - 3 diff --git a/pkgs/yaml_edit/test/crash_test/testdata/simple_comments.yaml b/pkgs/yaml_edit/test/crash_test/testdata/simple_comments.yaml new file mode 100644 index 000000000..2192c8d61 --- /dev/null +++ b/pkgs/yaml_edit/test/crash_test/testdata/simple_comments.yaml @@ -0,0 +1,9 @@ +# Initial comment +string: 'hello world' # Comment +map: + foo: 42 # Best number + bar: 'baz' +list: # Great starter list + - 1 + - 2 + - 3 From 0a1b82367c346fb890c8642bf17f8c777cd92430 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Thu, 18 Dec 2025 11:12:44 +0100 Subject: [PATCH 2/4] Fix debug leftovers --- .../yaml_edit/test/crash_test/crash_test.dart | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/pkgs/yaml_edit/test/crash_test/crash_test.dart b/pkgs/yaml_edit/test/crash_test/crash_test.dart index a9e1da27a..5aaf87e36 100644 --- a/pkgs/yaml_edit/test/crash_test/crash_test.dart +++ b/pkgs/yaml_edit/test/crash_test/crash_test.dart @@ -123,19 +123,14 @@ void _testJsonPath( for (final style in _scalarStyles) { test('$editorName.insertIntoList($path, $i, \'foo\\nbar\') as $style', () { - try { - final editor = YamlEditor(input); - editor.insertIntoList( - path, - i, - YamlScalar.wrap( - 'foo\nbar', - style: style, - )); - } catch (e) { - print(e.runtimeType); - rethrow; - } + final editor = YamlEditor(input); + editor.insertIntoList( + path, + i, + YamlScalar.wrap( + 'foo\nbar', + style: style, + )); }); } } From 224e1bb9486d0ed85b3958802a006ec882d6a785 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Thu, 18 Dec 2025 12:54:15 +0100 Subject: [PATCH 3/4] Update pkgs/yaml_edit/test/crash_test/testdata/json_comments.yaml Co-authored-by: Sigurd Meldgaard --- pkgs/yaml_edit/test/crash_test/testdata/json_comments.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/yaml_edit/test/crash_test/testdata/json_comments.yaml b/pkgs/yaml_edit/test/crash_test/testdata/json_comments.yaml index d0e8b1aca..e4fe234b4 100644 --- a/pkgs/yaml_edit/test/crash_test/testdata/json_comments.yaml +++ b/pkgs/yaml_edit/test/crash_test/testdata/json_comments.yaml @@ -7,7 +7,9 @@ }, "list": [ 1, # This is a good list - 2, + 2, # This comment is good + # But this? + # Or this? 3 ] } From e72cfd8638198cac2144f69186589d10316c3603 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Thu, 18 Dec 2025 12:58:20 +0100 Subject: [PATCH 4/4] Added test case for explicit key/value --- pkgs/yaml_edit/test/crash_test/crash_test.dart | 3 ++- .../test/crash_test/testdata/explicit_key_value.yaml | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 pkgs/yaml_edit/test/crash_test/testdata/explicit_key_value.yaml diff --git a/pkgs/yaml_edit/test/crash_test/crash_test.dart b/pkgs/yaml_edit/test/crash_test/crash_test.dart index 5aaf87e36..f6c6fcb10 100644 --- a/pkgs/yaml_edit/test/crash_test/crash_test.dart +++ b/pkgs/yaml_edit/test/crash_test/crash_test.dart @@ -23,9 +23,10 @@ final _scalarStyles = [ /// Files with tests that are broken, so we have to skip them final _skippedFiles = [ - 'mangled_json.yaml', 'block_strings.yaml', 'complex.yaml', + 'explicit_key_value.yaml', + 'mangled_json.yaml', 'simple_comments.yaml', ]; diff --git a/pkgs/yaml_edit/test/crash_test/testdata/explicit_key_value.yaml b/pkgs/yaml_edit/test/crash_test/testdata/explicit_key_value.yaml new file mode 100644 index 000000000..fbb0b8544 --- /dev/null +++ b/pkgs/yaml_edit/test/crash_test/testdata/explicit_key_value.yaml @@ -0,0 +1,12 @@ +? map +: k1: 1 + k2: 2 + k3: 3 +? list +: - 1 + - 2 + - 3 +? inlineMap +: {k1: 1, k2: 2, k3: 3} +? inlineList +: [1, 2, 3]