Skip to content

Commit

Permalink
Fix infinite loops after parsing final empty block scalar (#410)
Browse files Browse the repository at this point in the history
  • Loading branch information
@fktn-k
fktn-k authored Oct 13, 2024
1 parent 0af84d3 commit 6817d97
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 2 deletions.
6 changes: 5 additions & 1 deletion include/fkYAML/detail/input/lexical_analyzer.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -806,9 +806,13 @@ class lexical_analyzer {
constexpr str_view space_filter = " \t\n";
std::size_t first_non_space_pos = sv.find_first_not_of(space_filter);
if (first_non_space_pos == str_view::npos) {
// empty block scalar
// empty block scalar with no subsequent tokens.
indent = static_cast<uint32_t>(sv.size());
token = sv;

// Without the following iterator update, lexer cannot reach the end of input buffer and causes infinite
// loops from the next loop. (https://github.com/fktn-k/fkYAML/pull/410)
m_cur_itr = m_end_itr;
return;
}

Expand Down
6 changes: 5 additions & 1 deletion single_include/fkYAML/node.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3701,9 +3701,13 @@ class lexical_analyzer {
constexpr str_view space_filter = " \t\n";
std::size_t first_non_space_pos = sv.find_first_not_of(space_filter);
if (first_non_space_pos == str_view::npos) {
// empty block scalar
// empty block scalar with no subsequent tokens.
indent = static_cast<uint32_t>(sv.size());
token = sv;

// Without the following iterator update, lexer cannot reach the end of input buffer and causes infinite
// loops from the next loop. (https://github.com/fktn-k/fkYAML/pull/410)
m_cur_itr = m_end_itr;
return;
}

Expand Down
72 changes: 72 additions & 0 deletions test/unit_test/test_lexical_analyzer_class.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -612,6 +612,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 6);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::STRIP);
REQUIRE(lexer.get_block_scalar_header().indent == 3); // lexer returns content size if empty.

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("empty literal string scalar with clip chomping") {
Expand All @@ -625,6 +628,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 5);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 3); // lexer returns content size if empty.

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("empty literal string scalar with keep chomping") {
Expand All @@ -638,6 +644,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 6);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::KEEP);
REQUIRE(lexer.get_block_scalar_header().indent == 3); // lexer returns content size if empty.

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar with 0 indent level.") {
Expand Down Expand Up @@ -676,6 +685,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 7);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal scalar with the first line being more indented than the indicated level") {
Expand All @@ -691,6 +703,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 18);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar") {
Expand All @@ -705,6 +720,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 14);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar with implicit indentation and strip chomping") {
Expand All @@ -723,6 +741,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 24);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::STRIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar with explicit indentation and strip chomping") {
Expand All @@ -740,6 +761,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 26);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::STRIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar with implicit indentation and clip chomping") {
Expand All @@ -758,6 +782,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 23);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar with explicit indentation and clip chomping") {
Expand All @@ -775,6 +802,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 25);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar with clip chomping and no trailing newlines") {
Expand All @@ -791,6 +821,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 23);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar with implicit indentation and keep chomping") {
Expand All @@ -809,6 +842,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 24);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::KEEP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar with explicit indentation and keep chomping") {
Expand All @@ -826,6 +862,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + 26);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::KEEP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar with trailing spaces/tabs after the block scalar header.") {
Expand All @@ -841,6 +880,9 @@ TEST_CASE("LexicalAnalyzer_LiteralStringScalar") {
REQUIRE(token.str.end() == &input[0] + input.size());
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("literal string scalar with invalid block scalar headers") {
Expand Down Expand Up @@ -872,6 +914,9 @@ TEST_CASE("LexicalAnalyzer_FoldedString") {
REQUIRE(token.str.end() == &input[0] + 6);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::STRIP);
REQUIRE(lexer.get_block_scalar_header().indent == 3); // lexer returns content size if empty.

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("empty folded string scalar with clip chomping") {
Expand All @@ -885,6 +930,9 @@ TEST_CASE("LexicalAnalyzer_FoldedString") {
REQUIRE(token.str.end() == &input[0] + 5);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 3); // lexer returns content size if empty.

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("empty folded string scalar with keep chomping") {
Expand All @@ -898,6 +946,9 @@ TEST_CASE("LexicalAnalyzer_FoldedString") {
REQUIRE(token.str.end() == &input[0] + 6);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::KEEP);
REQUIRE(lexer.get_block_scalar_header().indent == 3); // lexer returns content size if empty.

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("folded string scalar with 0 indent level") {
Expand Down Expand Up @@ -928,6 +979,9 @@ TEST_CASE("LexicalAnalyzer_FoldedString") {
REQUIRE(token.str.end() == &input[0] + 17);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("folded string scalar with the non-first line being more indented than the indicated level") {
Expand All @@ -942,6 +996,9 @@ TEST_CASE("LexicalAnalyzer_FoldedString") {
REQUIRE(token.str.end() == &input[0] + 17);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("folded string scalar") {
Expand All @@ -959,6 +1016,9 @@ TEST_CASE("LexicalAnalyzer_FoldedString") {
REQUIRE(token.str.end() == &input[0] + 20);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("folded string scalar with implicit indentation and strip chomping") {
Expand All @@ -975,6 +1035,9 @@ TEST_CASE("LexicalAnalyzer_FoldedString") {
REQUIRE(token.str.end() == &input[0] + 18);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::STRIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("folded string scalar with implicit indentation and clip chomping") {
Expand All @@ -991,6 +1054,9 @@ TEST_CASE("LexicalAnalyzer_FoldedString") {
REQUIRE(token.str.end() == &input[0] + 18);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("folded string scalar with implicit indentation and keep chomping") {
Expand All @@ -1007,6 +1073,9 @@ TEST_CASE("LexicalAnalyzer_FoldedString") {
REQUIRE(token.str.end() == &input[0] + 18);
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::KEEP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("folded string scalar with trailing spaces/tabs/comments after the block scalar header.") {
Expand All @@ -1022,6 +1091,9 @@ TEST_CASE("LexicalAnalyzer_FoldedString") {
REQUIRE(token.str.end() == &input[0] + input.size());
REQUIRE(lexer.get_block_scalar_header().chomp == fkyaml::detail::chomping_indicator_t::CLIP);
REQUIRE(lexer.get_block_scalar_header().indent == 2);

REQUIRE_NOTHROW(token = lexer.get_next_token());
REQUIRE(token.type == fkyaml::detail::lexical_token_t::END_OF_BUFFER);
}

SECTION("folded string scalar with invalid block scalar headers") {
Expand Down

0 comments on commit 6817d97

Please sign in to comment.