From 338bb84a5f5f883fb50fbe4f64398ab40dad74fe Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Fri, 15 Sep 2023 13:13:28 -0400 Subject: Add ParseOptions: NoEscapeString to true to prevent escape behavior in strings Event and Decision grammars use string escaping (for now) Update lexy to latest master: Use `git submodule update --init --recursive` --- deps/lexy | 2 +- .../v2script/DecisionGrammar.hpp | 42 ++++++------ src/openvic-dataloader/v2script/EffectGrammar.hpp | 4 +- src/openvic-dataloader/v2script/EventGrammar.hpp | 42 ++++++------ .../v2script/ModifierGrammar.hpp | 6 +- src/openvic-dataloader/v2script/Parser.cpp | 2 +- src/openvic-dataloader/v2script/SimpleGrammar.hpp | 75 ++++++++++++++-------- src/openvic-dataloader/v2script/TriggerGrammar.hpp | 4 +- 8 files changed, 100 insertions(+), 77 deletions(-) diff --git a/deps/lexy b/deps/lexy index 5e2601b..d5041eb 160000 --- a/deps/lexy +++ b/deps/lexy @@ -1 +1 @@ -Subproject commit 5e2601b3de57e979b5d23b8936cf040eb228fb94 +Subproject commit d5041ebb894bd2a4c3a8dceeaa7a579781f6b851 diff --git a/src/openvic-dataloader/v2script/DecisionGrammar.hpp b/src/openvic-dataloader/v2script/DecisionGrammar.hpp index 93ba0f5..a1c1e3e 100644 --- a/src/openvic-dataloader/v2script/DecisionGrammar.hpp +++ b/src/openvic-dataloader/v2script/DecisionGrammar.hpp @@ -23,25 +23,25 @@ namespace ovdl::v2script::grammar { // Macros ////////////////// // Produces _rule and _p -#define OVDL_GRAMMAR_KEYWORD_DEFINE(KW_NAME) \ - struct KW_NAME##_rule { \ - static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_); \ - static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ - static constexpr auto value = lexy::noop; \ - }; \ +#define OVDL_GRAMMAR_KEYWORD_DEFINE(KW_NAME) \ + struct KW_NAME##_rule { \ + static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_>); \ + static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ + static constexpr auto value = lexy::noop; \ + }; \ static constexpr auto KW_NAME##_p = lexy::dsl::p // Produces _rule and _p and _rule::flag and _rule::too_many_error -#define OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(KW_NAME) \ - struct KW_NAME##_rule { \ - static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_); \ - static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ - static constexpr auto value = lexy::noop; \ - static constexpr auto flag = lexy::dsl::context_flag; \ - struct too_many_error { \ - static constexpr auto name = "expected left side " #KW_NAME " to be found once"; \ - }; \ - }; \ +#define OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(KW_NAME) \ + struct KW_NAME##_rule { \ + static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_>); \ + static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ + static constexpr auto value = lexy::noop; \ + static constexpr auto flag = lexy::dsl::context_flag; \ + struct too_many_error { \ + static constexpr auto name = "expected left side " #KW_NAME " to be found once"; \ + }; \ + }; \ static constexpr auto KW_NAME##_p = lexy::dsl::p >> (lexy::dsl::must(KW_NAME##_rule::flag.is_reset()).error + KW_NAME##_rule::flag.set()) ////////////////// // Macros @@ -49,7 +49,7 @@ namespace ovdl::v2script::grammar { struct DecisionStatement { template struct _StringStatement { - static constexpr auto rule = Production >> (lexy::dsl::p | lexy::dsl::p); + static constexpr auto rule = Production >> (lexy::dsl::p> | lexy::dsl::p>); static constexpr auto value = lexy::forward; }; template @@ -72,14 +72,14 @@ namespace ovdl::v2script::grammar { constexpr auto effect_statement = effect_p >> lexy::dsl::p; constexpr auto ai_will_do_statement = ai_will_do_p >> lexy::dsl::p; - return lexy::dsl::p >> + return lexy::dsl::p> >> (create_flags + lexy::dsl::equal_sign + lexy::dsl::curly_bracketed.list( potential_statement | allow_statement | effect_statement | ai_will_do_statement | - lexy::dsl::p)); + lexy::dsl::p>)); }(); static constexpr auto value = @@ -95,7 +95,7 @@ namespace ovdl::v2script::grammar { struct DecisionList { static constexpr auto rule = - LEXY_KEYWORD("political_decisions", lexy::dsl::inline_) >> + LEXY_KEYWORD("political_decisions", lexy::dsl::inline_>) >> (lexy::dsl::equal_sign + lexy::dsl::curly_bracketed.opt_list(lexy::dsl::p)); static constexpr auto value = @@ -116,7 +116,7 @@ namespace ovdl::v2script::grammar { static constexpr auto rule = lexy::dsl::terminator(lexy::dsl::eof).list( // lexy::dsl::p | // - lexy::dsl::p); + lexy::dsl::p>); static constexpr auto value = lexy::as_list> >> lexy::new_; }; diff --git a/src/openvic-dataloader/v2script/EffectGrammar.hpp b/src/openvic-dataloader/v2script/EffectGrammar.hpp index 9f164b2..1b85382 100644 --- a/src/openvic-dataloader/v2script/EffectGrammar.hpp +++ b/src/openvic-dataloader/v2script/EffectGrammar.hpp @@ -9,7 +9,7 @@ namespace ovdl::v2script::grammar { struct EffectStatement { - static constexpr auto rule = lexy::dsl::inline_; + static constexpr auto rule = lexy::dsl::inline_>; static constexpr auto value = lexy::callback( [](auto name, auto&& initalizer) { @@ -18,7 +18,7 @@ namespace ovdl::v2script::grammar { }; struct EffectList { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p); + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>); static constexpr auto value = lexy::as_list> >> diff --git a/src/openvic-dataloader/v2script/EventGrammar.hpp b/src/openvic-dataloader/v2script/EventGrammar.hpp index 93a52bf..e6e4c02 100644 --- a/src/openvic-dataloader/v2script/EventGrammar.hpp +++ b/src/openvic-dataloader/v2script/EventGrammar.hpp @@ -21,25 +21,25 @@ namespace ovdl::v2script::grammar { // Macros ////////////////// // Produces _rule and _p -#define OVDL_GRAMMAR_KEYWORD_DEFINE(KW_NAME) \ - struct KW_NAME##_rule { \ - static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_); \ - static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ - static constexpr auto value = lexy::noop; \ - }; \ +#define OVDL_GRAMMAR_KEYWORD_DEFINE(KW_NAME) \ + struct KW_NAME##_rule { \ + static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_>); \ + static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ + static constexpr auto value = lexy::noop; \ + }; \ static constexpr auto KW_NAME##_p = lexy::dsl::p // Produces _rule and _p and _rule::flag and _rule::too_many_error -#define OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(KW_NAME) \ - struct KW_NAME##_rule { \ - static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_); \ - static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ - static constexpr auto value = lexy::noop; \ - static constexpr auto flag = lexy::dsl::context_flag; \ - struct too_many_error { \ - static constexpr auto name = "expected left side " #KW_NAME " to be found once"; \ - }; \ - }; \ +#define OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(KW_NAME) \ + struct KW_NAME##_rule { \ + static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_>); \ + static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ + static constexpr auto value = lexy::noop; \ + static constexpr auto flag = lexy::dsl::context_flag; \ + struct too_many_error { \ + static constexpr auto name = "expected left side " #KW_NAME " to be found once"; \ + }; \ + }; \ static constexpr auto KW_NAME##_p = lexy::dsl::p >> (lexy::dsl::must(KW_NAME##_rule::flag.is_reset()).error + KW_NAME##_rule::flag.set()) ////////////////// // Macros @@ -52,7 +52,7 @@ namespace ovdl::v2script::grammar { OVDL_GRAMMAR_KEYWORD_DEFINE(months); struct MonthValue { - static constexpr auto rule = lexy::dsl::inline_; + static constexpr auto rule = lexy::dsl::inline_>; static constexpr auto value = lexy::as_string | lexy::new_; }; @@ -70,7 +70,7 @@ namespace ovdl::v2script::grammar { template struct _StringStatement { - static constexpr auto rule = Production >> (lexy::dsl::p | lexy::dsl::p); + static constexpr auto rule = Production >> (lexy::dsl::p> | lexy::dsl::p>); static constexpr auto value = lexy::callback( [](auto&& value) { @@ -116,7 +116,7 @@ namespace ovdl::v2script::grammar { OVDL_GRAMMAR_KEYWORD_DEFINE(option); static constexpr auto rule = [] { - constexpr auto symbol_value = lexy::dsl::symbol(lexy::dsl::inline_); + constexpr auto symbol_value = lexy::dsl::symbol(lexy::dsl::inline_>); constexpr auto create_flags = id_rule::flag.create() + @@ -153,7 +153,7 @@ namespace ovdl::v2script::grammar { mean_time_to_happen_statement | trigger_statement | option_statement | - lexy::dsl::p)); + lexy::dsl::p>)); }(); static constexpr auto value = @@ -171,7 +171,7 @@ namespace ovdl::v2script::grammar { // Allow arbitrary spaces between individual tokens. static constexpr auto whitespace = whitespace_specifier | comment_specifier; - static constexpr auto rule = lexy::dsl::terminator(lexy::dsl::eof).list(lexy::dsl::p | lexy::dsl::p); + static constexpr auto rule = lexy::dsl::terminator(lexy::dsl::eof).list(lexy::dsl::p | lexy::dsl::p>); static constexpr auto value = lexy::as_list> >> lexy::new_; }; diff --git a/src/openvic-dataloader/v2script/ModifierGrammar.hpp b/src/openvic-dataloader/v2script/ModifierGrammar.hpp index 632cfd5..96e928c 100644 --- a/src/openvic-dataloader/v2script/ModifierGrammar.hpp +++ b/src/openvic-dataloader/v2script/ModifierGrammar.hpp @@ -8,11 +8,11 @@ #include "TriggerGrammar.hpp" namespace ovdl::v2script::grammar { - constexpr auto modifier_keyword = LEXY_KEYWORD("modifier", lexy::dsl::inline_); - constexpr auto factor_keyword = LEXY_KEYWORD("factor", lexy::dsl::inline_); + constexpr auto modifier_keyword = LEXY_KEYWORD("modifier", lexy::dsl::inline_>); + constexpr auto factor_keyword = LEXY_KEYWORD("factor", lexy::dsl::inline_>); struct FactorStatement { - static constexpr auto rule = factor_keyword >> lexy::dsl::equal_sign + lexy::dsl::inline_; + static constexpr auto rule = factor_keyword >> lexy::dsl::equal_sign + lexy::dsl::inline_>; static constexpr auto value = lexy::as_string | lexy::new_; }; diff --git a/src/openvic-dataloader/v2script/Parser.cpp b/src/openvic-dataloader/v2script/Parser.cpp index 0ff7612..9e74e32 100644 --- a/src/openvic-dataloader/v2script/Parser.cpp +++ b/src/openvic-dataloader/v2script/Parser.cpp @@ -175,7 +175,7 @@ bool Parser::simple_parse() { _warnings.push_back(warnings::make_utf8_warning(_file_path)); } - auto errors = _buffer_handler->parse(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + auto errors = _buffer_handler->parse>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); if (errors) { _errors.reserve(errors->size()); for (auto& err : errors.value()) { diff --git a/src/openvic-dataloader/v2script/SimpleGrammar.hpp b/src/openvic-dataloader/v2script/SimpleGrammar.hpp index d478bdf..2679598 100644 --- a/src/openvic-dataloader/v2script/SimpleGrammar.hpp +++ b/src/openvic-dataloader/v2script/SimpleGrammar.hpp @@ -22,7 +22,18 @@ * DAT-643 */ namespace ovdl::v2script::grammar { + struct ParseOptions { + /// @brief Makes string parsing avoid string escapes + bool NoStringEscape; + }; + + static constexpr ParseOptions NoStringEscapeOption = ParseOptions { true }; + static constexpr ParseOptions StringEscapeOption = ParseOptions { false }; + + template struct StatementListBlock; + template + struct AssignmentStatement; /* REQUIREMENTS: DAT-630 */ static constexpr auto whitespace_specifier = lexy::dsl::ascii::blank / lexy::dsl::ascii::newline; @@ -45,6 +56,18 @@ namespace ovdl::v2script::grammar { static constexpr auto data_char_class = LEXY_CHAR_CLASS("DataSpecifier", data_specifier); + static constexpr auto escaped_symbols = lexy::symbol_table // + .map<'"'>('"') + .map<'\''>('\'') + .map<'\\'>('\\') + .map<'/'>('/') + .map<'b'>('\b') + .map<'f'>('\f') + .map<'n'>('\n') + .map<'r'>('\r') + .map<'t'>('\t'); + + template struct Identifier { static constexpr auto rule = lexy::dsl::identifier(data_char_class); static constexpr auto value = lexy::callback( @@ -58,27 +81,22 @@ namespace ovdl::v2script::grammar { * DAT-633 * DAT-634 */ + template struct StringExpression { - static constexpr auto escaped_symbols = lexy::symbol_table // - .map<'"'>('"') - .map<'\''>('\'') - .map<'\\'>('\\') - .map<'/'>('/') - .map<'b'>('\b') - .map<'f'>('\f') - .map<'n'>('\n') - .map<'r'>('\r') - .map<'t'>('\t'); static constexpr auto rule = [] { // Arbitrary code points that aren't control characters. auto c = ovdl::detail::lexydsl::make_range<0x20, 0xFF>() - lexy::dsl::ascii::control; - // Escape sequences start with a backlash. - // They either map one of the symbols, - // or a Unicode code point of the form uXXXX. - auto escape = lexy::dsl::backslash_escape // - .symbol(); - return lexy::dsl::delimited(lexy::dsl::position(lexy::dsl::lit_b<'"'>))(c, escape); + if constexpr (Options.NoStringEscape) { + return lexy::dsl::delimited(lexy::dsl::position(lexy::dsl::lit_b<'"'>))(c); + } else { + // Escape sequences start with a backlash. + // They either map one of the symbols, + // or a Unicode code point of the form uXXXX. + auto escape = lexy::dsl::backslash_escape // + .symbol(); + return lexy::dsl::delimited(lexy::dsl::position(lexy::dsl::lit_b<'"'>))(c, escape); + } }(); static constexpr auto value = @@ -90,16 +108,18 @@ namespace ovdl::v2script::grammar { }; /* REQUIREMENTS: DAT-638 */ + template struct ValueExpression { - static constexpr auto rule = lexy::dsl::p | lexy::dsl::p; + static constexpr auto rule = lexy::dsl::p> | lexy::dsl::p>; static constexpr auto value = lexy::forward; }; + template struct SimpleAssignmentStatement { static constexpr auto rule = - lexy::dsl::position(lexy::dsl::p) >> - lexy::dsl::equal_sign + - (lexy::dsl::p | lexy::dsl::recurse_branch); + lexy::dsl::position(lexy::dsl::p>) >> + (lexy::dsl::equal_sign + + (lexy::dsl::p> | lexy::dsl::recurse_branch>)); static constexpr auto value = lexy::callback( [](const char* pos, auto name, auto&& initalizer) { @@ -108,13 +128,14 @@ namespace ovdl::v2script::grammar { }; /* REQUIREMENTS: DAT-639 */ + template struct AssignmentStatement { static constexpr auto rule = - lexy::dsl::position(lexy::dsl::p) >> + lexy::dsl::position(lexy::dsl::p>) >> (lexy::dsl::equal_sign >> - (lexy::dsl::p | lexy::dsl::recurse_branch) | + (lexy::dsl::p> | lexy::dsl::recurse_branch>) | lexy::dsl::else_ >> lexy::dsl::return_) | - lexy::dsl::p; + lexy::dsl::p>; static constexpr auto value = lexy::callback( [](const char* pos, auto name, lexy::nullopt = {}) { @@ -129,11 +150,12 @@ namespace ovdl::v2script::grammar { }; /* REQUIREMENTS: DAT-640 */ + template struct StatementListBlock { static constexpr auto rule = lexy::dsl::position(lexy::dsl::curly_bracketed.open()) >> - lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p)) + - lexy::dsl::opt(lexy::dsl::semicolon) >> + (lexy::dsl::opt(lexy::dsl::list(lexy::dsl::recurse_branch>)) + + lexy::dsl::opt(lexy::dsl::semicolon)) >> lexy::dsl::position(lexy::dsl::curly_bracketed.close()); static constexpr auto value = @@ -156,11 +178,12 @@ namespace ovdl::v2script::grammar { }); }; + template struct File { // Allow arbitrary spaces between individual tokens. static constexpr auto whitespace = whitespace_specifier | comment_specifier; - static constexpr auto rule = lexy::dsl::position + lexy::dsl::terminator(lexy::dsl::eof).list(lexy::dsl::p); + static constexpr auto rule = lexy::dsl::position + lexy::dsl::terminator(lexy::dsl::eof).list(lexy::dsl::p>); static constexpr auto value = lexy::as_list> >> lexy::new_; }; diff --git a/src/openvic-dataloader/v2script/TriggerGrammar.hpp b/src/openvic-dataloader/v2script/TriggerGrammar.hpp index 6d964db..3849b12 100644 --- a/src/openvic-dataloader/v2script/TriggerGrammar.hpp +++ b/src/openvic-dataloader/v2script/TriggerGrammar.hpp @@ -9,7 +9,7 @@ namespace ovdl::v2script::grammar { struct TriggerStatement { - static constexpr auto rule = lexy::dsl::inline_; + static constexpr auto rule = lexy::dsl::inline_>; static constexpr auto value = lexy::callback( [](auto name, auto&& initalizer) { @@ -18,7 +18,7 @@ namespace ovdl::v2script::grammar { }; struct TriggerList { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p); + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>); static constexpr auto value = lexy::as_list> >> -- cgit v1.2.3-56-ga3b1