diff options
author | Spartan322 <Megacake1234@gmail.com> | 2024-07-02 09:51:59 +0200 |
---|---|---|
committer | Spartan322 <Megacake1234@gmail.com> | 2024-07-05 19:17:25 +0200 |
commit | 3eb78b27505b602c1ccfa952c4cc00f942ccb2b9 (patch) | |
tree | 457b7fbda6d4470465c05d59b0ca51ed30628640 /src/openvic-dataloader/v2script/SimpleGrammar.hpp | |
parent | deed8ec0ae23651529a58125012c1b4aab015d02 (diff) |
Fix string interning pointer invalidity for ASTsimplify-string-interning
Fix errorneous string intern compare
Add symbol interning tests
Optimize non-conversion identifier cases
Add `File::size()`
Add InstanceOf concept
Remove `FlatValue::value(const symbol_interner_type&)`
Add `AbstractSyntaxTree::intern/intern_cst(lexy::lexeme)` overload
Add `DiagnosticLogger::intern/intern_cstr(lexy::lexeme)` overload
Use pinned_vector to maintain string interning pointer validity for buffers
Add vmcontainer submodule for pinned_vector
pinned_vector reserves virtual memory at runtime using OS APIs to maintain pointer validity
Remove Exception and RTTI requirement from range-v3 submodule
Diffstat (limited to 'src/openvic-dataloader/v2script/SimpleGrammar.hpp')
-rw-r--r-- | src/openvic-dataloader/v2script/SimpleGrammar.hpp | 17 |
1 files changed, 15 insertions, 2 deletions
diff --git a/src/openvic-dataloader/v2script/SimpleGrammar.hpp b/src/openvic-dataloader/v2script/SimpleGrammar.hpp index d42ce07..c47b243 100644 --- a/src/openvic-dataloader/v2script/SimpleGrammar.hpp +++ b/src/openvic-dataloader/v2script/SimpleGrammar.hpp @@ -126,6 +126,15 @@ namespace ovdl::v2script::grammar { do { if constexpr (std::same_as<encoding, lexy::default_encoding> || std::same_as<encoding, lexy::byte_encoding>) { if (lexy::scan_result<lexy::lexeme<Reader>> ascii_result; scanner.branch(ascii_result, lexy::dsl::identifier(ascii))) { + if (!scanner.peek(data_char_class)) { + if (ascii_result.value().size() == 0) { + return lexy::scan_failed; + } + + auto value = state.ast().intern(ascii_result.value()); + return state.ast().template create<ast::IdentifierValue>(ovdl::NodeLocation::make_from(content_begin, scanner.position()), value); + } + value_result.append(ascii_result.value().begin(), ascii_result.value().end()); continue; } @@ -150,8 +159,12 @@ namespace ovdl::v2script::grammar { } else { auto lexeme_result = scanner.template parse<lexy::lexeme<Reader>>(lexy::dsl::identifier(utf_char_class)); if (lexeme_result) { - value_result.append(lexeme_result.value().begin(), lexeme_result.value().size()); - break; + if (lexeme_result.value().size() == 0) { + return lexy::scan_failed; + } + + auto value = state.ast().intern(lexeme_result.value()); + return state.ast().template create<ast::IdentifierValue>(ovdl::NodeLocation::make_from(content_begin, scanner.position()), value); } } } while (scanner); |