From 3eb78b27505b602c1ccfa952c4cc00f942ccb2b9 Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Tue, 2 Jul 2024 03:51:59 -0400 Subject: Fix string interning pointer invalidity for AST Fix errorneous string intern compare Add symbol interning tests Optimize non-conversion identifier cases Add `File::size()` Add InstanceOf concept Remove `FlatValue::value(const symbol_interner_type&)` Add `AbstractSyntaxTree::intern/intern_cst(lexy::lexeme)` overload Add `DiagnosticLogger::intern/intern_cstr(lexy::lexeme)` overload Use pinned_vector to maintain string interning pointer validity for buffers Add vmcontainer submodule for pinned_vector pinned_vector reserves virtual memory at runtime using OS APIs to maintain pointer validity Remove Exception and RTTI requirement from range-v3 submodule --- src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp | 6 +++--- src/openvic-dataloader/v2script/EventGrammar.hpp | 12 ++++++------ src/openvic-dataloader/v2script/Parser.cpp | 2 +- src/openvic-dataloader/v2script/SimpleGrammar.hpp | 17 +++++++++++++++-- 4 files changed, 25 insertions(+), 12 deletions(-) (limited to 'src/openvic-dataloader/v2script') diff --git a/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp b/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp index 71985f3..72ff627 100644 --- a/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp +++ b/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp @@ -75,7 +75,7 @@ std::string FileAbstractSyntaxTree::make_list_visualizer() const { dryad::visit_node( node, [&](const FlatValue* value) { - result.append(value->value(this->_symbol_interner)); + result.append(value->value().c_str()); }, [&](const ListValue* value) { }, @@ -107,10 +107,10 @@ std::string FileAbstractSyntaxTree::make_native_visualizer() const { dryad::visit_tree( this->_tree, [&](const IdentifierValue* value) { - result.append(value->value(this->_symbol_interner)); + result.append(value->value().c_str()); }, [&](const StringValue* value) { - result.append(1, '"').append(value->value(this->_symbol_interner)).append(1, '"'); + result.append(1, '"').append(value->value().c_str()).append(1, '"'); }, [&](dryad::child_visitor visitor, const ValueStatement* statement) { visitor(statement->value()); diff --git a/src/openvic-dataloader/v2script/EventGrammar.hpp b/src/openvic-dataloader/v2script/EventGrammar.hpp index 130a233..20168f8 100644 --- a/src/openvic-dataloader/v2script/EventGrammar.hpp +++ b/src/openvic-dataloader/v2script/EventGrammar.hpp @@ -30,7 +30,7 @@ namespace ovdl::v2script::grammar { static constexpr auto value = dsl::callback( [](detail::IsParseState auto& state, ast::IdentifierValue* value) { bool is_number = true; - for (auto* current = value->value(state.ast().symbol_interner()); *current; current++) { + for (auto* current = value->value().c_str(); *current; current++) { is_number = is_number && std::isdigit(*current); if (!is_number) break; } @@ -95,16 +95,16 @@ namespace ovdl::v2script::grammar { static constexpr auto value = dsl::callback( [](detail::IsParseState auto& state, NodeLocation loc, ast::IdentifierValue* name, ast::ListValue* list) { - static auto country_decl = state.ast().intern_cstr("country_event"); - static auto province_decl = state.ast().intern_cstr("province_event"); + auto country_decl = state.ast().intern("country_event"); + auto province_decl = state.ast().intern("province_event"); - if (name->value(state.ast().symbol_interner()) != country_decl || name->value(state.ast().symbol_interner()) != province_decl) { - state.logger().warning("event declarator \"{}\" is not {} or {}", name->value(state.ast().symbol_interner()), country_decl, province_decl) // + if (name->value() != country_decl || name->value() != province_decl) { + state.logger().warning("event declarator \"{}\" is not {} or {}", name->value().c_str(), country_decl.c_str(), province_decl.c_str()) // .primary(loc, "here") .finish(); } - return state.ast().template create(loc, name->value(state.ast().symbol_interner()) == province_decl, list); + return state.ast().template create(loc, name->value() == province_decl, list); }); }; diff --git a/src/openvic-dataloader/v2script/Parser.cpp b/src/openvic-dataloader/v2script/Parser.cpp index a0003b1..9c6a0c5 100644 --- a/src/openvic-dataloader/v2script/Parser.cpp +++ b/src/openvic-dataloader/v2script/Parser.cpp @@ -266,7 +266,7 @@ const FileTree* Parser::get_file_node() const { } std::string_view Parser::value(const ovdl::v2script::ast::FlatValue* node) const { - return node->value(_parse_handler->parse_state().ast().symbol_interner()); + return node->value().view(); } std::string Parser::make_native_string() const { diff --git a/src/openvic-dataloader/v2script/SimpleGrammar.hpp b/src/openvic-dataloader/v2script/SimpleGrammar.hpp index d42ce07..c47b243 100644 --- a/src/openvic-dataloader/v2script/SimpleGrammar.hpp +++ b/src/openvic-dataloader/v2script/SimpleGrammar.hpp @@ -126,6 +126,15 @@ namespace ovdl::v2script::grammar { do { if constexpr (std::same_as || std::same_as) { if (lexy::scan_result> ascii_result; scanner.branch(ascii_result, lexy::dsl::identifier(ascii))) { + if (!scanner.peek(data_char_class)) { + if (ascii_result.value().size() == 0) { + return lexy::scan_failed; + } + + auto value = state.ast().intern(ascii_result.value()); + return state.ast().template create(ovdl::NodeLocation::make_from(content_begin, scanner.position()), value); + } + value_result.append(ascii_result.value().begin(), ascii_result.value().end()); continue; } @@ -150,8 +159,12 @@ namespace ovdl::v2script::grammar { } else { auto lexeme_result = scanner.template parse>(lexy::dsl::identifier(utf_char_class)); if (lexeme_result) { - value_result.append(lexeme_result.value().begin(), lexeme_result.value().size()); - break; + if (lexeme_result.value().size() == 0) { + return lexy::scan_failed; + } + + auto value = state.ast().intern(lexeme_result.value()); + return state.ast().template create(ovdl::NodeLocation::make_from(content_begin, scanner.position()), value); } } } while (scanner); -- cgit v1.2.3-56-ga3b1