Merge pull request #50 from OpenVicProject/simplify-string-interning

Fix string interning pointer invalidity for AST

4 files changed, 25 insertions, 12 deletions

diff --git a/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp b/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp
index 71985f3..72ff627 100644
--- a/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp
+++ b/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp
@@ -75,7 +75,7 @@ std::string FileAbstractSyntaxTree::make_list_visualizer() const {
 		dryad::visit_node(
 			node,
 			[&](const FlatValue* value) {
-				result.append(value->value(this->_symbol_interner));
+				result.append(value->value().c_str());
 			},
 			[&](const ListValue* value) {
 			},
@@ -107,10 +107,10 @@ std::string FileAbstractSyntaxTree::make_native_visualizer() const {
 	dryad::visit_tree(
 		this->_tree,
 		[&](const IdentifierValue* value) {
-			result.append(value->value(this->_symbol_interner));
+			result.append(value->value().c_str());
 		},
 		[&](const StringValue* value) {
-			result.append(1, '"').append(value->value(this->_symbol_interner)).append(1, '"');
+			result.append(1, '"').append(value->value().c_str()).append(1, '"');
 		},
 		[&](dryad::child_visitor<NodeKind> visitor, const ValueStatement* statement) {
 			visitor(statement->value());
diff --git a/src/openvic-dataloader/v2script/EventGrammar.hpp b/src/openvic-dataloader/v2script/EventGrammar.hpp
index 130a233..20168f8 100644
--- a/src/openvic-dataloader/v2script/EventGrammar.hpp
+++ b/src/openvic-dataloader/v2script/EventGrammar.hpp
@@ -30,7 +30,7 @@ namespace ovdl::v2script::grammar {
 			static constexpr auto value = dsl::callback<ast::IdentifierValue*>(
 				[](detail::IsParseState auto& state, ast::IdentifierValue* value) {
 					bool is_number = true;
-					for (auto* current = value->value(state.ast().symbol_interner()); *current; current++) {
+					for (auto* current = value->value().c_str(); *current; current++) {
 						is_number = is_number && std::isdigit(*current);
 						if (!is_number) break;
 					}
@@ -95,16 +95,16 @@ namespace ovdl::v2script::grammar {
 		static constexpr auto value =
 			dsl::callback<ast::EventStatement*>(
 				[](detail::IsParseState auto& state, NodeLocation loc, ast::IdentifierValue* name, ast::ListValue* list) {
-					static auto country_decl = state.ast().intern_cstr("country_event");
-					static auto province_decl = state.ast().intern_cstr("province_event");
+					auto country_decl = state.ast().intern("country_event");
+					auto province_decl = state.ast().intern("province_event");
 
-					if (name->value(state.ast().symbol_interner()) != country_decl || name->value(state.ast().symbol_interner()) != province_decl) {
-						state.logger().warning("event declarator \"{}\" is not {} or {}", name->value(state.ast().symbol_interner()), country_decl, province_decl) //
+					if (name->value() != country_decl || name->value() != province_decl) {
+						state.logger().warning("event declarator \"{}\" is not {} or {}", name->value().c_str(), country_decl.c_str(), province_decl.c_str()) //
 							.primary(loc, "here")
 							.finish();
 					}
 
-					return state.ast().template create<ast::EventStatement>(loc, name->value(state.ast().symbol_interner()) == province_decl, list);
+					return state.ast().template create<ast::EventStatement>(loc, name->value() == province_decl, list);
 				});
 	};
 
diff --git a/src/openvic-dataloader/v2script/Parser.cpp b/src/openvic-dataloader/v2script/Parser.cpp
index a0003b1..9c6a0c5 100644
--- a/src/openvic-dataloader/v2script/Parser.cpp
+++ b/src/openvic-dataloader/v2script/Parser.cpp
@@ -266,7 +266,7 @@ const FileTree* Parser::get_file_node() const {
 }
 
 std::string_view Parser::value(const ovdl::v2script::ast::FlatValue* node) const {
-	return node->value(_parse_handler->parse_state().ast().symbol_interner());
+	return node->value().view();
 }
 
 std::string Parser::make_native_string() const {
diff --git a/src/openvic-dataloader/v2script/SimpleGrammar.hpp b/src/openvic-dataloader/v2script/SimpleGrammar.hpp
index d42ce07..c47b243 100644
--- a/src/openvic-dataloader/v2script/SimpleGrammar.hpp
+++ b/src/openvic-dataloader/v2script/SimpleGrammar.hpp
@@ -126,6 +126,15 @@ namespace ovdl::v2script::grammar {
 				do {
 					if constexpr (std::same_as<encoding, lexy::default_encoding> || std::same_as<encoding, lexy::byte_encoding>) {
 						if (lexy::scan_result<lexy::lexeme<Reader>> ascii_result; scanner.branch(ascii_result, lexy::dsl::identifier(ascii))) {
+							if (!scanner.peek(data_char_class)) {
+								if (ascii_result.value().size() == 0) {
+									return lexy::scan_failed;
+								}
+
+								auto value = state.ast().intern(ascii_result.value());
+								return state.ast().template create<ast::IdentifierValue>(ovdl::NodeLocation::make_from(content_begin, scanner.position()), value);
+							}
+
 							value_result.append(ascii_result.value().begin(), ascii_result.value().end());
 							continue;
 						}
@@ -150,8 +159,12 @@ namespace ovdl::v2script::grammar {
 					} else {
 						auto lexeme_result = scanner.template parse<lexy::lexeme<Reader>>(lexy::dsl::identifier(utf_char_class));
 						if (lexeme_result) {
-							value_result.append(lexeme_result.value().begin(), lexeme_result.value().size());
-							break;
+							if (lexeme_result.value().size() == 0) {
+								return lexy::scan_failed;
+							}
+
+							auto value = state.ast().intern(lexeme_result.value());
+							return state.ast().template create<ast::IdentifierValue>(ovdl::NodeLocation::make_from(content_begin, scanner.position()), value);
 						}
 					}
 				} while (scanner);