Add runtime encoding detection and conversionfix/char-detection

	Win-1251/1252 detection is a reduced C++ version of https://github.com/hsivonen/chardetng
Add manually-specified encoding fallback
Add default system encoding fallback
Add error recovery to v2script
Add unknown encoding detection warning
Remove csv::Parser templating
Fix lua files dropping data

Update lexy to foonathan/lexy@1e5d99fa3826b1c3c8628d3a11117fb4fb4cc0d0

Remove exclusive reliance on lexy::default_encoding for v2script

Move internal concepts to src/openvic-detail/InternalConcepts.hpp
Move contents of DetectUtf8.hpp to src/detail/Detect.hpp
Move openvic-dataloader/AbstractSyntaxTree.hpp to src
Move DiagnosticLogger.hpp to src
Move File.hpp to src
Move openvic-dataloader/detail/utlity files to openvic-dataloader/detail

Add ovdl::utility::type_concat
Add ovdl::utility::type_prepend
Add ovdl::utility::is_instance_of
Overhaul parse error messages

2 files changed, 26 insertions, 32 deletions

diff --git a/include/openvic-dataloader/v2script/AbstractSyntaxTree.hpp b/include/openvic-dataloader/v2script/AbstractSyntaxTree.hpp
index 27dbfcb..29e7866 100644
--- a/include/openvic-dataloader/v2script/AbstractSyntaxTree.hpp
+++ b/include/openvic-dataloader/v2script/AbstractSyntaxTree.hpp
@@ -1,12 +1,10 @@
 #pragma once
 
-#include <cstdio>
 #include <string_view>
 
-#include <openvic-dataloader/AbstractSyntaxTree.hpp>
-#include <openvic-dataloader/File.hpp>
 #include <openvic-dataloader/NodeLocation.hpp>
-#include <openvic-dataloader/detail/LexyFwdDeclaration.hpp>
+#include <openvic-dataloader/detail/SymbolIntern.hpp>
+#include <openvic-dataloader/detail/Utility.hpp>
 
 #include <dryad/_detail/assert.hpp>
 #include <dryad/_detail/config.hpp>
@@ -82,37 +80,34 @@ namespace ovdl::v2script::ast {
 	};
 
 	struct FlatValue : dryad::abstract_node_range<Value, NodeKind::FirstFlatValue, NodeKind::LastFlatValue> {
-		AbstractSyntaxTree::symbol_type value() const {
+		SymbolIntern::symbol_type value() const {
 			return _value;
 		}
 
-		const char* value(const AbstractSyntaxTree::symbol_interner_type& symbols) const {
+		const char* value(const SymbolIntern::symbol_interner_type& symbols) const {
 			return _value.c_str(symbols);
 		}
 
 	protected:
-		explicit FlatValue(dryad::node_ctor ctor, NodeKind kind, AbstractSyntaxTree::symbol_type value)
+		explicit FlatValue(dryad::node_ctor ctor, NodeKind kind, SymbolIntern::symbol_type value)
 			: node_base(ctor, kind),
 			  _value(value) {}
 
 	protected:
-		AbstractSyntaxTree::symbol_type _value;
+		SymbolIntern::symbol_type _value;
 	};
 
 	struct IdentifierValue : dryad::basic_node<NodeKind::IdentifierValue, FlatValue> {
-		explicit IdentifierValue(dryad::node_ctor ctor, AbstractSyntaxTree::symbol_type value) : node_base(ctor, value) {}
+		explicit IdentifierValue(dryad::node_ctor ctor, SymbolIntern::symbol_type value) : node_base(ctor, value) {}
 	};
 
 	struct StringValue : dryad::basic_node<NodeKind::StringValue, FlatValue> {
-		explicit StringValue(dryad::node_ctor ctor, AbstractSyntaxTree::symbol_type value) : node_base(ctor, value) {}
+		explicit StringValue(dryad::node_ctor ctor, SymbolIntern::symbol_type value) : node_base(ctor, value) {}
 	};
 
 	struct ListValue : dryad::basic_node<NodeKind::ListValue, dryad::container_node<Value>> {
 		explicit ListValue(dryad::node_ctor ctor, StatementList statements);
-		explicit ListValue(dryad::node_ctor ctor, AssignStatementList statements)
-			: node_base(ctor) {
-			insert_child_list_after(nullptr, statements);
-		}
+		explicit ListValue(dryad::node_ctor ctor, AssignStatementList statements);
 
 		explicit ListValue(dryad::node_ctor ctor) : ListValue(ctor, StatementList {}) {
 		}
@@ -171,10 +166,7 @@ namespace ovdl::v2script::ast {
 
 	struct FileTree : dryad::basic_node<NodeKind::FileTree, dryad::container_node<Node>> {
 		explicit FileTree(dryad::node_ctor ctor, StatementList statements);
-		explicit FileTree(dryad::node_ctor ctor, AssignStatementList statements) : node_base(ctor) {
-			insert_child_list_after(nullptr, statements);
-		}
-
+		explicit FileTree(dryad::node_ctor ctor, AssignStatementList statements);
 		explicit FileTree(dryad::node_ctor ctor) : FileTree(ctor, StatementList {}) {
 		}
 
diff --git a/include/openvic-dataloader/v2script/Parser.hpp b/include/openvic-dataloader/v2script/Parser.hpp
index f9f0ce8..1f6b158 100644
--- a/include/openvic-dataloader/v2script/Parser.hpp
+++ b/include/openvic-dataloader/v2script/Parser.hpp
@@ -3,6 +3,7 @@
 #include <cstddef>
 #include <filesystem>
 #include <memory>
+#include <optional>
 #include <ostream>
 #include <string>
 #include <string_view>
@@ -10,8 +11,9 @@
 #include <openvic-dataloader/Error.hpp>
 #include <openvic-dataloader/NodeLocation.hpp>
 #include <openvic-dataloader/Parser.hpp>
-#include <openvic-dataloader/detail/utility/Concepts.hpp>
-#include <openvic-dataloader/detail/utility/ErrorRange.hpp>
+#include <openvic-dataloader/detail/Concepts.hpp>
+#include <openvic-dataloader/detail/Encoding.hpp>
+#include <openvic-dataloader/detail/ErrorRange.hpp>
 #include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp>
 
 #include <dryad/node.hpp>
@@ -25,20 +27,20 @@ namespace ovdl::v2script {
 		Parser();
 		Parser(std::basic_ostream<char>& error_stream);
 
-		static Parser from_buffer(const char* data, std::size_t size);
-		static Parser from_buffer(const char* start, const char* end);
-		static Parser from_string(const std::string_view string);
-		static Parser from_file(const char* path);
-		static Parser from_file(const std::filesystem::path& path);
+		static Parser from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		static Parser from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		static Parser from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		static Parser from_file(const char* path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		static Parser from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 
-		constexpr Parser& load_from_buffer(const char* data, std::size_t size);
-		constexpr Parser& load_from_buffer(const char* start, const char* end);
-		constexpr Parser& load_from_string(const std::string_view string);
-		Parser& load_from_file(const char* path);
-		Parser& load_from_file(const std::filesystem::path& path);
+		constexpr Parser& load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		constexpr Parser& load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		constexpr Parser& load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		Parser& load_from_file(const char* path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		Parser& load_from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 
-		constexpr Parser& load_from_file(const detail::HasCstr auto& path) {
-			return load_from_file(path.c_str());
+		constexpr Parser& load_from_file(const detail::HasCstr auto& path, std::optional<detail::Encoding> encoding_fallback = std::nullopt) {
+			return load_from_file(path.c_str(), encoding_fallback);
 		}
 
 		bool simple_parse();