Add runtime encoding detection and conversionfix/char-detection

	Win-1251/1252 detection is a reduced C++ version of https://github.com/hsivonen/chardetng
Add manually-specified encoding fallback
Add default system encoding fallback
Add error recovery to v2script
Add unknown encoding detection warning
Remove csv::Parser templating
Fix lua files dropping data

Update lexy to foonathan/lexy@1e5d99fa3826b1c3c8628d3a11117fb4fb4cc0d0

Remove exclusive reliance on lexy::default_encoding for v2script

Move internal concepts to src/openvic-detail/InternalConcepts.hpp
Move contents of DetectUtf8.hpp to src/detail/Detect.hpp
Move openvic-dataloader/AbstractSyntaxTree.hpp to src
Move DiagnosticLogger.hpp to src
Move File.hpp to src
Move openvic-dataloader/detail/utlity files to openvic-dataloader/detail

Add ovdl::utility::type_concat
Add ovdl::utility::type_prepend
Add ovdl::utility::is_instance_of
Overhaul parse error messages

1 files changed, 89 insertions, 0 deletions

diff --git a/src/openvic-dataloader/AbstractSyntaxTree.hpp b/src/openvic-dataloader/AbstractSyntaxTree.hpp
new file mode 100644
index 0000000..a5b8886
--- /dev/null
+++ b/src/openvic-dataloader/AbstractSyntaxTree.hpp
@@ -0,0 +1,89 @@
+#pragma once
+
+#include <concepts>
+#include <cstdio>
+#include <string_view>
+#include <utility>
+
+#include <openvic-dataloader/NodeLocation.hpp>
+#include <openvic-dataloader/detail/SymbolIntern.hpp>
+#include <openvic-dataloader/detail/Utility.hpp>
+
+#include <dryad/node.hpp>
+#include <dryad/node_map.hpp>
+#include <dryad/symbol.hpp>
+#include <dryad/tree.hpp>
+
+#include <fmt/core.h>
+
+#include "detail/InternalConcepts.hpp"
+
+namespace ovdl {
+	struct AbstractSyntaxTree : SymbolIntern {
+		symbol_type intern(const char* str, std::size_t length);
+		symbol_type intern(std::string_view str);
+		const char* intern_cstr(const char* str, std::size_t length);
+		const char* intern_cstr(std::string_view str);
+		symbol_interner_type& symbol_interner();
+		const symbol_interner_type& symbol_interner() const;
+
+	protected:
+		symbol_interner_type _symbol_interner;
+	};
+
+	template<detail::IsFile FileT, std::derived_from<typename FileT::node_type> RootNodeT>
+	struct BasicAbstractSyntaxTree : AbstractSyntaxTree {
+		using file_type = FileT;
+		using root_node_type = RootNodeT;
+		using node_type = typename file_type::node_type;
+
+		explicit BasicAbstractSyntaxTree(file_type&& file) : _file { std::move(file) } {}
+
+		template<typename Encoding, typename MemoryResource = void>
+		explicit BasicAbstractSyntaxTree(lexy::buffer<Encoding, MemoryResource>&& buffer) : _file { std::move(buffer) } {}
+
+		void set_location(const node_type* n, NodeLocation loc) {
+			_file.set_location(n, loc);
+		}
+
+		NodeLocation location_of(const node_type* n) const {
+			return _file.location_of(n);
+		}
+
+		root_node_type* root() {
+			return _tree.root();
+		}
+
+		const root_node_type* root() const {
+			return _tree.root();
+		}
+
+		file_type& file() {
+			return _file;
+		}
+
+		const file_type& file() const {
+			return _file;
+		}
+
+		template<typename T, typename... Args>
+		T* create(NodeLocation loc, Args&&... args) {
+			auto node = _tree.template create<T>(DRYAD_FWD(args)...);
+			set_location(node, loc);
+			return node;
+		}
+
+		template<typename T, typename... Args>
+		T* create(const char* begin, const char* end, Args&&... args) {
+			return create<T>(NodeLocation::make_from(begin, end), DRYAD_FWD(args)...);
+		}
+
+		void set_root(root_node_type* node) {
+			_tree.set_root(node);
+		}
+
+	protected:
+		dryad::tree<root_node_type> _tree;
+		file_type _file;
+	};
+}
\ No newline at end of file