Add runtime encoding detection and conversionfix/char-detection

	Win-1251/1252 detection is a reduced C++ version of https://github.com/hsivonen/chardetng
Add manually-specified encoding fallback
Add default system encoding fallback
Add error recovery to v2script
Add unknown encoding detection warning
Remove csv::Parser templating
Fix lua files dropping data

Update lexy to foonathan/lexy@1e5d99fa3826b1c3c8628d3a11117fb4fb4cc0d0

Remove exclusive reliance on lexy::default_encoding for v2script

Move internal concepts to src/openvic-detail/InternalConcepts.hpp
Move contents of DetectUtf8.hpp to src/detail/Detect.hpp
Move openvic-dataloader/AbstractSyntaxTree.hpp to src
Move DiagnosticLogger.hpp to src
Move File.hpp to src
Move openvic-dataloader/detail/utlity files to openvic-dataloader/detail

Add ovdl::utility::type_concat
Add ovdl::utility::type_prepend
Add ovdl::utility::is_instance_of
Overhaul parse error messages

1 files changed, 127 insertions, 0 deletions

diff --git a/src/openvic-dataloader/detail/InternalConcepts.hpp b/src/openvic-dataloader/detail/InternalConcepts.hpp
new file mode 100644
index 0000000..0c7913d
--- /dev/null
+++ b/src/openvic-dataloader/detail/InternalConcepts.hpp
@@ -0,0 +1,127 @@
+#pragma once
+
+#include <concepts>
+#include <utility>
+
+#include <openvic-dataloader/NodeLocation.hpp>
+#include <openvic-dataloader/detail/Encoding.hpp>
+#include <openvic-dataloader/detail/SymbolIntern.hpp>
+
+#include <lexy/encoding.hpp>
+#include <lexy/input/buffer.hpp>
+
+#include <fmt/core.h>
+
+#include <lexy_ext/report_error.hpp>
+
+namespace ovdl::detail {
+	template<typename T>
+	concept IsFile =
+		requires(T t, const typename T::node_type* node, NodeLocation location) {
+			typename T::node_type;
+			{ t.set_location(node, location) } -> std::same_as<void>;
+			{ t.location_of(node) } -> std::same_as<NodeLocation>;
+		};
+
+	template<typename T>
+	concept IsAst =
+		requires(
+			T t,
+			const T ct,
+			const typename T::node_type* node,
+			NodeLocation loc //
+		) {
+			requires IsFile<typename T::file_type>;
+			typename T::root_node_type;
+			typename T::node_type;
+			requires std::derived_from<typename T::root_node_type, typename T::node_type>;
+			{ t.set_location(node, loc) } -> std::same_as<void>;
+			{ t.location_of(node) } -> std::same_as<NodeLocation>;
+			{ t.root() } -> std::same_as<typename T::root_node_type*>;
+			{ ct.root() } -> std::same_as<const typename T::root_node_type*>;
+			{ t.file() } -> std::same_as<typename T::file_type&>;
+			{ ct.file() } -> std::same_as<const typename T::file_type&>;
+		};
+
+	template<typename T>
+	concept IsDiagnosticLogger = requires(
+		T t,
+		const T ct,
+		const char* str,
+		std::size_t length,
+		std::string_view sv,
+		lexy_ext::diagnostic_kind diag_kind //
+	) {
+		typename T::error_range;
+		typename T::Writer;
+		{ static_cast<bool>(ct) } -> std::same_as<bool>;
+		{ ct.errored() } -> std::same_as<bool>;
+		{ ct.warned() } -> std::same_as<bool>;
+		{ ct.get_errors() } -> std::same_as<typename T::error_range>;
+		{ t.intern(str, length) } -> std::same_as<ovdl::SymbolIntern::symbol_type>;
+		{ t.intern(sv) } -> std::same_as<ovdl::SymbolIntern::symbol_type>;
+		{ t.intern_cstr(str, length) } -> std::same_as<const char*>;
+		{ t.intern_cstr(sv) } -> std::same_as<const char*>;
+		{ t.symbol_interner() } -> std::same_as<SymbolIntern::symbol_interner_type&>;
+		{ ct.symbol_interner() } -> std::same_as<const SymbolIntern::symbol_interner_type&>;
+		{ t.error(std::declval<typename T::template format_str<>>()) } -> std::same_as<typename T::Writer>;
+		{ t.warning(std::declval<typename T::template format_str<>>()) } -> std::same_as<typename T::Writer>;
+		{ t.note(std::declval<typename T::template format_str<>>()) } -> std::same_as<typename T::Writer>;
+		{ t.info(std::declval<typename T::template format_str<>>()) } -> std::same_as<typename T::Writer>;
+		{ t.debug(std::declval<typename T::template format_str<>>()) } -> std::same_as<typename T::Writer>;
+		{ t.fixit(std::declval<typename T::template format_str<>>()) } -> std::same_as<typename T::Writer>;
+		{ t.help(std::declval<typename T::template format_str<>>()) } -> std::same_as<typename T::Writer>;
+		{ t.error(sv) } -> std::same_as<typename T::Writer>;
+		{ t.warning(sv) } -> std::same_as<typename T::Writer>;
+		{ t.note(sv) } -> std::same_as<typename T::Writer>;
+		{ t.info(sv) } -> std::same_as<typename T::Writer>;
+		{ t.debug(sv) } -> std::same_as<typename T::Writer>;
+		{ t.fixit(sv) } -> std::same_as<typename T::Writer>;
+		{ t.help(sv) } -> std::same_as<typename T::Writer>;
+		{ std::move(t.error_callback().sink()).finish() } -> std::same_as<std::size_t>;
+		{ t.log(diag_kind, std::declval<typename T::template format_str<>>()) } -> std::same_as<typename T::Writer>;
+	};
+
+	template<typename T>
+	concept IsParseState = requires(
+		T t,
+		const T ct,
+		typename T::ast_type::file_type&& file,
+		lexy::buffer<lexy::default_encoding>&& buffer,
+		ovdl::detail::Encoding encoding,
+		const char* path //
+	) {
+		requires IsAst<typename T::ast_type>;
+		requires IsDiagnosticLogger<typename T::diagnostic_logger_type>;
+		{ T { std::move(file), encoding } } -> std::same_as<T>;
+		{ T { std::move(buffer), encoding } } -> std::same_as<T>;
+		{ T { path, std::move(buffer), encoding } } -> std::same_as<T>;
+		{ t.ast() } -> std::same_as<typename T::ast_type&>;
+		{ ct.ast() } -> std::same_as<const typename T::ast_type&>;
+		{ t.logger() } -> std::same_as<typename T::diagnostic_logger_type&>;
+		{ ct.logger() } -> std::same_as<const typename T::diagnostic_logger_type&>;
+	};
+
+	template<typename T>
+	concept IsFileParseState = requires(
+		T t,
+		const T ct,
+		typename T::file_type&& file,
+		lexy::buffer<lexy::default_encoding>&& buffer,
+		ovdl::detail::Encoding encoding,
+		const char* path //
+	) {
+		requires IsFile<typename T::file_type>;
+		requires IsDiagnosticLogger<typename T::diagnostic_logger_type>;
+		{ T { std::move(file), encoding } } -> std::same_as<T>;
+		{ T { std::move(buffer), encoding } } -> std::same_as<T>;
+		{ T { path, std::move(buffer), encoding } } -> std::same_as<T>;
+		{ t.file() } -> std::same_as<typename T::file_type&>;
+		{ ct.file() } -> std::same_as<const typename T::file_type&>;
+		{ t.logger() } -> std::same_as<typename T::diagnostic_logger_type&>;
+		{ ct.logger() } -> std::same_as<const typename T::diagnostic_logger_type&>;
+	};
+
+	template<typename T>
+	concept IsStateType = IsParseState<T> || IsFileParseState<T>;
+}
\ No newline at end of file