Add runtime encoding detection and conversionfix/char-detection

	Win-1251/1252 detection is a reduced C++ version of https://github.com/hsivonen/chardetng
Add manually-specified encoding fallback
Add default system encoding fallback
Add error recovery to v2script
Add unknown encoding detection warning
Remove csv::Parser templating
Fix lua files dropping data

Update lexy to foonathan/lexy@1e5d99fa3826b1c3c8628d3a11117fb4fb4cc0d0

Remove exclusive reliance on lexy::default_encoding for v2script

Move internal concepts to src/openvic-detail/InternalConcepts.hpp
Move contents of DetectUtf8.hpp to src/detail/Detect.hpp
Move openvic-dataloader/AbstractSyntaxTree.hpp to src
Move DiagnosticLogger.hpp to src
Move File.hpp to src
Move openvic-dataloader/detail/utlity files to openvic-dataloader/detail

Add ovdl::utility::type_concat
Add ovdl::utility::type_prepend
Add ovdl::utility::is_instance_of
Overhaul parse error messages

1 files changed, 105 insertions, 0 deletions

diff --git a/src/openvic-dataloader/ParseState.hpp b/src/openvic-dataloader/ParseState.hpp
new file mode 100644
index 0000000..806829c
--- /dev/null
+++ b/src/openvic-dataloader/ParseState.hpp
@@ -0,0 +1,105 @@
+#pragma once
+
+#include <utility>
+
+#include <openvic-dataloader/detail/Encoding.hpp>
+
+#include <lexy/encoding.hpp>
+#include <lexy/input/buffer.hpp>
+
+#include <dryad/tree.hpp>
+
+#include "DiagnosticLogger.hpp"
+#include "detail/InternalConcepts.hpp"
+
+namespace ovdl {
+	struct BasicParseState {
+		explicit BasicParseState(detail::Encoding encoding = detail::Encoding::Unknown) : _encoding(encoding) {}
+
+		detail::Encoding encoding() const {
+			return _encoding;
+		}
+
+	protected:
+		detail::Encoding _encoding;
+	};
+
+	template<detail::IsAst AstT>
+	struct ParseState : BasicParseState {
+		using ast_type = AstT;
+		using file_type = typename ast_type::file_type;
+		using diagnostic_logger_type = BasicDiagnosticLogger<ParseState>;
+
+		ParseState(typename ast_type::file_type&& file, detail::Encoding encoding)
+			: _ast { std::move(file) },
+			  _logger { this->ast().file() },
+			  BasicParseState(encoding) {}
+
+		template<typename Encoding, typename MemoryResource = void>
+		ParseState(lexy::buffer<Encoding, MemoryResource>&& buffer, detail::Encoding encoding)
+			: ParseState(typename ast_type::file_type { std::move(buffer) }, encoding) {}
+
+		template<typename Encoding, typename MemoryResource = void>
+		ParseState(const char* path, lexy::buffer<Encoding, MemoryResource>&& buffer, detail::Encoding encoding)
+			: ParseState(typename ast_type::file_type { path, std::move(buffer) }, encoding) {}
+
+		ast_type& ast() {
+			return _ast;
+		}
+
+		const ast_type& ast() const {
+			return _ast;
+		}
+
+		diagnostic_logger_type& logger() {
+			return _logger;
+		}
+
+		const diagnostic_logger_type& logger() const {
+			return _logger;
+		}
+
+	private:
+		ast_type _ast;
+		diagnostic_logger_type _logger;
+	};
+
+	template<detail::IsFile FileT>
+	struct FileParseState : BasicParseState {
+		using file_type = FileT;
+		using diagnostic_logger_type = BasicDiagnosticLogger<FileParseState>;
+
+		FileParseState(file_type&& file, detail::Encoding encoding)
+			: _file { std::move(file) },
+			  _logger { this->file() },
+			  BasicParseState(encoding) {}
+
+		template<typename Encoding, typename MemoryResource = void>
+		FileParseState(lexy::buffer<Encoding, MemoryResource>&& buffer, detail::Encoding encoding)
+			: FileParseState(file_type { std::move(buffer) }, encoding) {}
+
+		template<typename Encoding, typename MemoryResource = void>
+		FileParseState(const char* path, lexy::buffer<Encoding, MemoryResource>&& buffer, detail::Encoding encoding)
+			: FileParseState(file_type { path, std::move(buffer) }, encoding) {}
+
+		file_type& file() {
+			return _file;
+		}
+
+		const file_type& file() const {
+			return _file;
+		}
+
+		diagnostic_logger_type& logger() {
+			return _logger;
+		}
+
+		const diagnostic_logger_type& logger() const {
+			return _logger;
+		}
+
+	private:
+		file_type _file;
+		diagnostic_logger_type _logger;
+	};
+}
\ No newline at end of file