Add runtime encoding detection and conversionfix/char-detection

	Win-1251/1252 detection is a reduced C++ version of https://github.com/hsivonen/chardetng
Add manually-specified encoding fallback
Add default system encoding fallback
Add error recovery to v2script
Add unknown encoding detection warning
Remove csv::Parser templating
Fix lua files dropping data

Update lexy to foonathan/lexy@1e5d99fa3826b1c3c8628d3a11117fb4fb4cc0d0

Remove exclusive reliance on lexy::default_encoding for v2script

Move internal concepts to src/openvic-detail/InternalConcepts.hpp
Move contents of DetectUtf8.hpp to src/detail/Detect.hpp
Move openvic-dataloader/AbstractSyntaxTree.hpp to src
Move DiagnosticLogger.hpp to src
Move File.hpp to src
Move openvic-dataloader/detail/utlity files to openvic-dataloader/detail

Add ovdl::utility::type_concat
Add ovdl::utility::type_prepend
Add ovdl::utility::is_instance_of
Overhaul parse error messages

1 files changed, 139 insertions, 0 deletions

diff --git a/src/openvic-dataloader/File.hpp b/src/openvic-dataloader/File.hpp
new file mode 100644
index 0000000..90fcb11
--- /dev/null
+++ b/src/openvic-dataloader/File.hpp
@@ -0,0 +1,139 @@
+#pragma once
+
+#include <cassert>
+#include <concepts> // IWYU pragma: keep
+#include <type_traits>
+#include <variant>
+
+#include <openvic-dataloader/NodeLocation.hpp>
+#include <openvic-dataloader/detail/Utility.hpp>
+
+#include <lexy/encoding.hpp>
+#include <lexy/input/buffer.hpp>
+
+#include <dryad/node_map.hpp>
+
+namespace ovdl {
+	struct File {
+		using buffer_ids = detail::TypeRegister<
+			lexy::buffer<lexy::default_encoding, void>,
+			lexy::buffer<lexy::utf8_char_encoding, void>,
+			lexy::buffer<lexy::utf8_encoding, void>,
+			lexy::buffer<lexy::utf16_encoding, void>,
+			lexy::buffer<lexy::utf32_encoding, void>,
+			lexy::buffer<lexy::byte_encoding, void>>;
+
+		explicit File(const char* path);
+
+		const char* path() const noexcept;
+
+		bool is_valid() const noexcept;
+
+		template<typename Encoding, typename MemoryResource = void>
+		constexpr bool is_buffer() const {
+			return buffer_ids::type_id<lexy::buffer<Encoding, MemoryResource>>() + 1 == _buffer.index();
+		}
+
+		template<typename Encoding, typename MemoryResource = void>
+		lexy::buffer<Encoding, MemoryResource>* try_get_buffer_as() {
+			return std::get_if<lexy::buffer<Encoding, MemoryResource>>(&_buffer);
+		}
+
+		template<typename Encoding, typename MemoryResource = void>
+		const lexy::buffer<Encoding, MemoryResource>* try_get_buffer_as() const {
+			return std::get_if<lexy::buffer<Encoding, MemoryResource>>(&_buffer);
+		}
+
+		template<typename Encoding, typename MemoryResource = void>
+		lexy::buffer<Encoding, MemoryResource>& get_buffer_as() {
+			assert((is_buffer<Encoding, MemoryResource>()));
+			return *std::get_if<lexy::buffer<Encoding, MemoryResource>>(&_buffer);
+		}
+
+		template<typename Encoding, typename MemoryResource = void>
+		const lexy::buffer<Encoding, MemoryResource>& get_buffer_as() const {
+			assert((is_buffer<Encoding, MemoryResource>()));
+			return *std::get_if<lexy::buffer<Encoding, MemoryResource>>(&_buffer);
+		}
+
+#define SWITCH_LIST \
+	X(1)            \
+	X(2)            \
+	X(3)            \
+	X(4)            \
+	X(5)            \
+	X(6)
+
+#define X(NUM) \
+	case NUM:  \
+		return visitor(std::get<NUM>(_buffer));
+
+		template<typename Visitor>
+		decltype(auto) visit_buffer(Visitor&& visitor) {
+			switch (_buffer.index()) {
+				SWITCH_LIST
+				default: ovdl::detail::unreachable();
+			}
+		}
+
+		template<typename Return, typename Visitor>
+		Return visit_buffer(Visitor&& visitor) {
+			switch (_buffer.index()) {
+				SWITCH_LIST
+				default: ovdl::detail::unreachable();
+			}
+		}
+
+		template<typename Visitor>
+		decltype(auto) visit_buffer(Visitor&& visitor) const {
+			switch (_buffer.index()) {
+				SWITCH_LIST
+				default: ovdl::detail::unreachable();
+			}
+		}
+
+		template<typename Return, typename Visitor>
+		Return visit_buffer(Visitor&& visitor) const {
+			switch (_buffer.index()) {
+				SWITCH_LIST
+				default: ovdl::detail::unreachable();
+			}
+		}
+#undef X
+#undef SWITCH_LIST
+
+	protected:
+		const char* _path;
+		detail::type_prepend_t<buffer_ids::variant_type, std::monostate> _buffer;
+	};
+
+	template<typename NodeT>
+	struct BasicFile : File {
+		using node_type = NodeT;
+
+		template<typename Encoding, typename MemoryResource = void>
+		explicit BasicFile(const char* path, lexy::buffer<Encoding, MemoryResource>&& buffer)
+			: File(path) {
+			_buffer = static_cast<std::remove_reference_t<decltype(buffer)>&&>(buffer);
+		}
+
+		template<typename Encoding, typename MemoryResource = void>
+		explicit BasicFile(lexy::buffer<Encoding, MemoryResource>&& buffer)
+			: File("") {
+			_buffer = static_cast<std::remove_reference_t<decltype(buffer)>&&>(buffer);
+		}
+
+		void set_location(const node_type* n, NodeLocation loc) {
+			_map.insert(n, loc);
+		}
+
+		NodeLocation location_of(const node_type* n) const {
+			auto result = _map.lookup(n);
+			DRYAD_ASSERT(result != nullptr, "every Node should have a NodeLocation");
+			return *result;
+		}
+
+	protected:
+		dryad::node_map<const node_type, NodeLocation> _map;
+	};
+}
\ No newline at end of file