Merge pull request #46 from OpenVicProject/fix/char-detection

Add runtime encoding detection and conversion 

1 files changed, 17 insertions, 24 deletions

diff --git a/include/openvic-dataloader/csv/Parser.hpp b/include/openvic-dataloader/csv/Parser.hpp
index 06e7251..35421c8 100644
--- a/include/openvic-dataloader/csv/Parser.hpp
+++ b/include/openvic-dataloader/csv/Parser.hpp
@@ -1,41 +1,38 @@
 #pragma once
 
 #include <filesystem>
+#include <optional>
 
 #include <openvic-dataloader/Error.hpp>
+#include <openvic-dataloader/NodeLocation.hpp>
 #include <openvic-dataloader/Parser.hpp>
 #include <openvic-dataloader/csv/LineObject.hpp>
-#include <openvic-dataloader/detail/utility/Concepts.hpp>
-#include <openvic-dataloader/detail/utility/ErrorRange.hpp>
+#include <openvic-dataloader/detail/Concepts.hpp>
+#include <openvic-dataloader/detail/Encoding.hpp>
+#include <openvic-dataloader/detail/ErrorRange.hpp>
 
 #include <dryad/node.hpp>
 
 namespace ovdl::csv {
-	enum class EncodingType {
-		Windows1252,
-		Utf8
-	};
-
-	template<EncodingType Encoding = EncodingType::Windows1252>
 	class Parser final : public detail::BasicParser {
 	public:
 		Parser();
 		Parser(std::basic_ostream<char>& error_stream);
 
-		static Parser from_buffer(const char* data, std::size_t size);
-		static Parser from_buffer(const char* start, const char* end);
-		static Parser from_string(const std::string_view string);
-		static Parser from_file(const char* path);
-		static Parser from_file(const std::filesystem::path& path);
+		static Parser from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		static Parser from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		static Parser from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		static Parser from_file(const char* path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		static Parser from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 
-		constexpr Parser& load_from_buffer(const char* data, std::size_t size);
-		constexpr Parser& load_from_buffer(const char* start, const char* end);
-		constexpr Parser& load_from_string(const std::string_view string);
-		Parser& load_from_file(const char* path);
-		Parser& load_from_file(const std::filesystem::path& path);
+		constexpr Parser& load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		constexpr Parser& load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		constexpr Parser& load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		Parser& load_from_file(const char* path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		Parser& load_from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 
-		constexpr Parser& load_from_file(const detail::HasCstr auto& path) {
-			return load_from_file(path.c_str());
+		constexpr Parser& load_from_file(const detail::HasCstr auto& path, std::optional<detail::Encoding> encoding_fallback = std::nullopt) {
+			return load_from_file(path.c_str(), encoding_fallback);
 		}
 
 		bool parse_csv(bool handle_strings = false);
@@ -57,12 +54,8 @@ namespace ovdl::csv {
 	private:
 		class ParseHandler;
 		std::unique_ptr<ParseHandler> _parse_handler;
-		std::vector<csv::LineObject> _lines;
 
 		template<typename... Args>
 		constexpr void _run_load_func(detail::LoadCallback<ParseHandler, Args...> auto func, Args... args);
 	};
-
-	using Windows1252Parser = Parser<EncodingType::Windows1252>;
-	using Utf8Parser = Parser<EncodingType::Utf8>;
 }
\ No newline at end of file