Merge pull request #11 from OpenVicProject/fix/more-csv-problems

9 files changed, 448 insertions, 55 deletions

diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 0000000..6ec9117
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,35 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "cppdbg",
+            "request": "launch",
+            "name": "Debug",
+            "program": "${workspaceFolder}/bin/openvic-dataloader.headless.linux.template_debug.dev.x86_64",
+            "args": ["csv", "./output.csv"],
+            "cwd": "${workspaceFolder}/bin",
+            "preLaunchTask": "dev_build",
+            "windows": {
+                "type": "cppvsdb",
+                "request": "launch",
+                "name": "Debug",
+                "program": "${workspaceFolder}\\bin\\openvic-dataloader.headless.windows.template_debug.dev.x86_64",
+                "args": ["csv", "./output.csv"],
+                "cwd": "${workspaceFolder}\\bin",
+                "preLaunchTask": "dev_build",
+            },
+            "osx": {
+                "type": "cppvsdb",
+                "request": "launch",
+                "name": "Debug",
+                "program": "", // TODO: Mac executable?
+                "args": [],
+                "cwd": "${workspaceFolder}\\game",
+                "preLaunchTask": "dev_build",
+            }
+        }
+    ]
+}
\ No newline at end of file
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
new file mode 100644
index 0000000..65e62ec
--- /dev/null
+++ b/.vscode/tasks.json
@@ -0,0 +1,53 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "build",
+            "group": "build",
+            "type": "shell",
+            "command": "scons",
+            "args": [
+                // enable for debugging with breakpoints
+                //"dev_build=yes",
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "dev_build",
+            "group": "build",
+            "type": "shell",
+            "command": "scons",
+            "args": [
+                // enable for debugging with breakpoints
+                "dev_build=yes",
+                "debug_symbols=yes"
+            ],
+            "dependsOn": [
+                "build"
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "clean",
+            "group": "build",
+            "type": "shell",
+            "command": "scons",
+            "args": [
+                "--clean"
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "rebuild",
+            "group": "build",
+            "type": "shell",
+            "dependsOn": [
+                "clean",
+                "build"
+            ],
+            "problemMatcher": "$msCompile"
+        }
+    ]
+}
\ No newline at end of file
diff --git a/include/openvic-dataloader/csv/LineObject.hpp b/include/openvic-dataloader/csv/LineObject.hpp
index 8a9c2ec..c6d8e2a 100644
--- a/include/openvic-dataloader/csv/LineObject.hpp
+++ b/include/openvic-dataloader/csv/LineObject.hpp
@@ -1,13 +1,17 @@
 #pragma once
 
+#include <algorithm>
+#include <cctype>
 #include <cstddef>
 #include <cstdint>
 #include <functional>
 #include <initializer_list>
 #include <optional>
+#include <ostream>
+#include <set>
 #include <string>
 #include <string_view>
-#include <tuple>
+#include <utility>
 #include <vector>
 
 #include <openvic-dataloader/detail/VectorConstexpr.hpp>
@@ -24,13 +28,13 @@ namespace ovdl::csv {
 	/// ;a;b;c -> 0,4+ == ""
 	///
 	/// If this is incorrect, please report an issue.
-	class LineObject final : public std::vector<std::tuple<std::uint32_t, std::string>> {
+	class LineObject final : public std::vector<std::pair<std::uint32_t, std::string>> {
 	public:
 		// Stored position of value
 		using position_type = std::uint32_t;
 		// Value
 		using inner_value_type = std::string;
-		using container_type = std::vector<std::tuple<position_type, inner_value_type>>;
+		using container_type = std::vector<std::pair<position_type, inner_value_type>>;
 
 		OVDL_VECTOR_CONSTEXPR LineObject() = default;
 		OVDL_VECTOR_CONSTEXPR LineObject(LineObject&) = default;
@@ -54,7 +58,7 @@ namespace ovdl::csv {
 		/// Special Functionality
 		/// Retrieves value, produces "" for empty values
 		constexpr std::string_view get_value_for(std::size_t position) const {
-			if (position <= _prefix_end || position >= _suffix_end) return "";
+			if (position < _prefix_end || position >= _suffix_end) return "";
 			for (const auto& [pos, val] : *this) {
 				if (pos == position) return val;
 			}
@@ -62,7 +66,7 @@ namespace ovdl::csv {
 		}
 		/// Tries to retrieve reference, produces nullopt for empty values
 		constexpr std::optional<const std::reference_wrapper<const std::string>> try_get_string_at(std::size_t position) const {
-			if (position <= _prefix_end || position > _suffix_end) return std::nullopt;
+			if (position < _prefix_end || position >= _suffix_end) return std::nullopt;
 			for (const auto& [pos, val] : *this) {
 				if (pos == position) return std::cref(val);
 			}
@@ -83,4 +87,28 @@ namespace ovdl::csv {
 		// Should be position after last value or position after last seperator
 		position_type _suffix_end = 0;
 	};
+
+	inline std::ostream& operator<<(std::ostream& stream, const LineObject& line) {
+		static const char SEP = ';';
+		LineObject::position_type sep_index = 0;
+		for (const auto& [pos, val] : line) {
+			while (sep_index < pos) {
+				stream << SEP;
+				sep_index++;
+			}
+			if (std::any_of(val.begin(), val.end(), [](char c) { return c == SEP || std::isspace(c); })) {
+				stream << '"' << val << '"';
+			} else {
+				stream << val;
+			}
+		}
+		return stream;
+	}
+
+	inline std::ostream& operator<<(std::ostream& stream, const std::vector<LineObject>& lines) {
+		for (const LineObject& line : lines) {
+			stream << line << '\n';
+		}
+		return stream;
+	}
 }
\ No newline at end of file
diff --git a/include/openvic-dataloader/csv/Parser.hpp b/include/openvic-dataloader/csv/Parser.hpp
index 0bd0670..fadaf3a 100644
--- a/include/openvic-dataloader/csv/Parser.hpp
+++ b/include/openvic-dataloader/csv/Parser.hpp
@@ -4,6 +4,12 @@
 #include <openvic-dataloader/detail/BasicParser.hpp>
 
 namespace ovdl::csv {
+	enum class EncodingType {
+		Windows1252,
+		Utf8
+	};
+
+	template<EncodingType Encoding = EncodingType::Windows1252>
 	class Parser final : public detail::BasicParser {
 	public:
 		Parser();
@@ -39,4 +45,7 @@ namespace ovdl::csv {
 		template<typename... Args>
 		constexpr void _run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args);
 	};
+
+	using Windows1252Parser = Parser<EncodingType::Windows1252>;
+	using Utf8Parser = Parser<EncodingType::Utf8>;
 }
\ No newline at end of file
diff --git a/include/openvic-dataloader/detail/ClassExport.hpp b/include/openvic-dataloader/detail/ClassExport.hpp
new file mode 100644
index 0000000..27098ed
--- /dev/null
+++ b/include/openvic-dataloader/detail/ClassExport.hpp
@@ -0,0 +1,9 @@
+#pragma once
+
+#ifdef _MSC_VER
+#define OVDL_EXPORT __declspec(dllexport)
+#elif defined(__GNUC__)
+#define OVDL_EXPORT __attribute__((visibility("default")))
+#else
+#define OVDL_EXPORT
+#endif
\ No newline at end of file
diff --git a/src/headless/main.cpp b/src/headless/main.cpp
index 94d6b8d..c3ca8c2 100644
--- a/src/headless/main.cpp
+++ b/src/headless/main.cpp
@@ -1,17 +1,71 @@
+#include <algorithm>
+#include <cctype>
 #include <cstdio>
+#include <cstdlib>
 #include <iostream>
+#include <iterator>
 #include <string>
+#include <string_view>
 #include <vector>
 
+#include <openvic-dataloader/ParseWarning.hpp>
+#include <openvic-dataloader/csv/LineObject.hpp>
+#include <openvic-dataloader/csv/Parser.hpp>
+#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp>
 #include <openvic-dataloader/v2script/Parser.hpp>
 
-int main(int argc, char** argv) {
-	if (argc < 2) {
-		std::fprintf(stderr, "usage: %s <filename>", argv[0]);
+std::string_view trim(std::string_view str) {
+	std::string_view::iterator begin = str.begin();
+	std::string_view::iterator end = str.end();
+	for (;; begin++) {
+		if (begin == end) return std::string_view();
+		if (!std::isspace(*begin)) break;
+	}
+	end--;
+	for (;; end--) {
+		if (end == begin) return std::string_view();
+		if (!std::isspace(*end)) break;
+	}
+	return std::string_view(&*begin, std::distance(begin, end));
+}
+
+bool insenitive_trim_eq(std::string_view lhs, std::string_view rhs) {
+	lhs = trim(lhs);
+	rhs = trim(rhs);
+	return std::equal(
+		lhs.begin(), lhs.end(),
+		rhs.begin(), rhs.end(),
+		[](char a, char b) { return std::tolower(a) == std::tolower(b); });
+}
+
+template<ovdl::csv::EncodingType Encoding>
+int print_csv(const std::string_view path) {
+	auto parser = ovdl::csv::Parser<Encoding>::from_file(path);
+	if (parser.has_error()) {
 		return 1;
 	}
 
-	auto parser = ovdl::v2script::Parser::from_file(argv[1]);
+	parser.parse_csv();
+	if (parser.has_error()) {
+		return 2;
+	}
+
+	if (parser.has_warning()) {
+		for (auto& warning : parser.get_warnings()) {
+			std::cerr << "Warning: " << warning.message << std::endl;
+		}
+	}
+
+	std::cout << "lines:\t\t" << parser.get_lines().size() << std::endl;
+	for (const auto& line : parser.get_lines()) {
+		std::cout << "line size:\t" << line.value_count() << std::endl;
+		std::cout << "values:\t\t" << line << std::endl;
+	}
+	return EXIT_SUCCESS;
+}
+
+int print_v2script_simple(const std::string_view path) {
+	auto parser = ovdl::v2script::Parser::from_file(path);
 	if (parser.has_error()) {
 		return 1;
 	}
@@ -28,6 +82,28 @@ int main(int argc, char** argv) {
 	}
 
 	std::cout << parser.get_file_node() << std::endl;
+	return EXIT_SUCCESS;
+}
+
+int main(int argc, char** argv) {
+	switch (argc) {
+		case 2:
+			return print_v2script_simple(argv[1]);
+		case 4:
+			if (insenitive_trim_eq(argv[1], "csv") && insenitive_trim_eq(argv[2], "utf"))
+				return print_csv<ovdl::csv::EncodingType::Utf8>(argv[3]);
+			goto default_jump;
+		case 3:
+			if (insenitive_trim_eq(argv[1], "csv"))
+				return print_csv<ovdl::csv::EncodingType::Windows1252>(argv[2]);
+			[[fallthrough]];
+		default:
+		default_jump:
+			std::fprintf(stderr, "usage: %s <filename>\n", argv[0]);
+			std::fprintf(stderr, "usage: %s csv <filename>\n", argv[0]);
+			std::fprintf(stderr, "usage: %s csv utf <filename>", argv[0]);
+			return EXIT_FAILURE;
+	}
 
 	return 0;
 }
\ No newline at end of file
diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp
index edce97b..4765112 100644
--- a/src/openvic-dataloader/csv/CsvGrammar.hpp
+++ b/src/openvic-dataloader/csv/CsvGrammar.hpp
@@ -11,8 +11,141 @@
 #include <lexy/callback.hpp>
 #include <lexy/dsl.hpp>
 
+#include "detail/LexyLitRange.hpp"
+
 // Grammar Definitions //
-namespace ovdl::csv::grammar {
+namespace ovdl::csv::grammar::windows1252 {
+	constexpr auto character = detail::lexydsl::make_range<0x01, 0xFF>();
+	constexpr auto control =
+		lexy::dsl::ascii::control /
+		lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> /
+		lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>;
+
+	struct StringValue {
+		static constexpr auto escaped_symbols = lexy::symbol_table<char> //
+													.map<'"'>('"')
+													.map<'\''>('\'')
+													.map<'\\'>('\\')
+													.map<'/'>('/')
+													.map<'b'>('\b')
+													.map<'f'>('\f')
+													.map<'n'>('\n')
+													.map<'r'>('\r')
+													.map<'t'>('\t');
+		/// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from ""
+		// static constexpr auto escaped_quote = lexy::symbol_table<char> //
+		// 										  .map<'"'>('"');
+		static constexpr auto rule = [] {
+			// Arbitrary code points
+			auto c = character - (control / lexy::dsl::lit_b<'"'>);
+
+			auto back_escape = lexy::dsl::backslash_escape //
+								   .symbol<escaped_symbols>();
+
+			// auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
+			// 						.symbol<escaped_quote>();
+
+			return lexy::dsl::quoted(c, back_escape);
+		}();
+
+		static constexpr auto value = lexy::as_string<std::string>;
+	};
+
+	template<char Sep>
+	struct PlainValue {
+		static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b<Sep> / lexy::dsl::ascii::newline));
+		static constexpr auto value = lexy::as_string<std::string>;
+	};
+
+	template<char Sep>
+	struct Value {
+		static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>;
+		static constexpr auto value = lexy::forward<std::string>;
+	};
+
+	template<char Sep>
+	struct SepConst {
+		static constexpr auto rule = lexy::dsl::lit_b<Sep>;
+		static constexpr auto value = lexy::constant(1);
+	};
+
+	template<char Sep>
+	struct Seperator {
+		static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Sep>>);
+		static constexpr auto value = lexy::count;
+	};
+
+	template<char Sep>
+	struct LineEnd {
+		static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Sep>>));
+		static constexpr auto value = lexy::fold_inplace<csv::LineObject>(
+			std::initializer_list<csv::LineObject::value_type> {},
+			[](csv::LineObject& result, auto&& arg) {
+				if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
+					// Count seperators, adds to previous value, making it a position
+					using position_type = csv::LineObject::position_type;
+					result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
+				} else {
+					if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
+					else {
+						auto& [pos, value] = result.back();
+						value = arg;
+					}
+				}
+			});
+	};
+
+	template<char Sep>
+	struct Line {
+
+		static constexpr auto suffix_setter(csv::LineObject& line) {
+			auto& [position, value] = line.back();
+			if (value.empty()) {
+				line.set_suffix_end(position);
+				line.pop_back();
+			} else {
+				line.set_suffix_end(position + 1);
+			}
+		};
+
+		static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<Seperator<Sep>> >> lexy::dsl::p<LineEnd<Sep>>;
+		static constexpr auto value =
+			lexy::callback<csv::LineObject>(
+				[](csv::LineObject&& line) {
+					suffix_setter(line);
+					return LEXY_MOV(line);
+				},
+				[](std::size_t prefix_count, csv::LineObject&& line) {
+					line.set_prefix_end(prefix_count);
+					// position needs to be adjusted to prefix
+					for (auto& [position, value] : line) {
+						position += prefix_count;
+					}
+					suffix_setter(line);
+					return LEXY_MOV(line);
+				});
+	};
+
+	template<char Sep>
+	struct File {
+		static constexpr auto rule =
+			lexy::dsl::whitespace(lexy::dsl::newline) +
+			lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p<Line<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::eol)));
+
+		static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>;
+	};
+
+	using CommaFile = File<','>;
+	using ColonFile = File<':'>;
+	using SemiColonFile = File<';'>;
+	using TabFile = File<'\t'>;
+	using BarFile = File<'|'>;
+}
+
+namespace ovdl::csv::grammar::utf8 {
+	constexpr auto character = lexy::dsl::unicode::character;
+	constexpr auto control = lexy::dsl::unicode::control;
+
 	struct StringValue {
 		static constexpr auto escaped_symbols = lexy::symbol_table<char> //
 													.map<'"'>('"')
@@ -25,52 +158,58 @@ namespace ovdl::csv::grammar {
 													.map<'r'>('\r')
 													.map<'t'>('\t');
 		/// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from ""
-		static constexpr auto escaped_quote = lexy::symbol_table<char> //
-												  .map<'"'>('"');
+		// static constexpr auto escaped_quote = lexy::symbol_table<char> //
+		// 										  .map<'"'>('"');
 		static constexpr auto rule = [] {
 			// Arbitrary code points
-			auto c = -lexy::dsl::lit_c<'"'>;
+			auto c = character - (control / lexy::dsl::lit_b<'"'>);
 
 			auto back_escape = lexy::dsl::backslash_escape //
 								   .symbol<escaped_symbols>();
 
-			auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
-									.symbol<escaped_quote>();
+			// auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
+			// 						.symbol<escaped_quote>();
 
-			return lexy::dsl::quoted(c, back_escape, quote_escape);
+			return lexy::dsl::quoted(c, back_escape);
 		}();
 
 		static constexpr auto value = lexy::as_string<std::string>;
 	};
 
-	template<auto Sep>
+	template<char Sep>
 	struct PlainValue {
-		static constexpr auto rule = lexy::dsl::identifier(-(Sep / lexy::dsl::lit_c<'\n'>));
+		static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b<Sep> / lexy::dsl::ascii::newline));
 		static constexpr auto value = lexy::as_string<std::string>;
 	};
 
-	template<auto Sep>
+	template<char Sep>
 	struct Value {
 		static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>;
 		static constexpr auto value = lexy::forward<std::string>;
 	};
 
-	template<auto Sep>
-	struct SeperatorCount {
-		static constexpr auto rule = lexy::dsl::list(Sep);
+	template<char Sep>
+	struct SepConst {
+		static constexpr auto rule = lexy::dsl::lit_b<Sep>;
+		static constexpr auto value = lexy::constant(1);
+	};
+
+	template<char Sep>
+	struct Seperator {
+		static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Sep>>);
 		static constexpr auto value = lexy::count;
 	};
 
-	template<auto Sep>
+	template<char Sep>
 	struct LineEnd {
-		static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<SeperatorCount<Sep>>));
+		static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Sep>>));
 		static constexpr auto value = lexy::fold_inplace<csv::LineObject>(
 			std::initializer_list<csv::LineObject::value_type> {},
 			[](csv::LineObject& result, auto&& arg) {
 				if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
 					// Count seperators, adds to previous value, making it a position
 					using position_type = csv::LineObject::position_type;
-					result.emplace_back(static_cast<position_type>(arg + std::get<0>(result.back())), "");
+					result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
 				} else {
 					if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
 					else {
@@ -81,7 +220,7 @@ namespace ovdl::csv::grammar {
 			});
 	};
 
-	template<auto Sep>
+	template<char Sep>
 	struct Line {
 
 		static constexpr auto suffix_setter(csv::LineObject& line) {
@@ -94,7 +233,7 @@ namespace ovdl::csv::grammar {
 			}
 		};
 
-		static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<SeperatorCount<Sep>> >> lexy::dsl::p<LineEnd<Sep>>;
+		static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<Seperator<Sep>> >> lexy::dsl::p<LineEnd<Sep>>;
 		static constexpr auto value =
 			lexy::callback<csv::LineObject>(
 				[](csv::LineObject&& line) {
@@ -112,7 +251,7 @@ namespace ovdl::csv::grammar {
 				});
 	};
 
-	template<auto Sep>
+	template<char Sep>
 	struct File {
 		static constexpr auto rule =
 			lexy::dsl::whitespace(lexy::dsl::newline) +
@@ -121,9 +260,9 @@ namespace ovdl::csv::grammar {
 		static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>;
 	};
 
-	using CommaFile = File<lexy::dsl::lit_c<','>>;
-	using ColonFile = File<lexy::dsl::lit_c<':'>>;
-	using SemiColonFile = File<lexy::dsl::lit_c<';'>>;
-	using TabFile = File<lexy::dsl::lit_c<'\t'>>;
-	using BarFile = File<lexy::dsl::lit_c<'|'>>;
+	using CommaFile = File<','>;
+	using ColonFile = File<':'>;
+	using SemiColonFile = File<';'>;
+	using TabFile = File<'\t'>;
+	using BarFile = File<'|'>;
 }
\ No newline at end of file
diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp
index 45f3142..86ad02e 100644
--- a/src/openvic-dataloader/csv/Parser.cpp
+++ b/src/openvic-dataloader/csv/Parser.cpp
@@ -3,6 +3,7 @@
 
 #include <openvic-dataloader/csv/LineObject.hpp>
 #include <openvic-dataloader/csv/Parser.hpp>
+#include <openvic-dataloader/detail/ClassExport.hpp>
 
 #include <lexy/action/parse.hpp>
 #include <lexy/encoding.hpp>
@@ -20,11 +21,26 @@ using namespace ovdl::csv;
 
 ///	BufferHandler ///
 
-class Parser::BufferHandler final : public detail::BasicBufferHandler<lexy::utf8_char_encoding> {
+template<EncodingType>
+struct LexyEncodingFrom {
+};
+
+template<>
+struct LexyEncodingFrom<EncodingType::Windows1252> {
+	using encoding = lexy::default_encoding;
+};
+
+template<>
+struct LexyEncodingFrom<EncodingType::Utf8> {
+	using encoding = lexy::utf8_char_encoding;
+};
+
+template<EncodingType Encoding>
+class Parser<Encoding>::BufferHandler final : public detail::BasicBufferHandler<typename LexyEncodingFrom<Encoding>::encoding> {
 public:
 	template<typename Node, typename ErrorCallback>
 	std::optional<std::vector<ParseError>> parse(const ErrorCallback& callback) {
-		auto result = lexy::parse<Node>(_buffer, callback);
+		auto result = lexy::parse<Node>(this->_buffer, callback);
 		if (!result) {
 			return result.errors();
 		}
@@ -42,36 +58,45 @@ private:
 
 /// BufferHandler ///
 
-Parser::Parser()
+template<EncodingType Encoding>
+Parser<Encoding>::Parser()
 	: _buffer_handler(std::make_unique<BufferHandler>()) {
 	set_error_log_to_stderr();
 }
 
-Parser::Parser(Parser&&) = default;
-Parser& Parser::operator=(Parser&&) = default;
-Parser::~Parser() = default;
+template<EncodingType Encoding>
+Parser<Encoding>::Parser(Parser&&) = default;
+template<EncodingType Encoding>
+Parser<Encoding>& Parser<Encoding>::operator=(Parser&&) = default;
+template<EncodingType Encoding>
+Parser<Encoding>::~Parser() = default;
 
-Parser Parser::from_buffer(const char* data, std::size_t size) {
+template<EncodingType Encoding>
+Parser<Encoding> Parser<Encoding>::from_buffer(const char* data, std::size_t size) {
 	Parser result;
 	return std::move(result.load_from_buffer(data, size));
 }
 
-Parser Parser::from_buffer(const char* start, const char* end) {
+template<EncodingType Encoding>
+Parser<Encoding> Parser<Encoding>::from_buffer(const char* start, const char* end) {
 	Parser result;
 	return std::move(result.load_from_buffer(start, end));
 }
 
-Parser Parser::from_string(const std::string_view string) {
+template<EncodingType Encoding>
+Parser<Encoding> Parser<Encoding>::from_string(const std::string_view string) {
 	Parser result;
 	return std::move(result.load_from_string(string));
 }
 
-Parser Parser::from_file(const char* path) {
+template<EncodingType Encoding>
+Parser<Encoding> Parser<Encoding>::from_file(const char* path) {
 	Parser result;
 	return std::move(result.load_from_file(path));
 }
 
-Parser Parser::from_file(const std::filesystem::path& path) {
+template<EncodingType Encoding>
+Parser<Encoding> Parser<Encoding>::from_file(const std::filesystem::path& path) {
 	Parser result;
 	return std::move(result.load_from_file(path));
 }
@@ -89,8 +114,9 @@ Parser Parser::from_file(const std::filesystem::path& path) {
 /// @param func
 /// @param args
 ///
+template<EncodingType Encoding>
 template<typename... Args>
-constexpr void Parser::_run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args) {
+constexpr void Parser<Encoding>::_run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args) {
 	_warnings.clear();
 	_errors.clear();
 	_has_fatal_error = false;
@@ -101,43 +127,55 @@ constexpr void Parser::_run_load_func(detail::LoadCallback<BufferHandler, Args..
 	}
 }
 
-constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size) {
+template<EncodingType Encoding>
+constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* data, std::size_t size) {
 	// Type can't be deduced?
 	_run_load_func(std::mem_fn(&BufferHandler::load_buffer_size), data, size);
 	return *this;
 }
 
-constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) {
+template<EncodingType Encoding>
+constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* start, const char* end) {
 	// Type can't be deduced?
 	_run_load_func(std::mem_fn(&BufferHandler::load_buffer), start, end);
 	return *this;
 }
 
-constexpr Parser& Parser::load_from_string(const std::string_view string) {
+template<EncodingType Encoding>
+constexpr Parser<Encoding>& Parser<Encoding>::load_from_string(const std::string_view string) {
 	return load_from_buffer(string.data(), string.size());
 }
 
-constexpr Parser& Parser::load_from_file(const char* path) {
+template<EncodingType Encoding>
+constexpr Parser<Encoding>& Parser<Encoding>::load_from_file(const char* path) {
 	_file_path = path;
 	// Type can be deduced??
 	_run_load_func(std::mem_fn(&BufferHandler::load_file), path);
 	return *this;
 }
 
-Parser& Parser::load_from_file(const std::filesystem::path& path) {
+template<EncodingType Encoding>
+Parser<Encoding>& Parser<Encoding>::load_from_file(const std::filesystem::path& path) {
 	return load_from_file(path.string().c_str());
 }
 
-constexpr Parser& Parser::load_from_file(const detail::Has_c_str auto& path) {
+template<EncodingType Encoding>
+constexpr Parser<Encoding>& Parser<Encoding>::load_from_file(const detail::Has_c_str auto& path) {
 	return load_from_file(path.c_str());
 }
 
-bool Parser::parse_csv() {
+template<EncodingType Encoding>
+bool Parser<Encoding>::parse_csv() {
 	if (!_buffer_handler->is_valid()) {
 		return false;
 	}
 
-	auto errors = _buffer_handler->parse<csv::grammar::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }));
+	std::optional<std::vector<ParseError>> errors;
+	if constexpr (Encoding == EncodingType::Windows1252) {
+		errors = _buffer_handler->template parse<csv::grammar::windows1252::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }));
+	} else {
+		errors = _buffer_handler->template parse<csv::grammar::utf8::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }));
+	}
 	if (errors) {
 		_errors.reserve(errors->size());
 		for (auto& err : errors.value()) {
@@ -150,6 +188,10 @@ bool Parser::parse_csv() {
 	return true;
 }
 
-const std::vector<csv::LineObject>& Parser::get_lines() const {
+template<EncodingType Encoding>
+const std::vector<csv::LineObject>& Parser<Encoding>::get_lines() const {
 	return _lines;
-}
\ No newline at end of file
+}
+
+template class ovdl::csv::Parser<EncodingType::Windows1252>;
+template class ovdl::csv::Parser<EncodingType::Utf8>;
\ No newline at end of file
diff --git a/src/openvic-dataloader/detail/BasicBufferHandler.hpp b/src/openvic-dataloader/detail/BasicBufferHandler.hpp
index 75ea8ee..5f4f82d 100644
--- a/src/openvic-dataloader/detail/BasicBufferHandler.hpp
+++ b/src/openvic-dataloader/detail/BasicBufferHandler.hpp
@@ -15,6 +15,8 @@ namespace ovdl::detail {
 	template<typename Encoding = lexy::default_encoding, typename MemoryResource = void>
 	class BasicBufferHandler {
 	public:
+		using encoding_type = Encoding;
+
 		OVDL_OPTIONAL_CONSTEXPR bool is_valid() const {
 			return _buffer.size() != 0;
 		}