Overhaul CSV parsing and align with Victoria 2 CSVs

Fix "" not being converted into " in Csv files
Move duplicated Grammar in CsvGrammar.hpp to Grammar.inc
Add ParseOptions to simplify template options
Add StringValue disable option:
	Inlines value escapes
Add strings namespace to grammars for `ParseOptions.SupportStrings = true`
Add string parse handling to parse_csv via argument

3 files changed, 183 insertions, 241 deletions

diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp
index 4765112..8f8da82 100644
--- a/src/openvic-dataloader/csv/CsvGrammar.hpp
+++ b/src/openvic-dataloader/csv/CsvGrammar.hpp
@@ -21,248 +21,12 @@ namespace ovdl::csv::grammar::windows1252 {
 		lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> /
 		lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>;
 
-	struct StringValue {
-		static constexpr auto escaped_symbols = lexy::symbol_table<char> //
-													.map<'"'>('"')
-													.map<'\''>('\'')
-													.map<'\\'>('\\')
-													.map<'/'>('/')
-													.map<'b'>('\b')
-													.map<'f'>('\f')
-													.map<'n'>('\n')
-													.map<'r'>('\r')
-													.map<'t'>('\t');
-		/// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from ""
-		// static constexpr auto escaped_quote = lexy::symbol_table<char> //
-		// 										  .map<'"'>('"');
-		static constexpr auto rule = [] {
-			// Arbitrary code points
-			auto c = character - (control / lexy::dsl::lit_b<'"'>);
-
-			auto back_escape = lexy::dsl::backslash_escape //
-								   .symbol<escaped_symbols>();
-
-			// auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
-			// 						.symbol<escaped_quote>();
-
-			return lexy::dsl::quoted(c, back_escape);
-		}();
-
-		static constexpr auto value = lexy::as_string<std::string>;
-	};
-
-	template<char Sep>
-	struct PlainValue {
-		static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b<Sep> / lexy::dsl::ascii::newline));
-		static constexpr auto value = lexy::as_string<std::string>;
-	};
-
-	template<char Sep>
-	struct Value {
-		static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>;
-		static constexpr auto value = lexy::forward<std::string>;
-	};
-
-	template<char Sep>
-	struct SepConst {
-		static constexpr auto rule = lexy::dsl::lit_b<Sep>;
-		static constexpr auto value = lexy::constant(1);
-	};
-
-	template<char Sep>
-	struct Seperator {
-		static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Sep>>);
-		static constexpr auto value = lexy::count;
-	};
-
-	template<char Sep>
-	struct LineEnd {
-		static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Sep>>));
-		static constexpr auto value = lexy::fold_inplace<csv::LineObject>(
-			std::initializer_list<csv::LineObject::value_type> {},
-			[](csv::LineObject& result, auto&& arg) {
-				if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
-					// Count seperators, adds to previous value, making it a position
-					using position_type = csv::LineObject::position_type;
-					result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
-				} else {
-					if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
-					else {
-						auto& [pos, value] = result.back();
-						value = arg;
-					}
-				}
-			});
-	};
-
-	template<char Sep>
-	struct Line {
-
-		static constexpr auto suffix_setter(csv::LineObject& line) {
-			auto& [position, value] = line.back();
-			if (value.empty()) {
-				line.set_suffix_end(position);
-				line.pop_back();
-			} else {
-				line.set_suffix_end(position + 1);
-			}
-		};
-
-		static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<Seperator<Sep>> >> lexy::dsl::p<LineEnd<Sep>>;
-		static constexpr auto value =
-			lexy::callback<csv::LineObject>(
-				[](csv::LineObject&& line) {
-					suffix_setter(line);
-					return LEXY_MOV(line);
-				},
-				[](std::size_t prefix_count, csv::LineObject&& line) {
-					line.set_prefix_end(prefix_count);
-					// position needs to be adjusted to prefix
-					for (auto& [position, value] : line) {
-						position += prefix_count;
-					}
-					suffix_setter(line);
-					return LEXY_MOV(line);
-				});
-	};
-
-	template<char Sep>
-	struct File {
-		static constexpr auto rule =
-			lexy::dsl::whitespace(lexy::dsl::newline) +
-			lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p<Line<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::eol)));
-
-		static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>;
-	};
-
-	using CommaFile = File<','>;
-	using ColonFile = File<':'>;
-	using SemiColonFile = File<';'>;
-	using TabFile = File<'\t'>;
-	using BarFile = File<'|'>;
+#include "Grammar.inc"
 }
 
 namespace ovdl::csv::grammar::utf8 {
 	constexpr auto character = lexy::dsl::unicode::character;
 	constexpr auto control = lexy::dsl::unicode::control;
 
-	struct StringValue {
-		static constexpr auto escaped_symbols = lexy::symbol_table<char> //
-													.map<'"'>('"')
-													.map<'\''>('\'')
-													.map<'\\'>('\\')
-													.map<'/'>('/')
-													.map<'b'>('\b')
-													.map<'f'>('\f')
-													.map<'n'>('\n')
-													.map<'r'>('\r')
-													.map<'t'>('\t');
-		/// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from ""
-		// static constexpr auto escaped_quote = lexy::symbol_table<char> //
-		// 										  .map<'"'>('"');
-		static constexpr auto rule = [] {
-			// Arbitrary code points
-			auto c = character - (control / lexy::dsl::lit_b<'"'>);
-
-			auto back_escape = lexy::dsl::backslash_escape //
-								   .symbol<escaped_symbols>();
-
-			// auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
-			// 						.symbol<escaped_quote>();
-
-			return lexy::dsl::quoted(c, back_escape);
-		}();
-
-		static constexpr auto value = lexy::as_string<std::string>;
-	};
-
-	template<char Sep>
-	struct PlainValue {
-		static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b<Sep> / lexy::dsl::ascii::newline));
-		static constexpr auto value = lexy::as_string<std::string>;
-	};
-
-	template<char Sep>
-	struct Value {
-		static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>;
-		static constexpr auto value = lexy::forward<std::string>;
-	};
-
-	template<char Sep>
-	struct SepConst {
-		static constexpr auto rule = lexy::dsl::lit_b<Sep>;
-		static constexpr auto value = lexy::constant(1);
-	};
-
-	template<char Sep>
-	struct Seperator {
-		static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Sep>>);
-		static constexpr auto value = lexy::count;
-	};
-
-	template<char Sep>
-	struct LineEnd {
-		static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Sep>>));
-		static constexpr auto value = lexy::fold_inplace<csv::LineObject>(
-			std::initializer_list<csv::LineObject::value_type> {},
-			[](csv::LineObject& result, auto&& arg) {
-				if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
-					// Count seperators, adds to previous value, making it a position
-					using position_type = csv::LineObject::position_type;
-					result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
-				} else {
-					if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
-					else {
-						auto& [pos, value] = result.back();
-						value = arg;
-					}
-				}
-			});
-	};
-
-	template<char Sep>
-	struct Line {
-
-		static constexpr auto suffix_setter(csv::LineObject& line) {
-			auto& [position, value] = line.back();
-			if (value.empty()) {
-				line.set_suffix_end(position);
-				line.pop_back();
-			} else {
-				line.set_suffix_end(position + 1);
-			}
-		};
-
-		static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<Seperator<Sep>> >> lexy::dsl::p<LineEnd<Sep>>;
-		static constexpr auto value =
-			lexy::callback<csv::LineObject>(
-				[](csv::LineObject&& line) {
-					suffix_setter(line);
-					return LEXY_MOV(line);
-				},
-				[](std::size_t prefix_count, csv::LineObject&& line) {
-					line.set_prefix_end(prefix_count);
-					// position needs to be adjusted to prefix
-					for (auto& [position, value] : line) {
-						position += prefix_count;
-					}
-					suffix_setter(line);
-					return LEXY_MOV(line);
-				});
-	};
-
-	template<char Sep>
-	struct File {
-		static constexpr auto rule =
-			lexy::dsl::whitespace(lexy::dsl::newline) +
-			lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p<Line<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::eol)));
-
-		static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>;
-	};
-
-	using CommaFile = File<','>;
-	using ColonFile = File<':'>;
-	using SemiColonFile = File<';'>;
-	using TabFile = File<'\t'>;
-	using BarFile = File<'|'>;
+#include "Grammar.inc"
 }
\ No newline at end of file
diff --git a/src/openvic-dataloader/csv/Grammar.inc b/src/openvic-dataloader/csv/Grammar.inc
new file mode 100644
index 0000000..ed7f455
--- /dev/null
+++ b/src/openvic-dataloader/csv/Grammar.inc
@@ -0,0 +1,171 @@
+// This is designed to be reused in CsvGrammar.hpp multiple times
+// Please ignore undeclared variable use for character and control, this is intended
+// Sorry was the cleanest way I could think of to prevent code duplication
+
+// Includes to keep file errors small
+#include <initializer_list>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <vector>
+
+#include <openvic-dataloader/csv/LineObject.hpp>
+
+#include <lexy/callback.hpp>
+#include <lexy/dsl.hpp>
+
+#include "detail/LexyLitRange.hpp"
+
+struct ParseOptions {
+	/// @brief Seperator character
+	char SepChar;
+	/// @brief Determines whether StringValue is supported
+	bool SupportStrings;
+};
+
+struct StringValue {
+	static constexpr auto escaped_symbols = lexy::symbol_table<char> //
+												.map<'"'>('"')
+												.map<'\''>('\'')
+												.map<'\\'>('\\')
+												.map<'/'>('/')
+												.map<'b'>('\b')
+												.map<'f'>('\f')
+												.map<'n'>('\n')
+												.map<'r'>('\r')
+												.map<'t'>('\t');
+	/// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from ""
+	static constexpr auto escaped_quote = lexy::symbol_table<char> //
+											  .map<'"'>('"');
+
+	static constexpr auto rule = [] {
+		// Arbitrary code points
+		auto c = character - control;
+
+		auto back_escape = lexy::dsl::backslash_escape //
+							   .symbol<escaped_symbols>();
+
+		auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
+								.symbol<escaped_quote>();
+
+		return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape);
+	}();
+
+	static constexpr auto value = lexy::as_string<std::string>;
+};
+
+template<ParseOptions Options>
+struct PlainValue {
+	static constexpr auto rule = [] {
+		if constexpr (Options.SupportStrings) {
+			return lexy::dsl::identifier(character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline));
+		} else {
+			auto escape_check_char = character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline);
+			auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>;
+			auto id_segment = lexy::dsl::identifier(id_check_char);
+			auto escape_segement = lexy::dsl::token(escape_check_char);
+			auto escape_sym = lexy::dsl::symbol<StringValue::escaped_symbols>(escape_segement);
+			auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym;
+			return lexy::dsl::list(id_segment | escape_rule);
+		}
+	}();
+	static constexpr auto value = lexy::as_string<std::string>;
+};
+
+template<ParseOptions Options>
+struct Value {
+	static constexpr auto rule = [] {
+		if constexpr (Options.SupportStrings) {
+			return lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Options>>;
+		} else {
+			return lexy::dsl::p<PlainValue<Options>>;
+		}
+	}();
+	static constexpr auto value = lexy::forward<std::string>;
+};
+
+template<ParseOptions Options>
+struct SepConst {
+	static constexpr auto rule = lexy::dsl::lit_b<Options.SepChar>;
+	static constexpr auto value = lexy::constant(1);
+};
+
+template<ParseOptions Options>
+struct Seperator {
+	static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Options>>);
+	static constexpr auto value = lexy::count;
+};
+
+template<ParseOptions Options>
+struct LineEnd {
+	static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Options>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Options>>));
+	static constexpr auto value = lexy::fold_inplace<ovdl::csv::LineObject>(
+		std::initializer_list<ovdl::csv::LineObject::value_type> {},
+		[](ovdl::csv::LineObject& result, auto&& arg) {
+			if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
+				// Count seperators, adds to previous value, making it a position
+				using position_type = ovdl::csv::LineObject::position_type;
+				result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
+			} else {
+				if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
+				else {
+					auto& [pos, value] = result.back();
+					value = arg;
+				}
+			}
+		});
+};
+
+template<ParseOptions Options>
+struct Line {
+
+	static constexpr auto suffix_setter(ovdl::csv::LineObject& line) {
+		auto& [position, value] = line.back();
+		if (value.empty()) {
+			line.set_suffix_end(position);
+			line.pop_back();
+		} else {
+			line.set_suffix_end(position + 1);
+		}
+	};
+
+	static constexpr auto rule = lexy::dsl::p<LineEnd<Options>> | lexy::dsl::p<Seperator<Options>> >> lexy::dsl::p<LineEnd<Options>>;
+	static constexpr auto value =
+		lexy::callback<ovdl::csv::LineObject>(
+			[](ovdl::csv::LineObject&& line) {
+				suffix_setter(line);
+				return LEXY_MOV(line);
+			},
+			[](std::size_t prefix_count, ovdl::csv::LineObject&& line) {
+				line.set_prefix_end(prefix_count);
+				// position needs to be adjusted to prefix
+				for (auto& [position, value] : line) {
+					position += prefix_count;
+				}
+				suffix_setter(line);
+				return LEXY_MOV(line);
+			});
+};
+
+template<ParseOptions Options>
+struct File {
+	static constexpr auto rule =
+		lexy::dsl::whitespace(lexy::dsl::newline) +
+		lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p<Line<Options>>, lexy::dsl::trailing_sep(lexy::dsl::eol)));
+
+	static constexpr auto value = lexy::as_list<std::vector<ovdl::csv::LineObject>>;
+};
+
+using CommaFile = File<ParseOptions { ',' }>;
+using ColonFile = File<ParseOptions { ':' }>;
+using SemiColonFile = File<ParseOptions { ';' }>;
+using TabFile = File<ParseOptions { '\t' }>;
+using BarFile = File<ParseOptions { '|' }>;
+
+namespace strings {
+	using CommaFile = File<ParseOptions { ',', true }>;
+	using ColonFile = File<ParseOptions { ':', true }>;
+	using SemiColonFile = File<ParseOptions { ';', true }>;
+	using TabFile = File<ParseOptions { '\t', true }>;
+	using BarFile = File<ParseOptions { '|', true }>;
+}
\ No newline at end of file
diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp
index 86ad02e..14ef553 100644
--- a/src/openvic-dataloader/csv/Parser.cpp
+++ b/src/openvic-dataloader/csv/Parser.cpp
@@ -165,16 +165,23 @@ constexpr Parser<Encoding>& Parser<Encoding>::load_from_file(const detail::Has_c
 }
 
 template<EncodingType Encoding>
-bool Parser<Encoding>::parse_csv() {
+bool Parser<Encoding>::parse_csv(bool handle_strings) {
 	if (!_buffer_handler->is_valid()) {
 		return false;
 	}
 
 	std::optional<std::vector<ParseError>> errors;
+	auto report_error = ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream });
 	if constexpr (Encoding == EncodingType::Windows1252) {
-		errors = _buffer_handler->template parse<csv::grammar::windows1252::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }));
+		if (handle_strings)
+			errors = _buffer_handler->template parse<csv::grammar::windows1252::strings::SemiColonFile>(report_error);
+		else
+			errors = _buffer_handler->template parse<csv::grammar::windows1252::SemiColonFile>(report_error);
 	} else {
-		errors = _buffer_handler->template parse<csv::grammar::utf8::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }));
+		if (handle_strings)
+			errors = _buffer_handler->template parse<csv::grammar::utf8::strings::SemiColonFile>(report_error);
+		else
+			errors = _buffer_handler->template parse<csv::grammar::utf8::SemiColonFile>(report_error);
 	}
 	if (errors) {
 		_errors.reserve(errors->size());