Add `\x8F` to `Ę` conversion for Windows-1252fix/polish-todos

	To support special vanilla Polish TODOs that break utf8

Add CSV Win1252->Utf8 conversion tests

Fix map_value warning not triggering
Optimize for ascii characters

2 files changed, 215 insertions, 0 deletions

diff --git a/tests/SCsub b/tests/SCsub
index 0a18777..49e2742 100644
--- a/tests/SCsub
+++ b/tests/SCsub
@@ -47,6 +47,11 @@ tests_env.tests_sources = env.GlobRecursive("*.cpp", [source_path])
 
 SConscript("deps/SCsub", {"env": tests_env })
 
+# Blame Ubuntu 22's GCC-12 distribution for this crap
+# Compiler bug hangs if it can see if there is any reference to \x8F in a character
+if env["ubuntu_gcc_invalid_char_hang_bug"]:
+    tests_env.Append(CPPDEFINES=["_OVDL_TEST_UBUNTU_GCC_12_BUG_"])
+
 tests_program = tests_env.UnitTest(
     source=tests_env.tests_sources,
     target=os.path.join(BINDIR, tests_name),
diff --git a/tests/src/csv/Parser.cpp b/tests/src/csv/Parser.cpp
index e72c02a..fe26726 100644
--- a/tests/src/csv/Parser.cpp
+++ b/tests/src/csv/Parser.cpp
@@ -5,11 +5,14 @@
 #include <openvic-dataloader/csv/LineObject.hpp>
 #include <openvic-dataloader/csv/Parser.hpp>
 
+#include <fmt/core.h>
+
 #include "Helper.hpp"
 #include <detail/NullBuff.hpp>
 #include <range/v3/range/primitives.hpp>
 #include <range/v3/view/enumerate.hpp>
 #include <range/v3/view/iota.hpp>
+#include <range/v3/view/join.hpp>
 #include <snitch/snitch.hpp>
 
 using namespace ovdl;
@@ -568,4 +571,211 @@ TEST_CASE("CSV Parse", "[csv-parse]") {
 			}
 		}
 	}
+
+	SECTION("Score militaire;Militär;;Puntuación militar") {
+		static constexpr auto buffer = "Score militaire;Militär;;Puntuación militar"sv;
+		parser.load_from_string(buffer);
+
+		CHECK_PARSE();
+
+		const std::vector<LineObject>& line_list = parser.get_lines();
+		CHECK_FALSE(line_list.empty());
+		CHECK(ranges::size(line_list) == 1);
+
+		const LineObject& line = line_list.front();
+		CHECK_FALSE(line.empty());
+		CHECK(ranges::size(line) == 3);
+		CHECK(line.value_count() == 4);
+		CHECK(line.prefix_end() == 0);
+		CHECK(line.suffix_end() == 4);
+
+		for (const auto [index, val] : line | ranges::views::enumerate) {
+			CAPTURE(index);
+			CHECK_FALSE_OR_CONTINUE(val.second.empty());
+			switch (index) {
+				case 0:
+					CHECK_OR_CONTINUE(val.first == 0);
+					CHECK_OR_CONTINUE(val.second == "Score militaire"sv);
+					break;
+				case 1:
+					CHECK_OR_CONTINUE(val.first == 1);
+					CHECK_OR_CONTINUE(val.second == "Militär"sv);
+					break;
+				case 2:
+					CHECK_OR_CONTINUE(val.first == 3);
+					CHECK_OR_CONTINUE(val.second == "Puntuación militar"sv);
+					break;
+				default: CHECK_OR_CONTINUE(false); break;
+			}
+		}
+
+		CHECK(line.value_count() == 4);
+
+		for (const auto index : ranges::views::iota(size_t(0), line.value_count())) {
+			CAPTURE(index);
+			switch (index) {
+				case 0: CHECK_OR_CONTINUE(line.get_value_for(index) == "Score militaire"sv); break;
+				case 1: CHECK_OR_CONTINUE(line.get_value_for(index) == "Militär"sv); break;
+				case 2: CHECK_OR_CONTINUE(line.get_value_for(index) == ""sv); break;
+				case 3: CHECK_OR_CONTINUE(line.get_value_for(index) == "Puntuación militar"sv); break;
+				default: CHECK_OR_CONTINUE(false); break;
+			}
+		}
+	}
+
+	SECTION(";§RNo research set§W;§RAucune recherche définie§W;") {
+		static constexpr auto buffer = ";§RNo research set§W;§RAucune recherche définie§W;"sv;
+		parser.load_from_string(buffer);
+
+		CHECK_PARSE();
+
+		const std::vector<LineObject>& line_list = parser.get_lines();
+		CHECK_FALSE(line_list.empty());
+		CHECK(ranges::size(line_list) == 1);
+
+		const LineObject& line = line_list.front();
+		CHECK_FALSE(line.empty());
+		CHECK(ranges::size(line) == 2);
+		CHECK(line.value_count() == 3);
+		CHECK(line.prefix_end() == 1);
+		CHECK(line.suffix_end() == 3);
+
+		for (const auto [index, val] : line | ranges::views::enumerate) {
+			CAPTURE(index);
+			CHECK_FALSE_OR_CONTINUE(val.second.empty());
+			switch (index) {
+				case 0:
+					CHECK_OR_CONTINUE(val.first == 1);
+					CHECK_OR_CONTINUE(val.second == "§RNo research set§W"sv);
+					break;
+				case 1:
+					CHECK_OR_CONTINUE(val.first == 2);
+					CHECK_OR_CONTINUE(val.second == "§RAucune recherche définie§W"sv);
+					break;
+				default: CHECK_OR_CONTINUE(false); break;
+			}
+		}
+
+		CHECK(line.value_count() == 3);
+
+		for (const auto index : ranges::views::iota(size_t(0), line.value_count())) {
+			CAPTURE(index);
+			switch (index) {
+				case 0: CHECK_OR_CONTINUE(line.get_value_for(index) == ""sv); break;
+				case 1: CHECK_OR_CONTINUE(line.get_value_for(index) == "§RNo research set§W"sv); break;
+				case 2: CHECK_OR_CONTINUE(line.get_value_for(index) == "§RAucune recherche définie§W"sv); break;
+				default: CHECK_OR_CONTINUE(false); break;
+			}
+		}
+	}
+
+	SECTION("Württemberg;Wurtemberg;Württemberg;;Württemberg;") {
+		static constexpr auto buffer = "Württemberg;Wurtemberg;Württemberg;;Württemberg;"sv;
+		parser.load_from_string(buffer);
+
+		CHECK_PARSE();
+
+		const std::vector<LineObject>& line_list = parser.get_lines();
+		CHECK_FALSE(line_list.empty());
+		CHECK(ranges::size(line_list) == 1);
+
+		const LineObject& line = line_list.front();
+		CHECK_FALSE(line.empty());
+		CHECK(ranges::size(line) == 4);
+		CHECK(line.value_count() == 5);
+		CHECK(line.prefix_end() == 0);
+		CHECK(line.suffix_end() == 5);
+
+		for (const auto [index, val] : line | ranges::views::enumerate) {
+			CAPTURE(index);
+			CHECK_FALSE_OR_CONTINUE(val.second.empty());
+			switch (index) {
+				case 0:
+					CHECK_OR_CONTINUE(val.first == 0);
+					CHECK_OR_CONTINUE(val.second == "Württemberg"sv);
+					break;
+				case 1:
+					CHECK_OR_CONTINUE(val.first == 1);
+					CHECK_OR_CONTINUE(val.second == "Wurtemberg"sv);
+					break;
+				case 2:
+					CHECK_OR_CONTINUE(val.first == 2);
+					CHECK_OR_CONTINUE(val.second == "Württemberg"sv);
+					break;
+				case 3:
+					CHECK_OR_CONTINUE(val.first == 4);
+					CHECK_OR_CONTINUE(val.second == "Württemberg"sv);
+					break;
+				default: CHECK_OR_CONTINUE(false); break;
+			}
+		}
+
+		CHECK(line.value_count() == 5);
+
+		for (const auto index : ranges::views::iota(size_t(0), line.value_count())) {
+			CAPTURE(index);
+			switch (index) {
+				case 0: CHECK_OR_CONTINUE(line.get_value_for(index) == "Württemberg"sv); break;
+				case 1: CHECK_OR_CONTINUE(line.get_value_for(index) == "Wurtemberg"sv); break;
+				case 2: CHECK_OR_CONTINUE(line.get_value_for(index) == "Württemberg"sv); break;
+				case 3: CHECK_OR_CONTINUE(line.get_value_for(index) == ""sv); break;
+				case 4: CHECK_OR_CONTINUE(line.get_value_for(index) == "Württemberg"sv); break;
+				default: CHECK_OR_CONTINUE(false); break;
+			}
+		}
+	}
+
+	// Blame Ubuntu 22's GCC-12 distribution for this crap
+	// Compiler bug hangs if it can see if there is any reference to \x8F in a character
+#if !defined(_OVDL_TEST_UBUNTU_GCC_12_BUG_)
+	SECTION(";$NAME$ wurde in $PROV$ gebaut.;ID'\\x8F' DO;") {
+		static auto buffer = ";$NAME$ wurde in $PROV$ gebaut.;ID\x8F DO;";
+		parser.load_from_string(buffer);
+
+		CHECK_PARSE();
+
+		const std::vector<LineObject>& line_list = parser.get_lines();
+		CHECK_FALSE(line_list.empty());
+		CHECK(ranges::size(line_list) == 1);
+
+		const LineObject& line = line_list.front();
+		CHECK_FALSE(line.empty());
+		CHECK(ranges::size(line) == 2);
+		CHECK(line.value_count() == 3);
+		CHECK(line.prefix_end() == 1);
+		CHECK(line.suffix_end() == 3);
+
+		for (const auto [index, val] : line | ranges::views::enumerate) {
+			CAPTURE(index);
+			CHECK_FALSE_OR_CONTINUE(val.second.empty());
+			switch (index) {
+				case 0:
+					CHECK_OR_CONTINUE(val.first == 1);
+					CHECK_OR_CONTINUE(val.second == "$NAME$ wurde in $PROV$ gebaut."sv);
+					break;
+				case 1:
+					CHECK_OR_CONTINUE(val.first == 2);
+					CHECK_OR_CONTINUE(val.second == "IDĘ DO"sv);
+					break;
+				case 2:
+					CHECK_OR_CONTINUE(val.first == 3);
+					CHECK_OR_CONTINUE(val.second == ""sv);
+					break;
+				default: CHECK_OR_CONTINUE(false); break;
+			}
+		}
+
+		CHECK(line.value_count() == 3);
+
+		for (const auto index : ranges::views::iota(size_t(0), line.value_count())) {
+			CAPTURE(index);
+			switch (index) {
+				case 0: CHECK_OR_CONTINUE(line.get_value_for(index) == ""sv); break;
+				case 1: CHECK_OR_CONTINUE(line.get_value_for(index) == "$NAME$ wurde in $PROV$ gebaut."sv); break;
+				case 2: CHECK_OR_CONTINUE(line.get_value_for(index) == "IDĘ DO"sv); break;
+				default: CHECK_OR_CONTINUE(false); break;
+			}
+		}
+	}
+#endif
 }
\ No newline at end of file