Followup big dataloader commit

4 files changed, 171 insertions, 68 deletions

diff --git a/src/openvic/dataloader/Dataloader.cpp b/src/openvic/dataloader/Dataloader.cpp
index 90537c9..f2b725e 100644
--- a/src/openvic/dataloader/Dataloader.cpp
+++ b/src/openvic/dataloader/Dataloader.cpp
@@ -1,10 +1,10 @@
 #include "Dataloader.hpp"
 
+#include "openvic/GameManager.hpp"
 #include "openvic/utility/Logger.hpp"
-#include "openvic//GameManager.hpp"
-#include "openvic/dataloader/NodeTools.hpp"
 
 #include <openvic-dataloader/detail/CallbackOStream.hpp>
+#include <openvic-dataloader/v2script/Parser.hpp>
 
 using namespace OpenVic;
 using namespace ovdl::v2script;
@@ -14,19 +14,26 @@ return_t Dataloader::set_roots(std::vector<std::filesystem::path> new_roots) {
 		Logger::error("Overriding existing dataloader roots!");
 		roots.clear();
 	}
+	return_t ret = SUCCESS;
 	for (std::reverse_iterator<std::vector<std::filesystem::path>::const_iterator> it = new_roots.crbegin(); it != new_roots.crend(); ++it) {
-		if (std::filesystem::is_directory(*it)) {
-			Logger::info("Adding dataloader root: ", *it);
-			roots.push_back(*it);
+		if (std::find(roots.begin(), roots.end(), *it) == roots.end()) {
+			if (std::filesystem::is_directory(*it)) {
+				Logger::info("Adding dataloader root: ", *it);
+				roots.push_back(*it);
+			} else {
+				Logger::error("Invalid dataloader root (must be an existing directory): ", *it);
+				ret = FAILURE;
+			}
 		} else {
-			Logger::error("Invalid dataloader root (must be an existing directory): ", *it);
+			Logger::error("Duplicate dataloader root: ", *it);
+			ret = FAILURE;
 		}
 	}
 	if (roots.empty()) {
 		Logger::error("Dataloader has no roots after attempting to add ", new_roots.size());
-		return FAILURE;
+		ret = FAILURE;
 	}
-	return SUCCESS;
+	return ret;
 }
 
 std::filesystem::path Dataloader::lookup_file(std::filesystem::path const& path) const {
@@ -40,28 +47,52 @@ std::filesystem::path Dataloader::lookup_file(std::filesystem::path const& path)
 	return {};
 }
 
-static bool contains_file_with_name(std::vector<std::filesystem::path> const& paths, std::filesystem::path const& name) {
+const std::filesystem::path Dataloader::TXT = ".txt";
+
+static bool contains_file_with_name(std::vector<std::filesystem::path> const& paths,
+	std::filesystem::path const& name) {
+
 	for (std::filesystem::path const& path : paths) {
 		if (path.filename() == name) return true;
 	}
 	return false;
 }
 
-std::vector<std::filesystem::path> Dataloader::lookup_files_in_dir(std::filesystem::path const& path) const {
+std::vector<std::filesystem::path> Dataloader::lookup_files_in_dir(std::filesystem::path const& path,
+	std::filesystem::path const* extension) const {
+
 	std::vector<std::filesystem::path> ret;
 	for (std::filesystem::path const& root : roots) {
 		const std::filesystem::path composed = root / path;
 		std::error_code ec;
 		for (std::filesystem::directory_entry const& entry : std::filesystem::directory_iterator { composed, ec }) {
-			if (entry.is_regular_file() && !contains_file_with_name(ret, entry.path().filename())) {
-				ret.push_back(entry);
+			if (entry.is_regular_file()) {
+				const std::filesystem::path file = entry;
+				if (extension == nullptr || file.extension() == *extension) {
+					if (!contains_file_with_name(ret, file.filename())) {
+						ret.push_back(file);
+					}
+				}
 			}
 		}
 	}
 	return ret;
 }
 
-static Parser parse_defines(std::filesystem::path const& path) {
+return_t Dataloader::apply_to_files_in_dir(std::filesystem::path const& path,
+	std::function<return_t(std::filesystem::path const&)> callback,
+	std::filesystem::path const* extension) const {
+
+	return_t ret = SUCCESS;
+	for (std::filesystem::path const& file : lookup_files_in_dir(path, extension)) {
+		if (callback(file) != SUCCESS) {
+			ret = FAILURE;
+		}
+	}
+	return ret;
+}
+
+Parser Dataloader::parse_defines(std::filesystem::path const& path) {
 	Parser parser;
 	std::string buffer;
 	auto error_log_stream = ovdl::detail::CallbackStream {
@@ -81,8 +112,8 @@ static Parser parse_defines(std::filesystem::path const& path) {
 		Logger::error("Parser load errors:\n\n", buffer, "\n");
 		buffer.clear();
 	}
-	if (parser.has_fatal_error() || parser.has_error() || parser.has_warning()) {
-		Logger::error("Parser warnings/errors while loading ", path);
+	if (parser.has_fatal_error() || parser.has_error()) {
+		Logger::error("Parser errors while loading ", path);
 		return parser;
 	}
 	parser.simple_parse();
@@ -90,51 +121,74 @@ static Parser parse_defines(std::filesystem::path const& path) {
 		Logger::error("Parser parse errors:\n\n", buffer, "\n");
 		buffer.clear();
 	}
-	if (parser.has_fatal_error() || parser.has_error() || parser.has_warning()) {
-		Logger::error("Parser warnings/errors while parsing ", path);
+	if (parser.has_fatal_error() || parser.has_error()) {
+		Logger::error("Parser errors while parsing ", path);
 	}
 	return parser;
 }
 
+Parser Dataloader::parse_defines_lookup(std::filesystem::path const& path) const {
+	return parse_defines(lookup_file(path));
+}
+
+return_t Dataloader::_load_pop_types(PopManager& pop_manager, std::filesystem::path const& pop_type_directory) const {
+	return_t ret = SUCCESS;
+	if (apply_to_files_in_dir(pop_type_directory, [&pop_manager](std::filesystem::path const& file) -> return_t {
+		return pop_manager.load_pop_type_file(file, parse_defines(file).get_file_node());
+	}) != SUCCESS) {
+		Logger::error("Failed to load pop types!");
+		ret = FAILURE;
+	}
+	pop_manager.lock_pop_types();
+	return ret;
+}
+
 return_t Dataloader::load_defines(GameManager& game_manager) const {
+	static const std::filesystem::path good_file = "common/goods.txt";
+	static const std::filesystem::path pop_type_directory = "poptypes";
 	static const std::filesystem::path graphical_culture_type_file = "common/graphicalculturetype.txt";
 	static const std::filesystem::path culture_file = "common/cultures.txt";
+	static const std::filesystem::path religion_file = "common/religion.txt";
 
 	return_t ret = SUCCESS;
 
-	if (game_manager.pop_manager.culture_manager.load_graphical_culture_type_file(parse_defines(
-		lookup_file(graphical_culture_type_file)).get_file_node()) != SUCCESS) {
+	if (game_manager.good_manager.load_good_file(parse_defines_lookup(good_file).get_file_node()) != SUCCESS) {
+		Logger::error("Failed to load goods!");
+		ret = FAILURE;
+	}
+	if (_load_pop_types(game_manager.pop_manager, pop_type_directory) != SUCCESS) {
+		Logger::error("Failed to load pop types!");
+		ret = FAILURE;
+	}
+	if (game_manager.pop_manager.culture_manager.load_graphical_culture_type_file(parse_defines_lookup(graphical_culture_type_file).get_file_node()) != SUCCESS) {
 		Logger::error("Failed to load graphical culture types!");
 		ret = FAILURE;
 	}
-	if (game_manager.pop_manager.culture_manager.load_culture_file(parse_defines(
-		lookup_file(culture_file)).get_file_node()) != SUCCESS) {
+	if (game_manager.pop_manager.culture_manager.load_culture_file(parse_defines_lookup(culture_file).get_file_node()) != SUCCESS) {
 		Logger::error("Failed to load cultures!");
 		ret = FAILURE;
 	}
+	if (game_manager.pop_manager.religion_manager.load_religion_file(parse_defines_lookup(religion_file).get_file_node()) != SUCCESS) {
+		Logger::error("Failed to load religions!");
+		ret = FAILURE;
+	}
 
 	return ret;
 }
 
-static return_t load_pop_history_file(GameManager& game_manager, std::filesystem::path const& path) {
-	return NodeTools::expect_dictionary(parse_defines(path).get_file_node(), [&game_manager](std::string_view province_key, ast::NodeCPtr province_node) -> return_t {
-		Province* province = game_manager.map.get_province_by_identifier(province_key);
-		if (province == nullptr) {
-			Logger::error("Invalid province id: ", province_key);
-			return FAILURE;
-		}
-		return NodeTools::expect_list(province_node, [&game_manager, &province](ast::NodeCPtr pop_node) -> return_t {
-			return game_manager.pop_manager.load_pop_into_province(*province, pop_node);
-		});
-	}, true);
-}
-
 return_t Dataloader::load_pop_history(GameManager& game_manager, std::filesystem::path const& path) const {
-	return_t ret = SUCCESS;
-	for (std::filesystem::path const& file : lookup_files_in_dir(path)) {
-		if (load_pop_history_file(game_manager, file) != SUCCESS) {
-			ret = FAILURE;
-		}
-	}
-	return ret;
+	return apply_to_files_in_dir(path, [&game_manager](std::filesystem::path const& file) -> return_t {
+		return NodeTools::expect_dictionary(parse_defines(file).get_file_node(),
+			[&game_manager](std::string_view province_key, ast::NodeCPtr province_node) -> return_t {
+				Province* province = game_manager.map.get_province_by_identifier(province_key);
+				if (province == nullptr) {
+					Logger::error("Invalid province id: ", province_key);
+					return FAILURE;
+				}
+				return NodeTools::expect_list(province_node, [&game_manager, &province](ast::NodeCPtr pop_node) -> return_t {
+					return game_manager.pop_manager.load_pop_into_province(*province, pop_node);
+				}
+			);
+		}, true);
+	});
 }
diff --git a/src/openvic/dataloader/Dataloader.hpp b/src/openvic/dataloader/Dataloader.hpp
index b080300..3c868a3 100644
--- a/src/openvic/dataloader/Dataloader.hpp
+++ b/src/openvic/dataloader/Dataloader.hpp
@@ -1,14 +1,23 @@
 #pragma once
 
+#include <filesystem>
+
 #include "openvic/types/Return.hpp"
+
 #include "openvic-dataloader/v2script/Parser.hpp"
 
 namespace OpenVic {
 	struct GameManager;
+	struct PopManager;
 
 	class Dataloader {
 		std::vector<std::filesystem::path> roots;
 
+		static ovdl::v2script::Parser parse_defines(std::filesystem::path const& path);
+		ovdl::v2script::Parser parse_defines_lookup(std::filesystem::path const& path) const;
+
+		return_t _load_pop_types(PopManager& pop_manager, std::filesystem::path const& pop_type_directory) const;
+
 	public:
 		Dataloader() = default;
 
@@ -16,7 +25,12 @@ namespace OpenVic {
 		return_t set_roots(std::vector<std::filesystem::path> new_roots);
 
 		std::filesystem::path lookup_file(std::filesystem::path const& path) const;
-		std::vector<std::filesystem::path> lookup_files_in_dir(std::filesystem::path const& path) const;
+		static const std::filesystem::path TXT;
+		std::vector<std::filesystem::path> lookup_files_in_dir(std::filesystem::path const& path,
+			std::filesystem::path const* extension = &TXT) const;
+		return_t apply_to_files_in_dir(std::filesystem::path const& path,
+			std::function<return_t(std::filesystem::path const&)> callback,
+			std::filesystem::path const* extension = &TXT) const;
 
 		return_t load_defines(GameManager& game_manager) const;
 		return_t load_pop_history(GameManager& game_manager, std::filesystem::path const& path) const;
diff --git a/src/openvic/dataloader/NodeTools.cpp b/src/openvic/dataloader/NodeTools.cpp
index a37892f..6ab2ba6 100644
--- a/src/openvic/dataloader/NodeTools.cpp
+++ b/src/openvic/dataloader/NodeTools.cpp
@@ -66,11 +66,9 @@ return_t NodeTools::expect_bool(ast::NodeCPtr node, std::function<return_t(bool)
 
 return_t NodeTools::expect_int(ast::NodeCPtr node, std::function<return_t(int64_t)> callback) {
 	return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
-		int64_t val;
-		char const* const start = identifier.data();
-		char const* const end = start + identifier.size();
-		const std::from_chars_result result = std::from_chars(start, end, val);
-		if (result.ec == std::errc{} && result.ptr == end) {
+		bool successful = false;
+		const int64_t val = StringUtils::string_to_int64(identifier, &successful, 10);
+		if (successful) {
 			return callback(val);
 		}
 		Logger::error("Invalid int identifier text: ", identifier);
@@ -80,10 +78,9 @@ return_t NodeTools::expect_int(ast::NodeCPtr node, std::function<return_t(int64_
 
 return_t NodeTools::expect_uint(ast::NodeCPtr node, std::function<return_t(uint64_t)> callback) {
 	return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
-		uint64_t val;
-		char const* identifier_end = identifier.data() + identifier.size();
-		const std::from_chars_result result = std::from_chars(identifier.data(), identifier_end, val);
-		if (result.ec == std::errc{} && result.ptr == identifier_end) {
+		bool successful = false;
+		const uint64_t val = StringUtils::string_to_uint64(identifier, &successful, 10);
+		if (successful) {
 			return callback(val);
 		}
 		Logger::error("Invalid uint identifier text: ", identifier);
@@ -94,7 +91,7 @@ return_t NodeTools::expect_uint(ast::NodeCPtr node, std::function<return_t(uint6
 return_t NodeTools::expect_fixed_point(ast::NodeCPtr node, std::function<return_t(FP)> callback) {
 	return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
 		bool successful = false;
-		FP val = FP::parse(identifier.data(), identifier.length(), &successful);
+		const FP val = FP::parse(identifier.data(), identifier.length(), &successful);
 		if (successful) {
 			return callback(val);
 		}
@@ -106,7 +103,7 @@ return_t NodeTools::expect_fixed_point(ast::NodeCPtr node, std::function<return_
 return_t NodeTools::expect_colour(ast::NodeCPtr node, std::function<return_t(colour_t)> callback) {
 	colour_t col = NULL_COLOUR;
 	uint32_t components = 0;
-	return_t ret = expect_list(node, std::bind(expect_fixed_point, std::placeholders::_1,
+	return_t ret = expect_list_of_length(node, std::bind(expect_fixed_point, std::placeholders::_1,
 		[&col, &components](FP val) -> return_t {
 			return_t ret = SUCCESS;
 			if (val < 0 || val > 255) {
@@ -127,7 +124,7 @@ return_t NodeTools::expect_colour(ast::NodeCPtr node, std::function<return_t(col
 return_t NodeTools::expect_date(ast::NodeCPtr node, std::function<return_t(Date)> callback) {
 	return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
 		bool successful = false;
-		Date date = Date::from_string(identifier, &successful);
+		const Date date = Date::from_string(identifier, &successful);
 		if (successful) {
 			return callback(date);
 		}
@@ -142,16 +139,16 @@ return_t NodeTools::expect_assign(ast::NodeCPtr node, std::function<return_t(std
 	});
 }
 
-return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node) {
-	const std::function<return_t(std::vector<ast::NodeUPtr> const&)> list_func = [length, callback](std::vector<ast::NodeUPtr> const& list) -> return_t {
+return_t NodeTools::expect_list_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(ast::NodeCPtr)> callback, bool file_node) {
+	const std::function<return_t(std::vector<ast::NodeUPtr> const&)> list_func = [length_callback, callback](std::vector<ast::NodeUPtr> const& list) -> return_t {
 		return_t ret = SUCCESS;
-		size_t size = list.size();
-		if (length > 0 && size != length) {
-			Logger::error("List length ", size, " does not match expected length ", length);
+		size_t size = length_callback(list.size());
+		if (size > list.size()) {
+			Logger::error("Trying to read more values than the list contains: ", size, " > ", list.size());
+			size = list.size();
 			ret = FAILURE;
-			if (length < size) size = length;
 		}
-		std::for_each(list.begin(), list.begin() + size, [callback, &ret](ast::NodeUPtr const& sub_node) {
+		std::for_each(list.begin(), list.begin() + size, [callback, &ret](ast::NodeUPtr const& sub_node) -> void {
 			if (callback(sub_node.get()) != SUCCESS) ret = FAILURE;
 		});
 		return ret;
@@ -167,13 +164,34 @@ return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast::
 	}
 }
 
-return_t NodeTools::expect_dictionary(ast::NodeCPtr node, std::function<return_t(const std::string_view, ast::NodeCPtr)> callback, bool file_node) {
-	return expect_list(node, std::bind(expect_assign, std::placeholders::_1, callback), 0, file_node);
+return_t NodeTools::expect_list_of_length(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node) {
+	return_t ret = SUCCESS;
+	if (expect_list_and_length(node, [length, &ret](size_t size) -> size_t {
+		if (size != length) {
+			Logger::error("List length ", size, " does not match expected length ", length);
+			ret = FAILURE;
+			if (length < size) return length;
+		}
+		return size;
+	}, callback, file_node) != SUCCESS) ret = FAILURE;
+	return ret;
+}
+
+return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, bool file_node) {
+	return expect_list_and_length(node, default_length_callback, callback, file_node);
+}
+
+return_t NodeTools::expect_dictionary_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node) {
+	return expect_list_and_length(node, length_callback, std::bind(expect_assign, std::placeholders::_1, callback), file_node);
+}
+
+return_t NodeTools::expect_dictionary(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node) {
+	return expect_dictionary_and_length(node, default_length_callback, callback, file_node);
 }
 
 return_t NodeTools::expect_dictionary_keys(ast::NodeCPtr node, dictionary_key_map_t const& keys, bool allow_other_keys, bool file_node) {
 	std::map<std::string, size_t, std::less<void>> key_count;
-	return_t ret = expect_dictionary(node, [keys, allow_other_keys, &key_count](const std::string_view key, ast::NodeCPtr value) -> return_t {
+	return_t ret = expect_dictionary(node, [keys, allow_other_keys, &key_count](std::string_view key, ast::NodeCPtr value) -> return_t {
 		const dictionary_key_map_t::const_iterator it = keys.find(key);
 		if (it == keys.end()) {
 			if (allow_other_keys) return SUCCESS;
diff --git a/src/openvic/dataloader/NodeTools.hpp b/src/openvic/dataloader/NodeTools.hpp
index ca7130c..48c3710 100644
--- a/src/openvic/dataloader/NodeTools.hpp
+++ b/src/openvic/dataloader/NodeTools.hpp
@@ -2,13 +2,13 @@
 
 #include <map>
 
-#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp>
-
 #include "openvic/types/Colour.hpp"
-#include "openvic/types/Return.hpp"
 #include "openvic/types/Date.hpp"
+#include "openvic/types/Return.hpp"
 #include "openvic/types/fixed_point/FP.hpp"
 
+#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp>
+
 namespace OpenVic {
 	namespace ast = ovdl::v2script::ast;
 
@@ -27,7 +27,24 @@ namespace OpenVic {
 		return_t expect_colour(ast::NodeCPtr node, std::function<return_t(colour_t)> callback);
 		return_t expect_date(ast::NodeCPtr node, std::function<return_t(Date)> callback);
 		return_t expect_assign(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback);
-		return_t expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length = 0, bool file_node = false);
+
+		static const std::function<size_t(size_t)> default_length_callback = [](size_t size) -> size_t { return size; };
+
+		template<typename T> requires requires(T& t) {
+			{ t.size() } -> std::same_as<size_t>;
+			t.reserve( size_t {} );
+		}
+		std::function<size_t(size_t)> reserve_length_callback(T& t) {
+			return [&t](size_t size) -> size_t {
+				t.reserve(t.size() + size);
+				return size;
+			};
+		}
+
+		return_t expect_list_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(ast::NodeCPtr)> callback, bool file_node = false);
+		return_t expect_list_of_length(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node = false);
+		return_t expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, bool file_node = false);
+		return_t expect_dictionary_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node = false);
 		return_t expect_dictionary(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node = false);
 
 		static const std::function<return_t(ast::NodeCPtr)> success_callback = [](ast::NodeCPtr) -> return_t { return SUCCESS; };