4 files changed, 398 insertions, 10 deletions

diff --git a/src/openvic/dataloader/Dataloader.cpp b/src/openvic/dataloader/Dataloader.cpp
new file mode 100644
index 0000000..90537c9
--- /dev/null
+++ b/src/openvic/dataloader/Dataloader.cpp
@@ -0,0 +1,140 @@
+#include "Dataloader.hpp"
+
+#include "openvic/utility/Logger.hpp"
+#include "openvic//GameManager.hpp"
+#include "openvic/dataloader/NodeTools.hpp"
+
+#include <openvic-dataloader/detail/CallbackOStream.hpp>
+
+using namespace OpenVic;
+using namespace ovdl::v2script;
+
+return_t Dataloader::set_roots(std::vector<std::filesystem::path> new_roots) {
+	if (!roots.empty()) {
+		Logger::error("Overriding existing dataloader roots!");
+		roots.clear();
+	}
+	for (std::reverse_iterator<std::vector<std::filesystem::path>::const_iterator> it = new_roots.crbegin(); it != new_roots.crend(); ++it) {
+		if (std::filesystem::is_directory(*it)) {
+			Logger::info("Adding dataloader root: ", *it);
+			roots.push_back(*it);
+		} else {
+			Logger::error("Invalid dataloader root (must be an existing directory): ", *it);
+		}
+	}
+	if (roots.empty()) {
+		Logger::error("Dataloader has no roots after attempting to add ", new_roots.size());
+		return FAILURE;
+	}
+	return SUCCESS;
+}
+
+std::filesystem::path Dataloader::lookup_file(std::filesystem::path const& path) const {
+	for (std::filesystem::path const& root : roots) {
+		const std::filesystem::path composed = root / path;
+		if (std::filesystem::is_regular_file(composed)) {
+			return composed;
+		}
+	}
+	Logger::error("Lookup for ", path, " failed!");
+	return {};
+}
+
+static bool contains_file_with_name(std::vector<std::filesystem::path> const& paths, std::filesystem::path const& name) {
+	for (std::filesystem::path const& path : paths) {
+		if (path.filename() == name) return true;
+	}
+	return false;
+}
+
+std::vector<std::filesystem::path> Dataloader::lookup_files_in_dir(std::filesystem::path const& path) const {
+	std::vector<std::filesystem::path> ret;
+	for (std::filesystem::path const& root : roots) {
+		const std::filesystem::path composed = root / path;
+		std::error_code ec;
+		for (std::filesystem::directory_entry const& entry : std::filesystem::directory_iterator { composed, ec }) {
+			if (entry.is_regular_file() && !contains_file_with_name(ret, entry.path().filename())) {
+				ret.push_back(entry);
+			}
+		}
+	}
+	return ret;
+}
+
+static Parser parse_defines(std::filesystem::path const& path) {
+	Parser parser;
+	std::string buffer;
+	auto error_log_stream = ovdl::detail::CallbackStream {
+		[](void const* s, std::streamsize n, void* user_data) -> std::streamsize {
+			if (s != nullptr && n > 0 && user_data != nullptr) {
+				static_cast<std::string*>(user_data)->append(static_cast<char const*>(s), n);
+				return n;
+			} else {
+				Logger::error("Invalid input to parser error log callback: ", s, " / ", n, " / ", user_data);
+				return 0;
+			}
+		}, &buffer
+	};
+	parser.set_error_log_to(error_log_stream);
+	parser.load_from_file(path);
+	if (!buffer.empty()) {
+		Logger::error("Parser load errors:\n\n", buffer, "\n");
+		buffer.clear();
+	}
+	if (parser.has_fatal_error() || parser.has_error() || parser.has_warning()) {
+		Logger::error("Parser warnings/errors while loading ", path);
+		return parser;
+	}
+	parser.simple_parse();
+	if (!buffer.empty()) {
+		Logger::error("Parser parse errors:\n\n", buffer, "\n");
+		buffer.clear();
+	}
+	if (parser.has_fatal_error() || parser.has_error() || parser.has_warning()) {
+		Logger::error("Parser warnings/errors while parsing ", path);
+	}
+	return parser;
+}
+
+return_t Dataloader::load_defines(GameManager& game_manager) const {
+	static const std::filesystem::path graphical_culture_type_file = "common/graphicalculturetype.txt";
+	static const std::filesystem::path culture_file = "common/cultures.txt";
+
+	return_t ret = SUCCESS;
+
+	if (game_manager.pop_manager.culture_manager.load_graphical_culture_type_file(parse_defines(
+		lookup_file(graphical_culture_type_file)).get_file_node()) != SUCCESS) {
+		Logger::error("Failed to load graphical culture types!");
+		ret = FAILURE;
+	}
+	if (game_manager.pop_manager.culture_manager.load_culture_file(parse_defines(
+		lookup_file(culture_file)).get_file_node()) != SUCCESS) {
+		Logger::error("Failed to load cultures!");
+		ret = FAILURE;
+	}
+
+	return ret;
+}
+
+static return_t load_pop_history_file(GameManager& game_manager, std::filesystem::path const& path) {
+	return NodeTools::expect_dictionary(parse_defines(path).get_file_node(), [&game_manager](std::string_view province_key, ast::NodeCPtr province_node) -> return_t {
+		Province* province = game_manager.map.get_province_by_identifier(province_key);
+		if (province == nullptr) {
+			Logger::error("Invalid province id: ", province_key);
+			return FAILURE;
+		}
+		return NodeTools::expect_list(province_node, [&game_manager, &province](ast::NodeCPtr pop_node) -> return_t {
+			return game_manager.pop_manager.load_pop_into_province(*province, pop_node);
+		});
+	}, true);
+}
+
+return_t Dataloader::load_pop_history(GameManager& game_manager, std::filesystem::path const& path) const {
+	return_t ret = SUCCESS;
+	for (std::filesystem::path const& file : lookup_files_in_dir(path)) {
+		if (load_pop_history_file(game_manager, file) != SUCCESS) {
+			ret = FAILURE;
+		}
+	}
+	return ret;
+}
diff --git a/src/openvic/dataloader/Dataloader.hpp b/src/openvic/dataloader/Dataloader.hpp
index 062f3d5..b080300 100644
--- a/src/openvic/dataloader/Dataloader.hpp
+++ b/src/openvic/dataloader/Dataloader.hpp
@@ -1,16 +1,24 @@
 #pragma once
-#include <filesystem>
-#include "../Simulation.hpp"
+
+#include "openvic/types/Return.hpp"
+#include "openvic-dataloader/v2script/Parser.hpp"
 
 namespace OpenVic {
+	struct GameManager;
+
 	class Dataloader {
-		public:
-		enum class LoadingMode {
-			DL_COMPATIBILITY
-		};
+		std::vector<std::filesystem::path> roots;
+
+	public:
+		Dataloader() = default;
+
+		/* In reverse-load order, so base defines first and final loaded mod last */
+		return_t set_roots(std::vector<std::filesystem::path> new_roots);
+
+		std::filesystem::path lookup_file(std::filesystem::path const& path) const;
+		std::vector<std::filesystem::path> lookup_files_in_dir(std::filesystem::path const& path) const;
 
-		static bool loadDir(std::filesystem::path p, Simulation& sim, LoadingMode loadMode = LoadingMode::DL_COMPATIBILITY) {
-			return true;
-		}
+		return_t load_defines(GameManager& game_manager) const;
+		return_t load_pop_history(GameManager& game_manager, std::filesystem::path const& path) const;
 	};
-}
\ No newline at end of file
+}
diff --git a/src/openvic/dataloader/NodeTools.cpp b/src/openvic/dataloader/NodeTools.cpp
new file mode 100644
index 0000000..a37892f
--- /dev/null
+++ b/src/openvic/dataloader/NodeTools.cpp
@@ -0,0 +1,198 @@
+#include "NodeTools.hpp"
+
+#include <charconv>
+
+using namespace OpenVic;
+
+template<typename T>
+return_t NodeTools::expect_type(ast::NodeCPtr node, std::function<return_t(T const&)> callback) {
+	if (node != nullptr) {
+		if (node->is_type<T>()) {
+			return callback(ast::cast_node_cptr<T>(node));
+		}
+		Logger::error("Invalid node type ", node->get_type(), " when expecting ", T::get_type_static());
+	} else {
+		Logger::error("Null node when expecting ", T::get_type_static());
+	}
+	return FAILURE;
+}
+
+static return_t identifier_callback_wrapper(std::function<return_t(std::string_view)> callback, std::string_view identifier) {
+	if (!identifier.empty()) {
+		return callback(identifier);
+	}
+	Logger::error("Empty identifier node string");
+	return FAILURE;
+}
+
+return_t NodeTools::expect_identifier(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback) {
+	return expect_type<ast::IdentifierNode>(node, [callback](ast::IdentifierNode const& identifier_node) -> return_t {
+		return identifier_callback_wrapper(callback, identifier_node._name);
+	});
+}
+
+return_t NodeTools::expect_string(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback) {
+	return expect_type<ast::StringNode>(node, [callback](ast::StringNode const& string_node) -> return_t {
+		return callback(string_node._name);
+	});
+}
+
+return_t NodeTools::expect_identifier_or_string(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback) {
+	if (node != nullptr) {
+		if (node->is_type<ast::IdentifierNode>()) {
+			return identifier_callback_wrapper(callback, ast::cast_node_cptr<ast::IdentifierNode>(node)._name);
+		} else if (node->is_type<ast::StringNode>()) {
+			return callback(ast::cast_node_cptr<ast::StringNode>(node)._name);
+		} else {
+			Logger::error("Invalid node type ", node->get_type(), " when expecting ", ast::IdentifierNode::get_type_static(), " or ", ast::StringNode::get_type_static());
+		}
+	} else {
+		Logger::error("Null node when expecting ", ast::IdentifierNode::get_type_static(), " or ", ast::StringNode::get_type_static());
+	}
+	return FAILURE;
+}
+
+return_t NodeTools::expect_bool(ast::NodeCPtr node, std::function<return_t(bool)> callback) {
+	return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
+		if (identifier == "yes") {
+			return callback(true);
+		} else if (identifier == "no") {
+			return callback(false);
+		}
+		Logger::error("Invalid bool identifier text: ", identifier);
+		return FAILURE;
+	});
+}
+
+return_t NodeTools::expect_int(ast::NodeCPtr node, std::function<return_t(int64_t)> callback) {
+	return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
+		int64_t val;
+		char const* const start = identifier.data();
+		char const* const end = start + identifier.size();
+		const std::from_chars_result result = std::from_chars(start, end, val);
+		if (result.ec == std::errc{} && result.ptr == end) {
+			return callback(val);
+		}
+		Logger::error("Invalid int identifier text: ", identifier);
+		return FAILURE;
+	});
+}
+
+return_t NodeTools::expect_uint(ast::NodeCPtr node, std::function<return_t(uint64_t)> callback) {
+	return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
+		uint64_t val;
+		char const* identifier_end = identifier.data() + identifier.size();
+		const std::from_chars_result result = std::from_chars(identifier.data(), identifier_end, val);
+		if (result.ec == std::errc{} && result.ptr == identifier_end) {
+			return callback(val);
+		}
+		Logger::error("Invalid uint identifier text: ", identifier);
+		return FAILURE;
+	});
+}
+
+return_t NodeTools::expect_fixed_point(ast::NodeCPtr node, std::function<return_t(FP)> callback) {
+	return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
+		bool successful = false;
+		FP val = FP::parse(identifier.data(), identifier.length(), &successful);
+		if (successful) {
+			return callback(val);
+		}
+		Logger::error("Invalid fixed point identifier text: ", identifier);
+		return FAILURE;
+	});
+}
+
+return_t NodeTools::expect_colour(ast::NodeCPtr node, std::function<return_t(colour_t)> callback) {
+	colour_t col = NULL_COLOUR;
+	uint32_t components = 0;
+	return_t ret = expect_list(node, std::bind(expect_fixed_point, std::placeholders::_1,
+		[&col, &components](FP val) -> return_t {
+			return_t ret = SUCCESS;
+			if (val < 0 || val > 255) {
+				Logger::error("Invalid colour component: ", val);
+				val = FP::_0();
+				ret = FAILURE;
+			}
+			if (val <= 1) val *= 255;
+			col = (col << 8) | val.to_int32_t();
+			components++;
+			return ret;
+		}), 3);
+	if (components < 3) col <<= 8 * (3 - components);
+	if (callback(col) != SUCCESS) ret = FAILURE;
+	return ret;
+}
+
+return_t NodeTools::expect_date(ast::NodeCPtr node, std::function<return_t(Date)> callback) {
+	return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
+		bool successful = false;
+		Date date = Date::from_string(identifier, &successful);
+		if (successful) {
+			return callback(date);
+		}
+		Logger::error("Invalid date identifier text: ", identifier);
+		return FAILURE;
+	});
+}
+
+return_t NodeTools::expect_assign(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback) {
+	return expect_type<ast::AssignNode>(node, [callback](ast::AssignNode const& assign_node) -> return_t {
+		return callback(assign_node._name, assign_node._initializer.get());
+	});
+}
+
+return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node) {
+	const std::function<return_t(std::vector<ast::NodeUPtr> const&)> list_func = [length, callback](std::vector<ast::NodeUPtr> const& list) -> return_t {
+		return_t ret = SUCCESS;
+		size_t size = list.size();
+		if (length > 0 && size != length) {
+			Logger::error("List length ", size, " does not match expected length ", length);
+			ret = FAILURE;
+			if (length < size) size = length;
+		}
+		std::for_each(list.begin(), list.begin() + size, [callback, &ret](ast::NodeUPtr const& sub_node) {
+			if (callback(sub_node.get()) != SUCCESS) ret = FAILURE;
+		});
+		return ret;
+	};
+	if (!file_node) {
+		return expect_type<ast::ListNode>(node, [list_func](ast::ListNode const& list_node) -> return_t {
+			return list_func(list_node._statements);
+		});
+	} else {
+		return expect_type<ast::FileNode>(node, [list_func](ast::FileNode const& file_node) -> return_t {
+			return list_func(file_node._statements);
+		});
+	}
+}
+
+return_t NodeTools::expect_dictionary(ast::NodeCPtr node, std::function<return_t(const std::string_view, ast::NodeCPtr)> callback, bool file_node) {
+	return expect_list(node, std::bind(expect_assign, std::placeholders::_1, callback), 0, file_node);
+}
+
+return_t NodeTools::expect_dictionary_keys(ast::NodeCPtr node, dictionary_key_map_t const& keys, bool allow_other_keys, bool file_node) {
+	std::map<std::string, size_t, std::less<void>> key_count;
+	return_t ret = expect_dictionary(node, [keys, allow_other_keys, &key_count](const std::string_view key, ast::NodeCPtr value) -> return_t {
+		const dictionary_key_map_t::const_iterator it = keys.find(key);
+		if (it == keys.end()) {
+			if (allow_other_keys) return SUCCESS;
+			Logger::error("Invalid dictionary key: ", key);
+			return FAILURE;
+		}
+		const size_t count = ++key_count[it->first];
+		dictionary_entry_t const& entry = it->second;
+		if (!entry.can_repeat && count > 1) {
+			Logger::error("Invalid repeat of dictionary key: ", key);
+			return FAILURE;
+		}
+		return entry.callback(value);
+	}, file_node);
+	for (dictionary_key_map_t::value_type const& entry : keys) {
+		if (entry.second.must_appear && key_count.find(entry.first) == key_count.end()) {
+			Logger::error("Mandatory dictionary key not present: ", entry.first);
+			ret = FAILURE;
+		}
+	}
+	return ret;
+}
diff --git a/src/openvic/dataloader/NodeTools.hpp b/src/openvic/dataloader/NodeTools.hpp
new file mode 100644
index 0000000..ca7130c
--- /dev/null
+++ b/src/openvic/dataloader/NodeTools.hpp
@@ -0,0 +1,42 @@
+#pragma once
+
+#include <map>
+
+#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp>
+
+#include "openvic/types/Colour.hpp"
+#include "openvic/types/Return.hpp"
+#include "openvic/types/Date.hpp"
+#include "openvic/types/fixed_point/FP.hpp"
+
+namespace OpenVic {
+	namespace ast = ovdl::v2script::ast;
+
+	namespace NodeTools {
+
+		template<typename T>
+		return_t expect_type(ast::NodeCPtr node, std::function<return_t(T const&)> callback);
+
+		return_t expect_identifier(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback);
+		return_t expect_string(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback);
+		return_t expect_identifier_or_string(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback);
+		return_t expect_bool(ast::NodeCPtr node, std::function<return_t(bool)> callback);
+		return_t expect_int(ast::NodeCPtr node, std::function<return_t(int64_t)> callback);
+		return_t expect_uint(ast::NodeCPtr node, std::function<return_t(uint64_t)> callback);
+		return_t expect_fixed_point(ast::NodeCPtr node, std::function<return_t(FP)> callback);
+		return_t expect_colour(ast::NodeCPtr node, std::function<return_t(colour_t)> callback);
+		return_t expect_date(ast::NodeCPtr node, std::function<return_t(Date)> callback);
+		return_t expect_assign(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback);
+		return_t expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length = 0, bool file_node = false);
+		return_t expect_dictionary(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node = false);
+
+		static const std::function<return_t(ast::NodeCPtr)> success_callback = [](ast::NodeCPtr) -> return_t { return SUCCESS; };
+
+		struct dictionary_entry_t {
+			bool must_appear, can_repeat;
+			std::function<return_t(ast::NodeCPtr)> callback;
+		};
+		using dictionary_key_map_t = std::map<std::string, dictionary_entry_t, std::less<void>>;
+		return_t expect_dictionary_keys(ast::NodeCPtr node, dictionary_key_map_t const& keys, bool allow_other_keys = false, bool file_node = false);
+	}
+}