aboutsummaryrefslogtreecommitdiff
path: root/src/openvic/dataloader
diff options
context:
space:
mode:
author Hop311 <hop3114@gmail.com>2023-08-24 01:32:23 +0200
committer Hop311 <hop3114@gmail.com>2023-08-24 01:32:23 +0200
commitefa88c722fcb6c8fea7a86e1b3b8a83f1f59eb31 (patch)
tree0d64199bf4f19fabaaff083b35aa8625823d6c1e /src/openvic/dataloader
parent6f4a6c77c6f2613e65a403c3a2964d5041a538c7 (diff)
Big Dataloader Commit (openvic-simulation)
Diffstat (limited to 'src/openvic/dataloader')
-rw-r--r--src/openvic/dataloader/Dataloader.cpp140
-rw-r--r--src/openvic/dataloader/Dataloader.hpp28
-rw-r--r--src/openvic/dataloader/NodeTools.cpp198
-rw-r--r--src/openvic/dataloader/NodeTools.hpp42
4 files changed, 398 insertions, 10 deletions
diff --git a/src/openvic/dataloader/Dataloader.cpp b/src/openvic/dataloader/Dataloader.cpp
new file mode 100644
index 0000000..90537c9
--- /dev/null
+++ b/src/openvic/dataloader/Dataloader.cpp
@@ -0,0 +1,140 @@
+#include "Dataloader.hpp"
+
+#include "openvic/utility/Logger.hpp"
+#include "openvic//GameManager.hpp"
+#include "openvic/dataloader/NodeTools.hpp"
+
+#include <openvic-dataloader/detail/CallbackOStream.hpp>
+
+using namespace OpenVic;
+using namespace ovdl::v2script;
+
+return_t Dataloader::set_roots(std::vector<std::filesystem::path> new_roots) {
+ if (!roots.empty()) {
+ Logger::error("Overriding existing dataloader roots!");
+ roots.clear();
+ }
+ for (std::reverse_iterator<std::vector<std::filesystem::path>::const_iterator> it = new_roots.crbegin(); it != new_roots.crend(); ++it) {
+ if (std::filesystem::is_directory(*it)) {
+ Logger::info("Adding dataloader root: ", *it);
+ roots.push_back(*it);
+ } else {
+ Logger::error("Invalid dataloader root (must be an existing directory): ", *it);
+ }
+ }
+ if (roots.empty()) {
+ Logger::error("Dataloader has no roots after attempting to add ", new_roots.size());
+ return FAILURE;
+ }
+ return SUCCESS;
+}
+
+std::filesystem::path Dataloader::lookup_file(std::filesystem::path const& path) const {
+ for (std::filesystem::path const& root : roots) {
+ const std::filesystem::path composed = root / path;
+ if (std::filesystem::is_regular_file(composed)) {
+ return composed;
+ }
+ }
+ Logger::error("Lookup for ", path, " failed!");
+ return {};
+}
+
+static bool contains_file_with_name(std::vector<std::filesystem::path> const& paths, std::filesystem::path const& name) {
+ for (std::filesystem::path const& path : paths) {
+ if (path.filename() == name) return true;
+ }
+ return false;
+}
+
+std::vector<std::filesystem::path> Dataloader::lookup_files_in_dir(std::filesystem::path const& path) const {
+ std::vector<std::filesystem::path> ret;
+ for (std::filesystem::path const& root : roots) {
+ const std::filesystem::path composed = root / path;
+ std::error_code ec;
+ for (std::filesystem::directory_entry const& entry : std::filesystem::directory_iterator { composed, ec }) {
+ if (entry.is_regular_file() && !contains_file_with_name(ret, entry.path().filename())) {
+ ret.push_back(entry);
+ }
+ }
+ }
+ return ret;
+}
+
+static Parser parse_defines(std::filesystem::path const& path) {
+ Parser parser;
+ std::string buffer;
+ auto error_log_stream = ovdl::detail::CallbackStream {
+ [](void const* s, std::streamsize n, void* user_data) -> std::streamsize {
+ if (s != nullptr && n > 0 && user_data != nullptr) {
+ static_cast<std::string*>(user_data)->append(static_cast<char const*>(s), n);
+ return n;
+ } else {
+ Logger::error("Invalid input to parser error log callback: ", s, " / ", n, " / ", user_data);
+ return 0;
+ }
+ }, &buffer
+ };
+ parser.set_error_log_to(error_log_stream);
+ parser.load_from_file(path);
+ if (!buffer.empty()) {
+ Logger::error("Parser load errors:\n\n", buffer, "\n");
+ buffer.clear();
+ }
+ if (parser.has_fatal_error() || parser.has_error() || parser.has_warning()) {
+ Logger::error("Parser warnings/errors while loading ", path);
+ return parser;
+ }
+ parser.simple_parse();
+ if (!buffer.empty()) {
+ Logger::error("Parser parse errors:\n\n", buffer, "\n");
+ buffer.clear();
+ }
+ if (parser.has_fatal_error() || parser.has_error() || parser.has_warning()) {
+ Logger::error("Parser warnings/errors while parsing ", path);
+ }
+ return parser;
+}
+
+return_t Dataloader::load_defines(GameManager& game_manager) const {
+ static const std::filesystem::path graphical_culture_type_file = "common/graphicalculturetype.txt";
+ static const std::filesystem::path culture_file = "common/cultures.txt";
+
+ return_t ret = SUCCESS;
+
+ if (game_manager.pop_manager.culture_manager.load_graphical_culture_type_file(parse_defines(
+ lookup_file(graphical_culture_type_file)).get_file_node()) != SUCCESS) {
+ Logger::error("Failed to load graphical culture types!");
+ ret = FAILURE;
+ }
+ if (game_manager.pop_manager.culture_manager.load_culture_file(parse_defines(
+ lookup_file(culture_file)).get_file_node()) != SUCCESS) {
+ Logger::error("Failed to load cultures!");
+ ret = FAILURE;
+ }
+
+ return ret;
+}
+
+static return_t load_pop_history_file(GameManager& game_manager, std::filesystem::path const& path) {
+ return NodeTools::expect_dictionary(parse_defines(path).get_file_node(), [&game_manager](std::string_view province_key, ast::NodeCPtr province_node) -> return_t {
+ Province* province = game_manager.map.get_province_by_identifier(province_key);
+ if (province == nullptr) {
+ Logger::error("Invalid province id: ", province_key);
+ return FAILURE;
+ }
+ return NodeTools::expect_list(province_node, [&game_manager, &province](ast::NodeCPtr pop_node) -> return_t {
+ return game_manager.pop_manager.load_pop_into_province(*province, pop_node);
+ });
+ }, true);
+}
+
+return_t Dataloader::load_pop_history(GameManager& game_manager, std::filesystem::path const& path) const {
+ return_t ret = SUCCESS;
+ for (std::filesystem::path const& file : lookup_files_in_dir(path)) {
+ if (load_pop_history_file(game_manager, file) != SUCCESS) {
+ ret = FAILURE;
+ }
+ }
+ return ret;
+}
diff --git a/src/openvic/dataloader/Dataloader.hpp b/src/openvic/dataloader/Dataloader.hpp
index 062f3d5..b080300 100644
--- a/src/openvic/dataloader/Dataloader.hpp
+++ b/src/openvic/dataloader/Dataloader.hpp
@@ -1,16 +1,24 @@
#pragma once
-#include <filesystem>
-#include "../Simulation.hpp"
+
+#include "openvic/types/Return.hpp"
+#include "openvic-dataloader/v2script/Parser.hpp"
namespace OpenVic {
+ struct GameManager;
+
class Dataloader {
- public:
- enum class LoadingMode {
- DL_COMPATIBILITY
- };
+ std::vector<std::filesystem::path> roots;
+
+ public:
+ Dataloader() = default;
+
+ /* In reverse-load order, so base defines first and final loaded mod last */
+ return_t set_roots(std::vector<std::filesystem::path> new_roots);
+
+ std::filesystem::path lookup_file(std::filesystem::path const& path) const;
+ std::vector<std::filesystem::path> lookup_files_in_dir(std::filesystem::path const& path) const;
- static bool loadDir(std::filesystem::path p, Simulation& sim, LoadingMode loadMode = LoadingMode::DL_COMPATIBILITY) {
- return true;
- }
+ return_t load_defines(GameManager& game_manager) const;
+ return_t load_pop_history(GameManager& game_manager, std::filesystem::path const& path) const;
};
-} \ No newline at end of file
+}
diff --git a/src/openvic/dataloader/NodeTools.cpp b/src/openvic/dataloader/NodeTools.cpp
new file mode 100644
index 0000000..a37892f
--- /dev/null
+++ b/src/openvic/dataloader/NodeTools.cpp
@@ -0,0 +1,198 @@
+#include "NodeTools.hpp"
+
+#include <charconv>
+
+using namespace OpenVic;
+
+template<typename T>
+return_t NodeTools::expect_type(ast::NodeCPtr node, std::function<return_t(T const&)> callback) {
+ if (node != nullptr) {
+ if (node->is_type<T>()) {
+ return callback(ast::cast_node_cptr<T>(node));
+ }
+ Logger::error("Invalid node type ", node->get_type(), " when expecting ", T::get_type_static());
+ } else {
+ Logger::error("Null node when expecting ", T::get_type_static());
+ }
+ return FAILURE;
+}
+
+static return_t identifier_callback_wrapper(std::function<return_t(std::string_view)> callback, std::string_view identifier) {
+ if (!identifier.empty()) {
+ return callback(identifier);
+ }
+ Logger::error("Empty identifier node string");
+ return FAILURE;
+}
+
+return_t NodeTools::expect_identifier(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback) {
+ return expect_type<ast::IdentifierNode>(node, [callback](ast::IdentifierNode const& identifier_node) -> return_t {
+ return identifier_callback_wrapper(callback, identifier_node._name);
+ });
+}
+
+return_t NodeTools::expect_string(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback) {
+ return expect_type<ast::StringNode>(node, [callback](ast::StringNode const& string_node) -> return_t {
+ return callback(string_node._name);
+ });
+}
+
+return_t NodeTools::expect_identifier_or_string(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback) {
+ if (node != nullptr) {
+ if (node->is_type<ast::IdentifierNode>()) {
+ return identifier_callback_wrapper(callback, ast::cast_node_cptr<ast::IdentifierNode>(node)._name);
+ } else if (node->is_type<ast::StringNode>()) {
+ return callback(ast::cast_node_cptr<ast::StringNode>(node)._name);
+ } else {
+ Logger::error("Invalid node type ", node->get_type(), " when expecting ", ast::IdentifierNode::get_type_static(), " or ", ast::StringNode::get_type_static());
+ }
+ } else {
+ Logger::error("Null node when expecting ", ast::IdentifierNode::get_type_static(), " or ", ast::StringNode::get_type_static());
+ }
+ return FAILURE;
+}
+
+return_t NodeTools::expect_bool(ast::NodeCPtr node, std::function<return_t(bool)> callback) {
+ return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
+ if (identifier == "yes") {
+ return callback(true);
+ } else if (identifier == "no") {
+ return callback(false);
+ }
+ Logger::error("Invalid bool identifier text: ", identifier);
+ return FAILURE;
+ });
+}
+
+return_t NodeTools::expect_int(ast::NodeCPtr node, std::function<return_t(int64_t)> callback) {
+ return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
+ int64_t val;
+ char const* const start = identifier.data();
+ char const* const end = start + identifier.size();
+ const std::from_chars_result result = std::from_chars(start, end, val);
+ if (result.ec == std::errc{} && result.ptr == end) {
+ return callback(val);
+ }
+ Logger::error("Invalid int identifier text: ", identifier);
+ return FAILURE;
+ });
+}
+
+return_t NodeTools::expect_uint(ast::NodeCPtr node, std::function<return_t(uint64_t)> callback) {
+ return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
+ uint64_t val;
+ char const* identifier_end = identifier.data() + identifier.size();
+ const std::from_chars_result result = std::from_chars(identifier.data(), identifier_end, val);
+ if (result.ec == std::errc{} && result.ptr == identifier_end) {
+ return callback(val);
+ }
+ Logger::error("Invalid uint identifier text: ", identifier);
+ return FAILURE;
+ });
+}
+
+return_t NodeTools::expect_fixed_point(ast::NodeCPtr node, std::function<return_t(FP)> callback) {
+ return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
+ bool successful = false;
+ FP val = FP::parse(identifier.data(), identifier.length(), &successful);
+ if (successful) {
+ return callback(val);
+ }
+ Logger::error("Invalid fixed point identifier text: ", identifier);
+ return FAILURE;
+ });
+}
+
+return_t NodeTools::expect_colour(ast::NodeCPtr node, std::function<return_t(colour_t)> callback) {
+ colour_t col = NULL_COLOUR;
+ uint32_t components = 0;
+ return_t ret = expect_list(node, std::bind(expect_fixed_point, std::placeholders::_1,
+ [&col, &components](FP val) -> return_t {
+ return_t ret = SUCCESS;
+ if (val < 0 || val > 255) {
+ Logger::error("Invalid colour component: ", val);
+ val = FP::_0();
+ ret = FAILURE;
+ }
+ if (val <= 1) val *= 255;
+ col = (col << 8) | val.to_int32_t();
+ components++;
+ return ret;
+ }), 3);
+ if (components < 3) col <<= 8 * (3 - components);
+ if (callback(col) != SUCCESS) ret = FAILURE;
+ return ret;
+}
+
+return_t NodeTools::expect_date(ast::NodeCPtr node, std::function<return_t(Date)> callback) {
+ return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
+ bool successful = false;
+ Date date = Date::from_string(identifier, &successful);
+ if (successful) {
+ return callback(date);
+ }
+ Logger::error("Invalid date identifier text: ", identifier);
+ return FAILURE;
+ });
+}
+
+return_t NodeTools::expect_assign(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback) {
+ return expect_type<ast::AssignNode>(node, [callback](ast::AssignNode const& assign_node) -> return_t {
+ return callback(assign_node._name, assign_node._initializer.get());
+ });
+}
+
+return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node) {
+ const std::function<return_t(std::vector<ast::NodeUPtr> const&)> list_func = [length, callback](std::vector<ast::NodeUPtr> const& list) -> return_t {
+ return_t ret = SUCCESS;
+ size_t size = list.size();
+ if (length > 0 && size != length) {
+ Logger::error("List length ", size, " does not match expected length ", length);
+ ret = FAILURE;
+ if (length < size) size = length;
+ }
+ std::for_each(list.begin(), list.begin() + size, [callback, &ret](ast::NodeUPtr const& sub_node) {
+ if (callback(sub_node.get()) != SUCCESS) ret = FAILURE;
+ });
+ return ret;
+ };
+ if (!file_node) {
+ return expect_type<ast::ListNode>(node, [list_func](ast::ListNode const& list_node) -> return_t {
+ return list_func(list_node._statements);
+ });
+ } else {
+ return expect_type<ast::FileNode>(node, [list_func](ast::FileNode const& file_node) -> return_t {
+ return list_func(file_node._statements);
+ });
+ }
+}
+
+return_t NodeTools::expect_dictionary(ast::NodeCPtr node, std::function<return_t(const std::string_view, ast::NodeCPtr)> callback, bool file_node) {
+ return expect_list(node, std::bind(expect_assign, std::placeholders::_1, callback), 0, file_node);
+}
+
+return_t NodeTools::expect_dictionary_keys(ast::NodeCPtr node, dictionary_key_map_t const& keys, bool allow_other_keys, bool file_node) {
+ std::map<std::string, size_t, std::less<void>> key_count;
+ return_t ret = expect_dictionary(node, [keys, allow_other_keys, &key_count](const std::string_view key, ast::NodeCPtr value) -> return_t {
+ const dictionary_key_map_t::const_iterator it = keys.find(key);
+ if (it == keys.end()) {
+ if (allow_other_keys) return SUCCESS;
+ Logger::error("Invalid dictionary key: ", key);
+ return FAILURE;
+ }
+ const size_t count = ++key_count[it->first];
+ dictionary_entry_t const& entry = it->second;
+ if (!entry.can_repeat && count > 1) {
+ Logger::error("Invalid repeat of dictionary key: ", key);
+ return FAILURE;
+ }
+ return entry.callback(value);
+ }, file_node);
+ for (dictionary_key_map_t::value_type const& entry : keys) {
+ if (entry.second.must_appear && key_count.find(entry.first) == key_count.end()) {
+ Logger::error("Mandatory dictionary key not present: ", entry.first);
+ ret = FAILURE;
+ }
+ }
+ return ret;
+}
diff --git a/src/openvic/dataloader/NodeTools.hpp b/src/openvic/dataloader/NodeTools.hpp
new file mode 100644
index 0000000..ca7130c
--- /dev/null
+++ b/src/openvic/dataloader/NodeTools.hpp
@@ -0,0 +1,42 @@
+#pragma once
+
+#include <map>
+
+#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp>
+
+#include "openvic/types/Colour.hpp"
+#include "openvic/types/Return.hpp"
+#include "openvic/types/Date.hpp"
+#include "openvic/types/fixed_point/FP.hpp"
+
+namespace OpenVic {
+ namespace ast = ovdl::v2script::ast;
+
+ namespace NodeTools {
+
+ template<typename T>
+ return_t expect_type(ast::NodeCPtr node, std::function<return_t(T const&)> callback);
+
+ return_t expect_identifier(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback);
+ return_t expect_string(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback);
+ return_t expect_identifier_or_string(ast::NodeCPtr node, std::function<return_t(std::string_view)> callback);
+ return_t expect_bool(ast::NodeCPtr node, std::function<return_t(bool)> callback);
+ return_t expect_int(ast::NodeCPtr node, std::function<return_t(int64_t)> callback);
+ return_t expect_uint(ast::NodeCPtr node, std::function<return_t(uint64_t)> callback);
+ return_t expect_fixed_point(ast::NodeCPtr node, std::function<return_t(FP)> callback);
+ return_t expect_colour(ast::NodeCPtr node, std::function<return_t(colour_t)> callback);
+ return_t expect_date(ast::NodeCPtr node, std::function<return_t(Date)> callback);
+ return_t expect_assign(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback);
+ return_t expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length = 0, bool file_node = false);
+ return_t expect_dictionary(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node = false);
+
+ static const std::function<return_t(ast::NodeCPtr)> success_callback = [](ast::NodeCPtr) -> return_t { return SUCCESS; };
+
+ struct dictionary_entry_t {
+ bool must_appear, can_repeat;
+ std::function<return_t(ast::NodeCPtr)> callback;
+ };
+ using dictionary_key_map_t = std::map<std::string, dictionary_entry_t, std::less<void>>;
+ return_t expect_dictionary_keys(ast::NodeCPtr node, dictionary_key_map_t const& keys, bool allow_other_keys = false, bool file_node = false);
+ }
+}