diff options
author | 2023-08-26 00:25:12 +0200 | |
---|---|---|
committer | 2023-09-03 12:53:52 +0200 | |
commit | 366f1b3941315641bdcb0ee98465b4d2134eee86 (patch) | |
tree | ebb1e95b5e874c6eab09f19a323d721fd1fdac1b /src/openvic/dataloader | |
parent | efa88c722fcb6c8fea7a86e1b3b8a83f1f59eb31 (diff) |
Followup big dataloader commit
Diffstat (limited to 'src/openvic/dataloader')
-rw-r--r-- | src/openvic/dataloader/Dataloader.cpp | 136 | ||||
-rw-r--r-- | src/openvic/dataloader/Dataloader.hpp | 16 | ||||
-rw-r--r-- | src/openvic/dataloader/NodeTools.cpp | 62 | ||||
-rw-r--r-- | src/openvic/dataloader/NodeTools.hpp | 25 |
4 files changed, 171 insertions, 68 deletions
diff --git a/src/openvic/dataloader/Dataloader.cpp b/src/openvic/dataloader/Dataloader.cpp index 90537c9..f2b725e 100644 --- a/src/openvic/dataloader/Dataloader.cpp +++ b/src/openvic/dataloader/Dataloader.cpp @@ -1,10 +1,10 @@ #include "Dataloader.hpp" +#include "openvic/GameManager.hpp" #include "openvic/utility/Logger.hpp" -#include "openvic//GameManager.hpp" -#include "openvic/dataloader/NodeTools.hpp" #include <openvic-dataloader/detail/CallbackOStream.hpp> +#include <openvic-dataloader/v2script/Parser.hpp> using namespace OpenVic; using namespace ovdl::v2script; @@ -14,19 +14,26 @@ return_t Dataloader::set_roots(std::vector<std::filesystem::path> new_roots) { Logger::error("Overriding existing dataloader roots!"); roots.clear(); } + return_t ret = SUCCESS; for (std::reverse_iterator<std::vector<std::filesystem::path>::const_iterator> it = new_roots.crbegin(); it != new_roots.crend(); ++it) { - if (std::filesystem::is_directory(*it)) { - Logger::info("Adding dataloader root: ", *it); - roots.push_back(*it); + if (std::find(roots.begin(), roots.end(), *it) == roots.end()) { + if (std::filesystem::is_directory(*it)) { + Logger::info("Adding dataloader root: ", *it); + roots.push_back(*it); + } else { + Logger::error("Invalid dataloader root (must be an existing directory): ", *it); + ret = FAILURE; + } } else { - Logger::error("Invalid dataloader root (must be an existing directory): ", *it); + Logger::error("Duplicate dataloader root: ", *it); + ret = FAILURE; } } if (roots.empty()) { Logger::error("Dataloader has no roots after attempting to add ", new_roots.size()); - return FAILURE; + ret = FAILURE; } - return SUCCESS; + return ret; } std::filesystem::path Dataloader::lookup_file(std::filesystem::path const& path) const { @@ -40,28 +47,52 @@ std::filesystem::path Dataloader::lookup_file(std::filesystem::path const& path) return {}; } -static bool contains_file_with_name(std::vector<std::filesystem::path> const& paths, std::filesystem::path const& name) { +const std::filesystem::path Dataloader::TXT = ".txt"; + +static bool contains_file_with_name(std::vector<std::filesystem::path> const& paths, + std::filesystem::path const& name) { + for (std::filesystem::path const& path : paths) { if (path.filename() == name) return true; } return false; } -std::vector<std::filesystem::path> Dataloader::lookup_files_in_dir(std::filesystem::path const& path) const { +std::vector<std::filesystem::path> Dataloader::lookup_files_in_dir(std::filesystem::path const& path, + std::filesystem::path const* extension) const { + std::vector<std::filesystem::path> ret; for (std::filesystem::path const& root : roots) { const std::filesystem::path composed = root / path; std::error_code ec; for (std::filesystem::directory_entry const& entry : std::filesystem::directory_iterator { composed, ec }) { - if (entry.is_regular_file() && !contains_file_with_name(ret, entry.path().filename())) { - ret.push_back(entry); + if (entry.is_regular_file()) { + const std::filesystem::path file = entry; + if (extension == nullptr || file.extension() == *extension) { + if (!contains_file_with_name(ret, file.filename())) { + ret.push_back(file); + } + } } } } return ret; } -static Parser parse_defines(std::filesystem::path const& path) { +return_t Dataloader::apply_to_files_in_dir(std::filesystem::path const& path, + std::function<return_t(std::filesystem::path const&)> callback, + std::filesystem::path const* extension) const { + + return_t ret = SUCCESS; + for (std::filesystem::path const& file : lookup_files_in_dir(path, extension)) { + if (callback(file) != SUCCESS) { + ret = FAILURE; + } + } + return ret; +} + +Parser Dataloader::parse_defines(std::filesystem::path const& path) { Parser parser; std::string buffer; auto error_log_stream = ovdl::detail::CallbackStream { @@ -81,8 +112,8 @@ static Parser parse_defines(std::filesystem::path const& path) { Logger::error("Parser load errors:\n\n", buffer, "\n"); buffer.clear(); } - if (parser.has_fatal_error() || parser.has_error() || parser.has_warning()) { - Logger::error("Parser warnings/errors while loading ", path); + if (parser.has_fatal_error() || parser.has_error()) { + Logger::error("Parser errors while loading ", path); return parser; } parser.simple_parse(); @@ -90,51 +121,74 @@ static Parser parse_defines(std::filesystem::path const& path) { Logger::error("Parser parse errors:\n\n", buffer, "\n"); buffer.clear(); } - if (parser.has_fatal_error() || parser.has_error() || parser.has_warning()) { - Logger::error("Parser warnings/errors while parsing ", path); + if (parser.has_fatal_error() || parser.has_error()) { + Logger::error("Parser errors while parsing ", path); } return parser; } +Parser Dataloader::parse_defines_lookup(std::filesystem::path const& path) const { + return parse_defines(lookup_file(path)); +} + +return_t Dataloader::_load_pop_types(PopManager& pop_manager, std::filesystem::path const& pop_type_directory) const { + return_t ret = SUCCESS; + if (apply_to_files_in_dir(pop_type_directory, [&pop_manager](std::filesystem::path const& file) -> return_t { + return pop_manager.load_pop_type_file(file, parse_defines(file).get_file_node()); + }) != SUCCESS) { + Logger::error("Failed to load pop types!"); + ret = FAILURE; + } + pop_manager.lock_pop_types(); + return ret; +} + return_t Dataloader::load_defines(GameManager& game_manager) const { + static const std::filesystem::path good_file = "common/goods.txt"; + static const std::filesystem::path pop_type_directory = "poptypes"; static const std::filesystem::path graphical_culture_type_file = "common/graphicalculturetype.txt"; static const std::filesystem::path culture_file = "common/cultures.txt"; + static const std::filesystem::path religion_file = "common/religion.txt"; return_t ret = SUCCESS; - if (game_manager.pop_manager.culture_manager.load_graphical_culture_type_file(parse_defines( - lookup_file(graphical_culture_type_file)).get_file_node()) != SUCCESS) { + if (game_manager.good_manager.load_good_file(parse_defines_lookup(good_file).get_file_node()) != SUCCESS) { + Logger::error("Failed to load goods!"); + ret = FAILURE; + } + if (_load_pop_types(game_manager.pop_manager, pop_type_directory) != SUCCESS) { + Logger::error("Failed to load pop types!"); + ret = FAILURE; + } + if (game_manager.pop_manager.culture_manager.load_graphical_culture_type_file(parse_defines_lookup(graphical_culture_type_file).get_file_node()) != SUCCESS) { Logger::error("Failed to load graphical culture types!"); ret = FAILURE; } - if (game_manager.pop_manager.culture_manager.load_culture_file(parse_defines( - lookup_file(culture_file)).get_file_node()) != SUCCESS) { + if (game_manager.pop_manager.culture_manager.load_culture_file(parse_defines_lookup(culture_file).get_file_node()) != SUCCESS) { Logger::error("Failed to load cultures!"); ret = FAILURE; } + if (game_manager.pop_manager.religion_manager.load_religion_file(parse_defines_lookup(religion_file).get_file_node()) != SUCCESS) { + Logger::error("Failed to load religions!"); + ret = FAILURE; + } return ret; } -static return_t load_pop_history_file(GameManager& game_manager, std::filesystem::path const& path) { - return NodeTools::expect_dictionary(parse_defines(path).get_file_node(), [&game_manager](std::string_view province_key, ast::NodeCPtr province_node) -> return_t { - Province* province = game_manager.map.get_province_by_identifier(province_key); - if (province == nullptr) { - Logger::error("Invalid province id: ", province_key); - return FAILURE; - } - return NodeTools::expect_list(province_node, [&game_manager, &province](ast::NodeCPtr pop_node) -> return_t { - return game_manager.pop_manager.load_pop_into_province(*province, pop_node); - }); - }, true); -} - return_t Dataloader::load_pop_history(GameManager& game_manager, std::filesystem::path const& path) const { - return_t ret = SUCCESS; - for (std::filesystem::path const& file : lookup_files_in_dir(path)) { - if (load_pop_history_file(game_manager, file) != SUCCESS) { - ret = FAILURE; - } - } - return ret; + return apply_to_files_in_dir(path, [&game_manager](std::filesystem::path const& file) -> return_t { + return NodeTools::expect_dictionary(parse_defines(file).get_file_node(), + [&game_manager](std::string_view province_key, ast::NodeCPtr province_node) -> return_t { + Province* province = game_manager.map.get_province_by_identifier(province_key); + if (province == nullptr) { + Logger::error("Invalid province id: ", province_key); + return FAILURE; + } + return NodeTools::expect_list(province_node, [&game_manager, &province](ast::NodeCPtr pop_node) -> return_t { + return game_manager.pop_manager.load_pop_into_province(*province, pop_node); + } + ); + }, true); + }); } diff --git a/src/openvic/dataloader/Dataloader.hpp b/src/openvic/dataloader/Dataloader.hpp index b080300..3c868a3 100644 --- a/src/openvic/dataloader/Dataloader.hpp +++ b/src/openvic/dataloader/Dataloader.hpp @@ -1,14 +1,23 @@ #pragma once +#include <filesystem> + #include "openvic/types/Return.hpp" + #include "openvic-dataloader/v2script/Parser.hpp" namespace OpenVic { struct GameManager; + struct PopManager; class Dataloader { std::vector<std::filesystem::path> roots; + static ovdl::v2script::Parser parse_defines(std::filesystem::path const& path); + ovdl::v2script::Parser parse_defines_lookup(std::filesystem::path const& path) const; + + return_t _load_pop_types(PopManager& pop_manager, std::filesystem::path const& pop_type_directory) const; + public: Dataloader() = default; @@ -16,7 +25,12 @@ namespace OpenVic { return_t set_roots(std::vector<std::filesystem::path> new_roots); std::filesystem::path lookup_file(std::filesystem::path const& path) const; - std::vector<std::filesystem::path> lookup_files_in_dir(std::filesystem::path const& path) const; + static const std::filesystem::path TXT; + std::vector<std::filesystem::path> lookup_files_in_dir(std::filesystem::path const& path, + std::filesystem::path const* extension = &TXT) const; + return_t apply_to_files_in_dir(std::filesystem::path const& path, + std::function<return_t(std::filesystem::path const&)> callback, + std::filesystem::path const* extension = &TXT) const; return_t load_defines(GameManager& game_manager) const; return_t load_pop_history(GameManager& game_manager, std::filesystem::path const& path) const; diff --git a/src/openvic/dataloader/NodeTools.cpp b/src/openvic/dataloader/NodeTools.cpp index a37892f..6ab2ba6 100644 --- a/src/openvic/dataloader/NodeTools.cpp +++ b/src/openvic/dataloader/NodeTools.cpp @@ -66,11 +66,9 @@ return_t NodeTools::expect_bool(ast::NodeCPtr node, std::function<return_t(bool) return_t NodeTools::expect_int(ast::NodeCPtr node, std::function<return_t(int64_t)> callback) { return expect_identifier(node, [callback](std::string_view identifier) -> return_t { - int64_t val; - char const* const start = identifier.data(); - char const* const end = start + identifier.size(); - const std::from_chars_result result = std::from_chars(start, end, val); - if (result.ec == std::errc{} && result.ptr == end) { + bool successful = false; + const int64_t val = StringUtils::string_to_int64(identifier, &successful, 10); + if (successful) { return callback(val); } Logger::error("Invalid int identifier text: ", identifier); @@ -80,10 +78,9 @@ return_t NodeTools::expect_int(ast::NodeCPtr node, std::function<return_t(int64_ return_t NodeTools::expect_uint(ast::NodeCPtr node, std::function<return_t(uint64_t)> callback) { return expect_identifier(node, [callback](std::string_view identifier) -> return_t { - uint64_t val; - char const* identifier_end = identifier.data() + identifier.size(); - const std::from_chars_result result = std::from_chars(identifier.data(), identifier_end, val); - if (result.ec == std::errc{} && result.ptr == identifier_end) { + bool successful = false; + const uint64_t val = StringUtils::string_to_uint64(identifier, &successful, 10); + if (successful) { return callback(val); } Logger::error("Invalid uint identifier text: ", identifier); @@ -94,7 +91,7 @@ return_t NodeTools::expect_uint(ast::NodeCPtr node, std::function<return_t(uint6 return_t NodeTools::expect_fixed_point(ast::NodeCPtr node, std::function<return_t(FP)> callback) { return expect_identifier(node, [callback](std::string_view identifier) -> return_t { bool successful = false; - FP val = FP::parse(identifier.data(), identifier.length(), &successful); + const FP val = FP::parse(identifier.data(), identifier.length(), &successful); if (successful) { return callback(val); } @@ -106,7 +103,7 @@ return_t NodeTools::expect_fixed_point(ast::NodeCPtr node, std::function<return_ return_t NodeTools::expect_colour(ast::NodeCPtr node, std::function<return_t(colour_t)> callback) { colour_t col = NULL_COLOUR; uint32_t components = 0; - return_t ret = expect_list(node, std::bind(expect_fixed_point, std::placeholders::_1, + return_t ret = expect_list_of_length(node, std::bind(expect_fixed_point, std::placeholders::_1, [&col, &components](FP val) -> return_t { return_t ret = SUCCESS; if (val < 0 || val > 255) { @@ -127,7 +124,7 @@ return_t NodeTools::expect_colour(ast::NodeCPtr node, std::function<return_t(col return_t NodeTools::expect_date(ast::NodeCPtr node, std::function<return_t(Date)> callback) { return expect_identifier(node, [callback](std::string_view identifier) -> return_t { bool successful = false; - Date date = Date::from_string(identifier, &successful); + const Date date = Date::from_string(identifier, &successful); if (successful) { return callback(date); } @@ -142,16 +139,16 @@ return_t NodeTools::expect_assign(ast::NodeCPtr node, std::function<return_t(std }); } -return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node) { - const std::function<return_t(std::vector<ast::NodeUPtr> const&)> list_func = [length, callback](std::vector<ast::NodeUPtr> const& list) -> return_t { +return_t NodeTools::expect_list_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(ast::NodeCPtr)> callback, bool file_node) { + const std::function<return_t(std::vector<ast::NodeUPtr> const&)> list_func = [length_callback, callback](std::vector<ast::NodeUPtr> const& list) -> return_t { return_t ret = SUCCESS; - size_t size = list.size(); - if (length > 0 && size != length) { - Logger::error("List length ", size, " does not match expected length ", length); + size_t size = length_callback(list.size()); + if (size > list.size()) { + Logger::error("Trying to read more values than the list contains: ", size, " > ", list.size()); + size = list.size(); ret = FAILURE; - if (length < size) size = length; } - std::for_each(list.begin(), list.begin() + size, [callback, &ret](ast::NodeUPtr const& sub_node) { + std::for_each(list.begin(), list.begin() + size, [callback, &ret](ast::NodeUPtr const& sub_node) -> void { if (callback(sub_node.get()) != SUCCESS) ret = FAILURE; }); return ret; @@ -167,13 +164,34 @@ return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast:: } } -return_t NodeTools::expect_dictionary(ast::NodeCPtr node, std::function<return_t(const std::string_view, ast::NodeCPtr)> callback, bool file_node) { - return expect_list(node, std::bind(expect_assign, std::placeholders::_1, callback), 0, file_node); +return_t NodeTools::expect_list_of_length(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node) { + return_t ret = SUCCESS; + if (expect_list_and_length(node, [length, &ret](size_t size) -> size_t { + if (size != length) { + Logger::error("List length ", size, " does not match expected length ", length); + ret = FAILURE; + if (length < size) return length; + } + return size; + }, callback, file_node) != SUCCESS) ret = FAILURE; + return ret; +} + +return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, bool file_node) { + return expect_list_and_length(node, default_length_callback, callback, file_node); +} + +return_t NodeTools::expect_dictionary_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node) { + return expect_list_and_length(node, length_callback, std::bind(expect_assign, std::placeholders::_1, callback), file_node); +} + +return_t NodeTools::expect_dictionary(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node) { + return expect_dictionary_and_length(node, default_length_callback, callback, file_node); } return_t NodeTools::expect_dictionary_keys(ast::NodeCPtr node, dictionary_key_map_t const& keys, bool allow_other_keys, bool file_node) { std::map<std::string, size_t, std::less<void>> key_count; - return_t ret = expect_dictionary(node, [keys, allow_other_keys, &key_count](const std::string_view key, ast::NodeCPtr value) -> return_t { + return_t ret = expect_dictionary(node, [keys, allow_other_keys, &key_count](std::string_view key, ast::NodeCPtr value) -> return_t { const dictionary_key_map_t::const_iterator it = keys.find(key); if (it == keys.end()) { if (allow_other_keys) return SUCCESS; diff --git a/src/openvic/dataloader/NodeTools.hpp b/src/openvic/dataloader/NodeTools.hpp index ca7130c..48c3710 100644 --- a/src/openvic/dataloader/NodeTools.hpp +++ b/src/openvic/dataloader/NodeTools.hpp @@ -2,13 +2,13 @@ #include <map> -#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp> - #include "openvic/types/Colour.hpp" -#include "openvic/types/Return.hpp" #include "openvic/types/Date.hpp" +#include "openvic/types/Return.hpp" #include "openvic/types/fixed_point/FP.hpp" +#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp> + namespace OpenVic { namespace ast = ovdl::v2script::ast; @@ -27,7 +27,24 @@ namespace OpenVic { return_t expect_colour(ast::NodeCPtr node, std::function<return_t(colour_t)> callback); return_t expect_date(ast::NodeCPtr node, std::function<return_t(Date)> callback); return_t expect_assign(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback); - return_t expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length = 0, bool file_node = false); + + static const std::function<size_t(size_t)> default_length_callback = [](size_t size) -> size_t { return size; }; + + template<typename T> requires requires(T& t) { + { t.size() } -> std::same_as<size_t>; + t.reserve( size_t {} ); + } + std::function<size_t(size_t)> reserve_length_callback(T& t) { + return [&t](size_t size) -> size_t { + t.reserve(t.size() + size); + return size; + }; + } + + return_t expect_list_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(ast::NodeCPtr)> callback, bool file_node = false); + return_t expect_list_of_length(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node = false); + return_t expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, bool file_node = false); + return_t expect_dictionary_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node = false); return_t expect_dictionary(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node = false); static const std::function<return_t(ast::NodeCPtr)> success_callback = [](ast::NodeCPtr) -> return_t { return SUCCESS; }; |