aboutsummaryrefslogtreecommitdiff
path: root/src/openvic/dataloader
diff options
context:
space:
mode:
Diffstat (limited to 'src/openvic/dataloader')
-rw-r--r--src/openvic/dataloader/Dataloader.cpp136
-rw-r--r--src/openvic/dataloader/Dataloader.hpp16
-rw-r--r--src/openvic/dataloader/NodeTools.cpp62
-rw-r--r--src/openvic/dataloader/NodeTools.hpp25
4 files changed, 171 insertions, 68 deletions
diff --git a/src/openvic/dataloader/Dataloader.cpp b/src/openvic/dataloader/Dataloader.cpp
index 90537c9..f2b725e 100644
--- a/src/openvic/dataloader/Dataloader.cpp
+++ b/src/openvic/dataloader/Dataloader.cpp
@@ -1,10 +1,10 @@
#include "Dataloader.hpp"
+#include "openvic/GameManager.hpp"
#include "openvic/utility/Logger.hpp"
-#include "openvic//GameManager.hpp"
-#include "openvic/dataloader/NodeTools.hpp"
#include <openvic-dataloader/detail/CallbackOStream.hpp>
+#include <openvic-dataloader/v2script/Parser.hpp>
using namespace OpenVic;
using namespace ovdl::v2script;
@@ -14,19 +14,26 @@ return_t Dataloader::set_roots(std::vector<std::filesystem::path> new_roots) {
Logger::error("Overriding existing dataloader roots!");
roots.clear();
}
+ return_t ret = SUCCESS;
for (std::reverse_iterator<std::vector<std::filesystem::path>::const_iterator> it = new_roots.crbegin(); it != new_roots.crend(); ++it) {
- if (std::filesystem::is_directory(*it)) {
- Logger::info("Adding dataloader root: ", *it);
- roots.push_back(*it);
+ if (std::find(roots.begin(), roots.end(), *it) == roots.end()) {
+ if (std::filesystem::is_directory(*it)) {
+ Logger::info("Adding dataloader root: ", *it);
+ roots.push_back(*it);
+ } else {
+ Logger::error("Invalid dataloader root (must be an existing directory): ", *it);
+ ret = FAILURE;
+ }
} else {
- Logger::error("Invalid dataloader root (must be an existing directory): ", *it);
+ Logger::error("Duplicate dataloader root: ", *it);
+ ret = FAILURE;
}
}
if (roots.empty()) {
Logger::error("Dataloader has no roots after attempting to add ", new_roots.size());
- return FAILURE;
+ ret = FAILURE;
}
- return SUCCESS;
+ return ret;
}
std::filesystem::path Dataloader::lookup_file(std::filesystem::path const& path) const {
@@ -40,28 +47,52 @@ std::filesystem::path Dataloader::lookup_file(std::filesystem::path const& path)
return {};
}
-static bool contains_file_with_name(std::vector<std::filesystem::path> const& paths, std::filesystem::path const& name) {
+const std::filesystem::path Dataloader::TXT = ".txt";
+
+static bool contains_file_with_name(std::vector<std::filesystem::path> const& paths,
+ std::filesystem::path const& name) {
+
for (std::filesystem::path const& path : paths) {
if (path.filename() == name) return true;
}
return false;
}
-std::vector<std::filesystem::path> Dataloader::lookup_files_in_dir(std::filesystem::path const& path) const {
+std::vector<std::filesystem::path> Dataloader::lookup_files_in_dir(std::filesystem::path const& path,
+ std::filesystem::path const* extension) const {
+
std::vector<std::filesystem::path> ret;
for (std::filesystem::path const& root : roots) {
const std::filesystem::path composed = root / path;
std::error_code ec;
for (std::filesystem::directory_entry const& entry : std::filesystem::directory_iterator { composed, ec }) {
- if (entry.is_regular_file() && !contains_file_with_name(ret, entry.path().filename())) {
- ret.push_back(entry);
+ if (entry.is_regular_file()) {
+ const std::filesystem::path file = entry;
+ if (extension == nullptr || file.extension() == *extension) {
+ if (!contains_file_with_name(ret, file.filename())) {
+ ret.push_back(file);
+ }
+ }
}
}
}
return ret;
}
-static Parser parse_defines(std::filesystem::path const& path) {
+return_t Dataloader::apply_to_files_in_dir(std::filesystem::path const& path,
+ std::function<return_t(std::filesystem::path const&)> callback,
+ std::filesystem::path const* extension) const {
+
+ return_t ret = SUCCESS;
+ for (std::filesystem::path const& file : lookup_files_in_dir(path, extension)) {
+ if (callback(file) != SUCCESS) {
+ ret = FAILURE;
+ }
+ }
+ return ret;
+}
+
+Parser Dataloader::parse_defines(std::filesystem::path const& path) {
Parser parser;
std::string buffer;
auto error_log_stream = ovdl::detail::CallbackStream {
@@ -81,8 +112,8 @@ static Parser parse_defines(std::filesystem::path const& path) {
Logger::error("Parser load errors:\n\n", buffer, "\n");
buffer.clear();
}
- if (parser.has_fatal_error() || parser.has_error() || parser.has_warning()) {
- Logger::error("Parser warnings/errors while loading ", path);
+ if (parser.has_fatal_error() || parser.has_error()) {
+ Logger::error("Parser errors while loading ", path);
return parser;
}
parser.simple_parse();
@@ -90,51 +121,74 @@ static Parser parse_defines(std::filesystem::path const& path) {
Logger::error("Parser parse errors:\n\n", buffer, "\n");
buffer.clear();
}
- if (parser.has_fatal_error() || parser.has_error() || parser.has_warning()) {
- Logger::error("Parser warnings/errors while parsing ", path);
+ if (parser.has_fatal_error() || parser.has_error()) {
+ Logger::error("Parser errors while parsing ", path);
}
return parser;
}
+Parser Dataloader::parse_defines_lookup(std::filesystem::path const& path) const {
+ return parse_defines(lookup_file(path));
+}
+
+return_t Dataloader::_load_pop_types(PopManager& pop_manager, std::filesystem::path const& pop_type_directory) const {
+ return_t ret = SUCCESS;
+ if (apply_to_files_in_dir(pop_type_directory, [&pop_manager](std::filesystem::path const& file) -> return_t {
+ return pop_manager.load_pop_type_file(file, parse_defines(file).get_file_node());
+ }) != SUCCESS) {
+ Logger::error("Failed to load pop types!");
+ ret = FAILURE;
+ }
+ pop_manager.lock_pop_types();
+ return ret;
+}
+
return_t Dataloader::load_defines(GameManager& game_manager) const {
+ static const std::filesystem::path good_file = "common/goods.txt";
+ static const std::filesystem::path pop_type_directory = "poptypes";
static const std::filesystem::path graphical_culture_type_file = "common/graphicalculturetype.txt";
static const std::filesystem::path culture_file = "common/cultures.txt";
+ static const std::filesystem::path religion_file = "common/religion.txt";
return_t ret = SUCCESS;
- if (game_manager.pop_manager.culture_manager.load_graphical_culture_type_file(parse_defines(
- lookup_file(graphical_culture_type_file)).get_file_node()) != SUCCESS) {
+ if (game_manager.good_manager.load_good_file(parse_defines_lookup(good_file).get_file_node()) != SUCCESS) {
+ Logger::error("Failed to load goods!");
+ ret = FAILURE;
+ }
+ if (_load_pop_types(game_manager.pop_manager, pop_type_directory) != SUCCESS) {
+ Logger::error("Failed to load pop types!");
+ ret = FAILURE;
+ }
+ if (game_manager.pop_manager.culture_manager.load_graphical_culture_type_file(parse_defines_lookup(graphical_culture_type_file).get_file_node()) != SUCCESS) {
Logger::error("Failed to load graphical culture types!");
ret = FAILURE;
}
- if (game_manager.pop_manager.culture_manager.load_culture_file(parse_defines(
- lookup_file(culture_file)).get_file_node()) != SUCCESS) {
+ if (game_manager.pop_manager.culture_manager.load_culture_file(parse_defines_lookup(culture_file).get_file_node()) != SUCCESS) {
Logger::error("Failed to load cultures!");
ret = FAILURE;
}
+ if (game_manager.pop_manager.religion_manager.load_religion_file(parse_defines_lookup(religion_file).get_file_node()) != SUCCESS) {
+ Logger::error("Failed to load religions!");
+ ret = FAILURE;
+ }
return ret;
}
-static return_t load_pop_history_file(GameManager& game_manager, std::filesystem::path const& path) {
- return NodeTools::expect_dictionary(parse_defines(path).get_file_node(), [&game_manager](std::string_view province_key, ast::NodeCPtr province_node) -> return_t {
- Province* province = game_manager.map.get_province_by_identifier(province_key);
- if (province == nullptr) {
- Logger::error("Invalid province id: ", province_key);
- return FAILURE;
- }
- return NodeTools::expect_list(province_node, [&game_manager, &province](ast::NodeCPtr pop_node) -> return_t {
- return game_manager.pop_manager.load_pop_into_province(*province, pop_node);
- });
- }, true);
-}
-
return_t Dataloader::load_pop_history(GameManager& game_manager, std::filesystem::path const& path) const {
- return_t ret = SUCCESS;
- for (std::filesystem::path const& file : lookup_files_in_dir(path)) {
- if (load_pop_history_file(game_manager, file) != SUCCESS) {
- ret = FAILURE;
- }
- }
- return ret;
+ return apply_to_files_in_dir(path, [&game_manager](std::filesystem::path const& file) -> return_t {
+ return NodeTools::expect_dictionary(parse_defines(file).get_file_node(),
+ [&game_manager](std::string_view province_key, ast::NodeCPtr province_node) -> return_t {
+ Province* province = game_manager.map.get_province_by_identifier(province_key);
+ if (province == nullptr) {
+ Logger::error("Invalid province id: ", province_key);
+ return FAILURE;
+ }
+ return NodeTools::expect_list(province_node, [&game_manager, &province](ast::NodeCPtr pop_node) -> return_t {
+ return game_manager.pop_manager.load_pop_into_province(*province, pop_node);
+ }
+ );
+ }, true);
+ });
}
diff --git a/src/openvic/dataloader/Dataloader.hpp b/src/openvic/dataloader/Dataloader.hpp
index b080300..3c868a3 100644
--- a/src/openvic/dataloader/Dataloader.hpp
+++ b/src/openvic/dataloader/Dataloader.hpp
@@ -1,14 +1,23 @@
#pragma once
+#include <filesystem>
+
#include "openvic/types/Return.hpp"
+
#include "openvic-dataloader/v2script/Parser.hpp"
namespace OpenVic {
struct GameManager;
+ struct PopManager;
class Dataloader {
std::vector<std::filesystem::path> roots;
+ static ovdl::v2script::Parser parse_defines(std::filesystem::path const& path);
+ ovdl::v2script::Parser parse_defines_lookup(std::filesystem::path const& path) const;
+
+ return_t _load_pop_types(PopManager& pop_manager, std::filesystem::path const& pop_type_directory) const;
+
public:
Dataloader() = default;
@@ -16,7 +25,12 @@ namespace OpenVic {
return_t set_roots(std::vector<std::filesystem::path> new_roots);
std::filesystem::path lookup_file(std::filesystem::path const& path) const;
- std::vector<std::filesystem::path> lookup_files_in_dir(std::filesystem::path const& path) const;
+ static const std::filesystem::path TXT;
+ std::vector<std::filesystem::path> lookup_files_in_dir(std::filesystem::path const& path,
+ std::filesystem::path const* extension = &TXT) const;
+ return_t apply_to_files_in_dir(std::filesystem::path const& path,
+ std::function<return_t(std::filesystem::path const&)> callback,
+ std::filesystem::path const* extension = &TXT) const;
return_t load_defines(GameManager& game_manager) const;
return_t load_pop_history(GameManager& game_manager, std::filesystem::path const& path) const;
diff --git a/src/openvic/dataloader/NodeTools.cpp b/src/openvic/dataloader/NodeTools.cpp
index a37892f..6ab2ba6 100644
--- a/src/openvic/dataloader/NodeTools.cpp
+++ b/src/openvic/dataloader/NodeTools.cpp
@@ -66,11 +66,9 @@ return_t NodeTools::expect_bool(ast::NodeCPtr node, std::function<return_t(bool)
return_t NodeTools::expect_int(ast::NodeCPtr node, std::function<return_t(int64_t)> callback) {
return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
- int64_t val;
- char const* const start = identifier.data();
- char const* const end = start + identifier.size();
- const std::from_chars_result result = std::from_chars(start, end, val);
- if (result.ec == std::errc{} && result.ptr == end) {
+ bool successful = false;
+ const int64_t val = StringUtils::string_to_int64(identifier, &successful, 10);
+ if (successful) {
return callback(val);
}
Logger::error("Invalid int identifier text: ", identifier);
@@ -80,10 +78,9 @@ return_t NodeTools::expect_int(ast::NodeCPtr node, std::function<return_t(int64_
return_t NodeTools::expect_uint(ast::NodeCPtr node, std::function<return_t(uint64_t)> callback) {
return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
- uint64_t val;
- char const* identifier_end = identifier.data() + identifier.size();
- const std::from_chars_result result = std::from_chars(identifier.data(), identifier_end, val);
- if (result.ec == std::errc{} && result.ptr == identifier_end) {
+ bool successful = false;
+ const uint64_t val = StringUtils::string_to_uint64(identifier, &successful, 10);
+ if (successful) {
return callback(val);
}
Logger::error("Invalid uint identifier text: ", identifier);
@@ -94,7 +91,7 @@ return_t NodeTools::expect_uint(ast::NodeCPtr node, std::function<return_t(uint6
return_t NodeTools::expect_fixed_point(ast::NodeCPtr node, std::function<return_t(FP)> callback) {
return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
bool successful = false;
- FP val = FP::parse(identifier.data(), identifier.length(), &successful);
+ const FP val = FP::parse(identifier.data(), identifier.length(), &successful);
if (successful) {
return callback(val);
}
@@ -106,7 +103,7 @@ return_t NodeTools::expect_fixed_point(ast::NodeCPtr node, std::function<return_
return_t NodeTools::expect_colour(ast::NodeCPtr node, std::function<return_t(colour_t)> callback) {
colour_t col = NULL_COLOUR;
uint32_t components = 0;
- return_t ret = expect_list(node, std::bind(expect_fixed_point, std::placeholders::_1,
+ return_t ret = expect_list_of_length(node, std::bind(expect_fixed_point, std::placeholders::_1,
[&col, &components](FP val) -> return_t {
return_t ret = SUCCESS;
if (val < 0 || val > 255) {
@@ -127,7 +124,7 @@ return_t NodeTools::expect_colour(ast::NodeCPtr node, std::function<return_t(col
return_t NodeTools::expect_date(ast::NodeCPtr node, std::function<return_t(Date)> callback) {
return expect_identifier(node, [callback](std::string_view identifier) -> return_t {
bool successful = false;
- Date date = Date::from_string(identifier, &successful);
+ const Date date = Date::from_string(identifier, &successful);
if (successful) {
return callback(date);
}
@@ -142,16 +139,16 @@ return_t NodeTools::expect_assign(ast::NodeCPtr node, std::function<return_t(std
});
}
-return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node) {
- const std::function<return_t(std::vector<ast::NodeUPtr> const&)> list_func = [length, callback](std::vector<ast::NodeUPtr> const& list) -> return_t {
+return_t NodeTools::expect_list_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(ast::NodeCPtr)> callback, bool file_node) {
+ const std::function<return_t(std::vector<ast::NodeUPtr> const&)> list_func = [length_callback, callback](std::vector<ast::NodeUPtr> const& list) -> return_t {
return_t ret = SUCCESS;
- size_t size = list.size();
- if (length > 0 && size != length) {
- Logger::error("List length ", size, " does not match expected length ", length);
+ size_t size = length_callback(list.size());
+ if (size > list.size()) {
+ Logger::error("Trying to read more values than the list contains: ", size, " > ", list.size());
+ size = list.size();
ret = FAILURE;
- if (length < size) size = length;
}
- std::for_each(list.begin(), list.begin() + size, [callback, &ret](ast::NodeUPtr const& sub_node) {
+ std::for_each(list.begin(), list.begin() + size, [callback, &ret](ast::NodeUPtr const& sub_node) -> void {
if (callback(sub_node.get()) != SUCCESS) ret = FAILURE;
});
return ret;
@@ -167,13 +164,34 @@ return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast::
}
}
-return_t NodeTools::expect_dictionary(ast::NodeCPtr node, std::function<return_t(const std::string_view, ast::NodeCPtr)> callback, bool file_node) {
- return expect_list(node, std::bind(expect_assign, std::placeholders::_1, callback), 0, file_node);
+return_t NodeTools::expect_list_of_length(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node) {
+ return_t ret = SUCCESS;
+ if (expect_list_and_length(node, [length, &ret](size_t size) -> size_t {
+ if (size != length) {
+ Logger::error("List length ", size, " does not match expected length ", length);
+ ret = FAILURE;
+ if (length < size) return length;
+ }
+ return size;
+ }, callback, file_node) != SUCCESS) ret = FAILURE;
+ return ret;
+}
+
+return_t NodeTools::expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, bool file_node) {
+ return expect_list_and_length(node, default_length_callback, callback, file_node);
+}
+
+return_t NodeTools::expect_dictionary_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node) {
+ return expect_list_and_length(node, length_callback, std::bind(expect_assign, std::placeholders::_1, callback), file_node);
+}
+
+return_t NodeTools::expect_dictionary(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node) {
+ return expect_dictionary_and_length(node, default_length_callback, callback, file_node);
}
return_t NodeTools::expect_dictionary_keys(ast::NodeCPtr node, dictionary_key_map_t const& keys, bool allow_other_keys, bool file_node) {
std::map<std::string, size_t, std::less<void>> key_count;
- return_t ret = expect_dictionary(node, [keys, allow_other_keys, &key_count](const std::string_view key, ast::NodeCPtr value) -> return_t {
+ return_t ret = expect_dictionary(node, [keys, allow_other_keys, &key_count](std::string_view key, ast::NodeCPtr value) -> return_t {
const dictionary_key_map_t::const_iterator it = keys.find(key);
if (it == keys.end()) {
if (allow_other_keys) return SUCCESS;
diff --git a/src/openvic/dataloader/NodeTools.hpp b/src/openvic/dataloader/NodeTools.hpp
index ca7130c..48c3710 100644
--- a/src/openvic/dataloader/NodeTools.hpp
+++ b/src/openvic/dataloader/NodeTools.hpp
@@ -2,13 +2,13 @@
#include <map>
-#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp>
-
#include "openvic/types/Colour.hpp"
-#include "openvic/types/Return.hpp"
#include "openvic/types/Date.hpp"
+#include "openvic/types/Return.hpp"
#include "openvic/types/fixed_point/FP.hpp"
+#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp>
+
namespace OpenVic {
namespace ast = ovdl::v2script::ast;
@@ -27,7 +27,24 @@ namespace OpenVic {
return_t expect_colour(ast::NodeCPtr node, std::function<return_t(colour_t)> callback);
return_t expect_date(ast::NodeCPtr node, std::function<return_t(Date)> callback);
return_t expect_assign(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback);
- return_t expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length = 0, bool file_node = false);
+
+ static const std::function<size_t(size_t)> default_length_callback = [](size_t size) -> size_t { return size; };
+
+ template<typename T> requires requires(T& t) {
+ { t.size() } -> std::same_as<size_t>;
+ t.reserve( size_t {} );
+ }
+ std::function<size_t(size_t)> reserve_length_callback(T& t) {
+ return [&t](size_t size) -> size_t {
+ t.reserve(t.size() + size);
+ return size;
+ };
+ }
+
+ return_t expect_list_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(ast::NodeCPtr)> callback, bool file_node = false);
+ return_t expect_list_of_length(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, size_t length, bool file_node = false);
+ return_t expect_list(ast::NodeCPtr node, std::function<return_t(ast::NodeCPtr)> callback, bool file_node = false);
+ return_t expect_dictionary_and_length(ast::NodeCPtr node, std::function<size_t(size_t)> length_callback, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node = false);
return_t expect_dictionary(ast::NodeCPtr node, std::function<return_t(std::string_view, ast::NodeCPtr)> callback, bool file_node = false);
static const std::function<return_t(ast::NodeCPtr)> success_callback = [](ast::NodeCPtr) -> return_t { return SUCCESS; };
generated by our cgit fork (git 2.39.0) at 2024-01-23 02:24:14 +0100