diff options
Diffstat (limited to 'src/openvic-simulation/dataloader')
| -rw-r--r-- | src/openvic-simulation/dataloader/Dataloader.cpp | 65 | ||||
| -rw-r--r-- | src/openvic-simulation/dataloader/Dataloader.hpp | 2 |
2 files changed, 42 insertions, 25 deletions
diff --git a/src/openvic-simulation/dataloader/Dataloader.cpp b/src/openvic-simulation/dataloader/Dataloader.cpp index 382c7aa..4687925 100644 --- a/src/openvic-simulation/dataloader/Dataloader.cpp +++ b/src/openvic-simulation/dataloader/Dataloader.cpp @@ -1,6 +1,5 @@ #include "Dataloader.hpp" -#include <filesystem> #include <openvic-dataloader/csv/Parser.hpp> #include <openvic-dataloader/detail/CallbackOStream.hpp> #include <openvic-dataloader/v2script/Parser.hpp> @@ -26,6 +25,15 @@ using StringUtils::append_string_views; #define FILESYSTEM_NEEDS_FORWARD_SLASHES #endif +static fs::path ensure_forward_slash_path(std::string_view path) { +#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES) + /* Back-slashes need to be converted into forward-slashes */ + return StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path)); +#else + return path; +#endif +} + static constexpr bool path_equals_case_insensitive(std::string_view lhs, std::string_view rhs) { constexpr auto ichar_equals = [](unsigned char l, unsigned char r) { return std::tolower(l) == std::tolower(r); @@ -61,13 +69,7 @@ bool Dataloader::set_roots(path_vector_t const& new_roots) { } fs::path Dataloader::lookup_file(std::string_view path, bool print_error) const { -#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES) - /* Back-slashes need to be converted into forward-slashes */ - const std::string forward_slash_path { StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path)) }; - path = forward_slash_path; -#endif - - const fs::path filepath { path }; + const fs::path filepath { ensure_forward_slash_path(path) }; #if defined(FILESYSTEM_CASE_INSENSITIVE) /* Case-insensitive filesystem */ @@ -121,12 +123,7 @@ requires requires (_UniqueKey const& unique_key, std::string_view path) { Dataloader::path_vector_t Dataloader::_lookup_files_in_dir( std::string_view path, fs::path const& extension, _UniqueKey const& unique_key ) const { -#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES) - /* Back-slashes need to be converted into forward-slashes */ - const std::string forward_slash_path { StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path)) }; - path = forward_slash_path; -#endif - const fs::path dirpath { path }; + const fs::path dirpath { ensure_forward_slash_path(path) }; path_vector_t ret; struct file_entry_t { fs::path file; @@ -135,9 +132,8 @@ Dataloader::path_vector_t Dataloader::_lookup_files_in_dir( string_map_t<file_entry_t> found_files; for (fs::path const& root : roots) { const size_t root_len = root.string().size(); - const fs::path composed = root / dirpath; std::error_code ec; - for (fs::directory_entry const& entry : _DirIterator { composed, ec }) { + for (fs::directory_entry const& entry : _DirIterator { root / dirpath, ec }) { if (entry.is_regular_file()) { fs::path file = entry; if ((extension.empty() || file.extension() == extension)) { @@ -200,6 +196,20 @@ bool Dataloader::apply_to_files(path_vector_t const& files, callback_t<fs::path return ret; } +string_set_t Dataloader::lookup_dirs_in_dir(std::string_view path) const { + const fs::path dirpath { ensure_forward_slash_path(path) }; + string_set_t ret; + for (fs::path const& root : roots) { + std::error_code ec; + for (fs::directory_entry const& entry : fs::directory_iterator { root / dirpath, ec }) { + if (entry.is_directory()) { + ret.emplace(entry.path().filename().string()); + } + } + } + return ret; +} + template<std::derived_from<detail::BasicParser> Parser, bool (*parse_func)(Parser&)> static Parser _run_ovdl_parser(fs::path const& path) { Parser parser; @@ -501,20 +511,25 @@ bool Dataloader::_load_history(GameManager& game_manager, bool unused_history_fi ); /* Pop History */ - static constexpr std::string_view pop_history_directory = "history/pops"; - for (fs::path root : roots) { - fs::path concat = root / pop_history_directory; - for (fs::path dir : fs::directory_iterator(concat)) { - const Date date = Date::from_string(dir.filename().string()); + static constexpr std::string_view pop_history_directory = "history/pops/"; + const string_set_t pop_history_dirs = lookup_dirs_in_dir(pop_history_directory); + const Date last_bookmark_date = game_manager.get_history_manager().get_bookmark_manager().get_last_bookmark_date(); + for (std::string const& dir : pop_history_dirs) { + bool successful = false; + const Date date = Date::from_string(dir, &successful); + if (successful && date <= last_bookmark_date) { + bool non_integer_size = false; ret &= apply_to_files( - lookup_basic_indentifier_prefixed_files_in_dir_recursive(dir.string(), ".txt"), - [this, &game_manager, &date](fs::path const& file) -> bool { - const std::string filename = file.stem().string(); + lookup_files_in_dir(StringUtils::append_string_views(pop_history_directory, dir), ".txt"), + [this, &game_manager, date, &non_integer_size](fs::path const& file) -> bool { return game_manager.get_history_manager().get_province_manager().load_pop_history_file( - game_manager, date, parse_defines(file).get_file_node() + game_manager, date, parse_defines(file).get_file_node(), &non_integer_size ); } ); + if (non_integer_size) { + Logger::warning("Non-integer pop sizes in pop history files for ", date); + } } } diff --git a/src/openvic-simulation/dataloader/Dataloader.hpp b/src/openvic-simulation/dataloader/Dataloader.hpp index 7f5d0aa..3512e59 100644 --- a/src/openvic-simulation/dataloader/Dataloader.hpp +++ b/src/openvic-simulation/dataloader/Dataloader.hpp @@ -94,6 +94,8 @@ namespace OpenVic { ) const; bool apply_to_files(path_vector_t const& files, NodeTools::callback_t<fs::path const&> callback) const; + string_set_t lookup_dirs_in_dir(std::string_view path) const; + bool load_defines(GameManager& game_manager) const; enum locale_t : size_t { |