aboutsummaryrefslogtreecommitdiff
path: root/src/openvic-simulation/dataloader
diff options
context:
space:
mode:
Diffstat (limited to 'src/openvic-simulation/dataloader')
-rw-r--r--src/openvic-simulation/dataloader/Dataloader.cpp65
-rw-r--r--src/openvic-simulation/dataloader/Dataloader.hpp2
2 files changed, 42 insertions, 25 deletions
diff --git a/src/openvic-simulation/dataloader/Dataloader.cpp b/src/openvic-simulation/dataloader/Dataloader.cpp
index 382c7aa..4687925 100644
--- a/src/openvic-simulation/dataloader/Dataloader.cpp
+++ b/src/openvic-simulation/dataloader/Dataloader.cpp
@@ -1,6 +1,5 @@
#include "Dataloader.hpp"
-#include <filesystem>
#include <openvic-dataloader/csv/Parser.hpp>
#include <openvic-dataloader/detail/CallbackOStream.hpp>
#include <openvic-dataloader/v2script/Parser.hpp>
@@ -26,6 +25,15 @@ using StringUtils::append_string_views;
#define FILESYSTEM_NEEDS_FORWARD_SLASHES
#endif
+static fs::path ensure_forward_slash_path(std::string_view path) {
+#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES)
+ /* Back-slashes need to be converted into forward-slashes */
+ return StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path));
+#else
+ return path;
+#endif
+}
+
static constexpr bool path_equals_case_insensitive(std::string_view lhs, std::string_view rhs) {
constexpr auto ichar_equals = [](unsigned char l, unsigned char r) {
return std::tolower(l) == std::tolower(r);
@@ -61,13 +69,7 @@ bool Dataloader::set_roots(path_vector_t const& new_roots) {
}
fs::path Dataloader::lookup_file(std::string_view path, bool print_error) const {
-#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES)
- /* Back-slashes need to be converted into forward-slashes */
- const std::string forward_slash_path { StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path)) };
- path = forward_slash_path;
-#endif
-
- const fs::path filepath { path };
+ const fs::path filepath { ensure_forward_slash_path(path) };
#if defined(FILESYSTEM_CASE_INSENSITIVE)
/* Case-insensitive filesystem */
@@ -121,12 +123,7 @@ requires requires (_UniqueKey const& unique_key, std::string_view path) {
Dataloader::path_vector_t Dataloader::_lookup_files_in_dir(
std::string_view path, fs::path const& extension, _UniqueKey const& unique_key
) const {
-#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES)
- /* Back-slashes need to be converted into forward-slashes */
- const std::string forward_slash_path { StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path)) };
- path = forward_slash_path;
-#endif
- const fs::path dirpath { path };
+ const fs::path dirpath { ensure_forward_slash_path(path) };
path_vector_t ret;
struct file_entry_t {
fs::path file;
@@ -135,9 +132,8 @@ Dataloader::path_vector_t Dataloader::_lookup_files_in_dir(
string_map_t<file_entry_t> found_files;
for (fs::path const& root : roots) {
const size_t root_len = root.string().size();
- const fs::path composed = root / dirpath;
std::error_code ec;
- for (fs::directory_entry const& entry : _DirIterator { composed, ec }) {
+ for (fs::directory_entry const& entry : _DirIterator { root / dirpath, ec }) {
if (entry.is_regular_file()) {
fs::path file = entry;
if ((extension.empty() || file.extension() == extension)) {
@@ -200,6 +196,20 @@ bool Dataloader::apply_to_files(path_vector_t const& files, callback_t<fs::path
return ret;
}
+string_set_t Dataloader::lookup_dirs_in_dir(std::string_view path) const {
+ const fs::path dirpath { ensure_forward_slash_path(path) };
+ string_set_t ret;
+ for (fs::path const& root : roots) {
+ std::error_code ec;
+ for (fs::directory_entry const& entry : fs::directory_iterator { root / dirpath, ec }) {
+ if (entry.is_directory()) {
+ ret.emplace(entry.path().filename().string());
+ }
+ }
+ }
+ return ret;
+}
+
template<std::derived_from<detail::BasicParser> Parser, bool (*parse_func)(Parser&)>
static Parser _run_ovdl_parser(fs::path const& path) {
Parser parser;
@@ -501,20 +511,25 @@ bool Dataloader::_load_history(GameManager& game_manager, bool unused_history_fi
);
/* Pop History */
- static constexpr std::string_view pop_history_directory = "history/pops";
- for (fs::path root : roots) {
- fs::path concat = root / pop_history_directory;
- for (fs::path dir : fs::directory_iterator(concat)) {
- const Date date = Date::from_string(dir.filename().string());
+ static constexpr std::string_view pop_history_directory = "history/pops/";
+ const string_set_t pop_history_dirs = lookup_dirs_in_dir(pop_history_directory);
+ const Date last_bookmark_date = game_manager.get_history_manager().get_bookmark_manager().get_last_bookmark_date();
+ for (std::string const& dir : pop_history_dirs) {
+ bool successful = false;
+ const Date date = Date::from_string(dir, &successful);
+ if (successful && date <= last_bookmark_date) {
+ bool non_integer_size = false;
ret &= apply_to_files(
- lookup_basic_indentifier_prefixed_files_in_dir_recursive(dir.string(), ".txt"),
- [this, &game_manager, &date](fs::path const& file) -> bool {
- const std::string filename = file.stem().string();
+ lookup_files_in_dir(StringUtils::append_string_views(pop_history_directory, dir), ".txt"),
+ [this, &game_manager, date, &non_integer_size](fs::path const& file) -> bool {
return game_manager.get_history_manager().get_province_manager().load_pop_history_file(
- game_manager, date, parse_defines(file).get_file_node()
+ game_manager, date, parse_defines(file).get_file_node(), &non_integer_size
);
}
);
+ if (non_integer_size) {
+ Logger::warning("Non-integer pop sizes in pop history files for ", date);
+ }
}
}
diff --git a/src/openvic-simulation/dataloader/Dataloader.hpp b/src/openvic-simulation/dataloader/Dataloader.hpp
index 7f5d0aa..3512e59 100644
--- a/src/openvic-simulation/dataloader/Dataloader.hpp
+++ b/src/openvic-simulation/dataloader/Dataloader.hpp
@@ -94,6 +94,8 @@ namespace OpenVic {
) const;
bool apply_to_files(path_vector_t const& files, NodeTools::callback_t<fs::path const&> callback) const;
+ string_set_t lookup_dirs_in_dir(std::string_view path) const;
+
bool load_defines(GameManager& game_manager) const;
enum locale_t : size_t {