Merge pull request #89 from OpenVicProject/dataloading-pop-history

Pop History Loading (with the new system)

20 files changed, 311 insertions, 108 deletions

diff --git a/src/openvic-simulation/dataloader/Dataloader.cpp b/src/openvic-simulation/dataloader/Dataloader.cpp
index f49ae2d..4687925 100644
--- a/src/openvic-simulation/dataloader/Dataloader.cpp
+++ b/src/openvic-simulation/dataloader/Dataloader.cpp
@@ -25,6 +25,15 @@ using StringUtils::append_string_views;
 #define FILESYSTEM_NEEDS_FORWARD_SLASHES
 #endif
 
+static fs::path ensure_forward_slash_path(std::string_view path) {
+#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES)
+	/* Back-slashes need to be converted into forward-slashes */
+	return StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path));
+#else
+	return path;
+#endif
+}
+
 static constexpr bool path_equals_case_insensitive(std::string_view lhs, std::string_view rhs) {
 	constexpr auto ichar_equals = [](unsigned char l, unsigned char r) {
 		return std::tolower(l) == std::tolower(r);
@@ -60,13 +69,7 @@ bool Dataloader::set_roots(path_vector_t const& new_roots) {
 }
 
 fs::path Dataloader::lookup_file(std::string_view path, bool print_error) const {
-#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES)
-	/* Back-slashes need to be converted into forward-slashes */
-	const std::string forward_slash_path { StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path)) };
-	path = forward_slash_path;
-#endif
-
-	const fs::path filepath { path };
+	const fs::path filepath { ensure_forward_slash_path(path) };
 
 #if defined(FILESYSTEM_CASE_INSENSITIVE)
 	/* Case-insensitive filesystem */
@@ -120,12 +123,7 @@ requires requires (_UniqueKey const& unique_key, std::string_view path) {
 Dataloader::path_vector_t Dataloader::_lookup_files_in_dir(
 	std::string_view path, fs::path const& extension, _UniqueKey const& unique_key
 ) const {
-#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES)
-	/* Back-slashes need to be converted into forward-slashes */
-	const std::string forward_slash_path { StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path)) };
-	path = forward_slash_path;
-#endif
-	const fs::path dirpath { path };
+	const fs::path dirpath { ensure_forward_slash_path(path) };
 	path_vector_t ret;
 	struct file_entry_t {
 		fs::path file;
@@ -134,9 +132,8 @@ Dataloader::path_vector_t Dataloader::_lookup_files_in_dir(
 	string_map_t<file_entry_t> found_files;
 	for (fs::path const& root : roots) {
 		const size_t root_len = root.string().size();
-		const fs::path composed = root / dirpath;
 		std::error_code ec;
-		for (fs::directory_entry const& entry : _DirIterator { composed, ec }) {
+		for (fs::directory_entry const& entry : _DirIterator { root / dirpath, ec }) {
 			if (entry.is_regular_file()) {
 				fs::path file = entry;
 				if ((extension.empty() || file.extension() == extension)) {
@@ -199,6 +196,20 @@ bool Dataloader::apply_to_files(path_vector_t const& files, callback_t<fs::path
 	return ret;
 }
 
+string_set_t Dataloader::lookup_dirs_in_dir(std::string_view path) const {
+	const fs::path dirpath { ensure_forward_slash_path(path) };
+	string_set_t ret;
+	for (fs::path const& root : roots) {
+		std::error_code ec;
+		for (fs::directory_entry const& entry : fs::directory_iterator { root / dirpath, ec }) {
+			if (entry.is_directory()) {
+				ret.emplace(entry.path().filename().string());
+			}
+		}
+	}
+	return ret;
+}
+
 template<std::derived_from<detail::BasicParser> Parser, bool (*parse_func)(Parser&)>
 static Parser _run_ovdl_parser(fs::path const& path) {
 	Parser parser;
@@ -498,6 +509,30 @@ bool Dataloader::_load_history(GameManager& game_manager, bool unused_history_fi
 			);
 		}
 	);
+
+	/* Pop History */
+	static constexpr std::string_view pop_history_directory = "history/pops/";
+	const string_set_t pop_history_dirs = lookup_dirs_in_dir(pop_history_directory);
+	const Date last_bookmark_date = game_manager.get_history_manager().get_bookmark_manager().get_last_bookmark_date();
+	for (std::string const& dir : pop_history_dirs) {
+		bool successful = false;
+		const Date date = Date::from_string(dir, &successful);
+		if (successful && date <= last_bookmark_date) {
+			bool non_integer_size = false;
+			ret &= apply_to_files(
+				lookup_files_in_dir(StringUtils::append_string_views(pop_history_directory, dir), ".txt"),
+				[this, &game_manager, date, &non_integer_size](fs::path const& file) -> bool {
+					return game_manager.get_history_manager().get_province_manager().load_pop_history_file(
+						game_manager, date, parse_defines(file).get_file_node(), &non_integer_size
+					);
+				}
+			);
+			if (non_integer_size) {
+				Logger::warning("Non-integer pop sizes in pop history files for ", date);
+			}
+		}
+	}
+
 	game_manager.get_history_manager().get_province_manager().lock_province_histories(game_manager.get_map(), false);
 
 	static constexpr std::string_view diplomacy_history_directory = "history/diplomacy";
@@ -837,19 +872,6 @@ bool Dataloader::load_defines(GameManager& game_manager) const {
 	return ret;
 }
 
-bool Dataloader::load_pop_history(GameManager& game_manager, std::string_view path) const {
-	return apply_to_files(
-		lookup_files_in_dir(path, ".txt"),
-		[&game_manager](fs::path const& file) -> bool {
-			return game_manager.get_map().expect_province_dictionary(
-				[&game_manager](Province& province, ast::NodeCPtr value) -> bool {
-					return province.load_pop_list(game_manager.get_pop_manager(), value);
-				}
-			)(parse_defines(file).get_file_node());
-		}
-	);
-}
-
 static bool _load_localisation_file(Dataloader::localisation_callback_t callback, std::vector<csv::LineObject> const& lines) {
 	bool ret = true;
 	for (csv::LineObject const& line : lines) {
diff --git a/src/openvic-simulation/dataloader/Dataloader.hpp b/src/openvic-simulation/dataloader/Dataloader.hpp
index 78021b3..3512e59 100644
--- a/src/openvic-simulation/dataloader/Dataloader.hpp
+++ b/src/openvic-simulation/dataloader/Dataloader.hpp
@@ -94,8 +94,9 @@ namespace OpenVic {
 		) const;
 		bool apply_to_files(path_vector_t const& files, NodeTools::callback_t<fs::path const&> callback) const;
 
+		string_set_t lookup_dirs_in_dir(std::string_view path) const;
+
 		bool load_defines(GameManager& game_manager) const;
-		bool load_pop_history(GameManager& game_manager, std::string_view path) const;
 
 		enum locale_t : size_t {
 			English, French, German, Polish, Spanish, Italian, Swedish,
diff --git a/src/openvic-simulation/history/Bookmark.cpp b/src/openvic-simulation/history/Bookmark.cpp
index 2927950..b758867 100644
--- a/src/openvic-simulation/history/Bookmark.cpp
+++ b/src/openvic-simulation/history/Bookmark.cpp
@@ -48,3 +48,13 @@ bool BookmarkManager::load_bookmark_file(ast::NodeCPtr root) {
 
 	return ret;
 }
+
+Date BookmarkManager::get_last_bookmark_date() const {
+	Date ret {};
+	for (Bookmark const& bookmark : get_bookmarks()) {
+		if (bookmark.get_date() > ret) {
+			ret = bookmark.get_date();
+		}
+	}
+	return ret;
+}
diff --git a/src/openvic-simulation/history/Bookmark.hpp b/src/openvic-simulation/history/Bookmark.hpp
index 7401105..6f30586 100644
--- a/src/openvic-simulation/history/Bookmark.hpp
+++ b/src/openvic-simulation/history/Bookmark.hpp
@@ -38,5 +38,7 @@ namespace OpenVic {
 			uint32_t initial_camera_y
 		);
 		bool load_bookmark_file(ast::NodeCPtr root);
+
+		Date get_last_bookmark_date() const;
 	};
 }
diff --git a/src/openvic-simulation/history/CountryHistory.cpp b/src/openvic-simulation/history/CountryHistory.cpp
index 9ee7d65..00c88b8 100644
--- a/src/openvic-simulation/history/CountryHistory.cpp
+++ b/src/openvic-simulation/history/CountryHistory.cpp
@@ -25,9 +25,10 @@ bool CountryHistoryMap::_load_history_entry(
 	CountryManager const& country_manager = game_manager.get_country_manager();
 	TechnologyManager const& technology_manager = game_manager.get_research_manager().get_technology_manager();
 	InventionManager const& invention_manager = game_manager.get_research_manager().get_invention_manager();
+	DecisionManager const& decision_manager = game_manager.get_decision_manager();
 
 	return expect_dictionary_keys_and_default(
-		[this, &game_manager, &dataloader, &deployment_manager, &issue_manager, 
+		[this, &game_manager, &dataloader, &deployment_manager, &issue_manager,
 			&technology_manager, &invention_manager, &country_manager, &entry](std::string_view key, ast::NodeCPtr value) -> bool {
 			ReformGroup const* reform_group = issue_manager.get_reform_group_by_identifier(key);
 			if (reform_group != nullptr) {
@@ -66,14 +67,11 @@ bool CountryHistoryMap::_load_history_entry(
 					return entry.inventions.emplace(invention, flag).second;
 				} else return false;
 			}
-			
+
 			return _load_history_sub_entry_callback(
 				game_manager, dataloader, deployment_manager, entry.get_date(), value, key, value
 			);
 		},
-		/* we have to use a lambda, assign_variable_callback_pointer
-		 * apparently doesn't play nice with const & non-const accessors */
-		// TODO - fix this issue (cause by provinces having non-const accessors)
 		"capital", ZERO_OR_ONE,
 			game_manager.get_map().expect_province_identifier(assign_variable_callback_pointer(entry.capital)),
 		"primary_culture", ZERO_OR_ONE,
@@ -122,24 +120,74 @@ bool CountryHistoryMap::_load_history_entry(
 		"consciousness", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(entry.consciousness)),
 		"nonstate_consciousness", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(entry.nonstate_consciousness)),
 		"is_releasable_vassal", ZERO_OR_ONE, expect_bool(assign_variable_callback(entry.releasable_vassal)),
-		"decision", ZERO_OR_ONE, success_callback, //TODO: decisions
+		"decision", ZERO_OR_MORE, decision_manager.expect_decision_identifier(set_callback_pointer(entry.decisions)),
 		"govt_flag", ZERO_OR_ONE, [&entry, &politics_manager](ast::NodeCPtr value) -> bool {
+			GovernmentTypeManager const& government_type_manager = politics_manager.get_government_type_manager();
 			GovernmentType const* government_type = nullptr;
-			std::string_view flag;
-			bool ret = expect_dictionary_keys(
-				"government", ONE_EXACTLY, politics_manager.get_government_type_manager()
-					.expect_government_type_identifier(assign_variable_callback_pointer(government_type)),
-				"flag", ONE_EXACTLY, expect_identifier_or_string(assign_variable_callback(flag))
+			bool flag_expected = false;
+			bool ret = expect_dictionary(
+				[&entry, &government_type_manager, &government_type, &flag_expected](std::string_view id,
+					ast::NodeCPtr node) -> bool {
+					if (id == "government") {
+						bool ret = true;
+						if (flag_expected) {
+							Logger::error(
+								"Government key found when expect flag type override for ", government_type,
+								" in history of ", entry.get_country().get_identifier()
+							);
+							ret = false;
+						}
+						flag_expected = true;
+						government_type = nullptr;
+						ret &= government_type_manager.expect_government_type_identifier(
+							assign_variable_callback_pointer(government_type)
+						)(node);
+						return ret;
+					} else if (id == "flag") {
+						if (flag_expected) {
+							flag_expected = false;
+							GovernmentType const* flag_override_government_type = nullptr;
+							bool ret = government_type_manager.expect_government_type_identifier(
+								assign_variable_callback_pointer(flag_override_government_type)
+							)(node);
+							/* If the first government type is null, the "government" section will have already output
+							 * an error, so no need to output another one here. */
+							if (government_type != nullptr && flag_override_government_type != nullptr) {
+								ret &= entry.government_flag_overrides.emplace(
+									government_type, flag_override_government_type
+								).second;
+							}
+							return ret;
+						} else {
+							Logger::error(
+								"Flag key found when expecting government type for flag type override in history of ",
+								entry.get_country().get_identifier()
+							);
+							return false;
+						}
+					} else {
+						Logger::error(
+							"Invalid key ", id, " in government flag overrides in history of ",
+							entry.get_country().get_identifier()
+						);
+						return false;
+					}
+				}
 			)(value);
-			if (government_type != nullptr) {
-				return ret & entry.government_flags.emplace(government_type, flag).second;
-			} else return false;
+			if (flag_expected) {
+				Logger::error(
+					"Missing flag type override for government type ", government_type, " in history of ",
+					entry.get_country().get_identifier()
+				);
+				ret = false;
+			}
+			return ret;
 		},
-		"colonial_points", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(entry.colonial_points)), 
-		"set_country_flag", ZERO_OR_ONE, expect_identifier_or_string([&entry](std::string_view flag) -> bool {
+		"colonial_points", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(entry.colonial_points)),
+		"set_country_flag", ZERO_OR_MORE, expect_identifier_or_string([&entry](std::string_view flag) -> bool {
 			return entry.country_flags.emplace(flag).second;
 		}),
-		"set_global_flag", ZERO_OR_ONE, expect_identifier_or_string([&entry](std::string_view flag) -> bool {
+		"set_global_flag", ZERO_OR_MORE, expect_identifier_or_string([&entry](std::string_view flag) -> bool {
 			return entry.global_flags.emplace(flag).second;
 		})
 	)(root);
diff --git a/src/openvic-simulation/history/CountryHistory.hpp b/src/openvic-simulation/history/CountryHistory.hpp
index af7d502..106b1c3 100644
--- a/src/openvic-simulation/history/CountryHistory.hpp
+++ b/src/openvic-simulation/history/CountryHistory.hpp
@@ -3,6 +3,7 @@
 #include <map>
 #include <optional>
 
+#include "openvic-simulation/misc/Decision.hpp"
 #include "openvic-simulation/country/Country.hpp"
 #include "openvic-simulation/history/Bookmark.hpp"
 #include "openvic-simulation/history/HistoryMap.hpp"
@@ -54,9 +55,8 @@ namespace OpenVic {
 		std::optional<fixed_point_t> PROPERTY(colonial_points);
 		string_set_t PROPERTY(country_flags);
 		string_set_t PROPERTY(global_flags);
-		std::map<GovernmentType const*, std::string> PROPERTY(government_flags);
-
-		//TODO: decisions
+		std::map<GovernmentType const*, GovernmentType const*> PROPERTY(government_flag_overrides);
+		std::set<Decision const*> decisions;
 
 		CountryHistoryEntry(Country const& new_country, Date new_date);
 	};
diff --git a/src/openvic-simulation/history/HistoryMap.cpp b/src/openvic-simulation/history/HistoryMap.cpp
index b669208..7b5353f 100644
--- a/src/openvic-simulation/history/HistoryMap.cpp
+++ b/src/openvic-simulation/history/HistoryMap.cpp
@@ -6,10 +6,10 @@ using namespace OpenVic;
 
 HistoryEntry::HistoryEntry(Date new_date) : date { new_date } {}
 
-Date OpenVic::_get_start_date(GameManager const& game_manager) {
+Date _HistoryMapHelperFuncs::_get_start_date(GameManager const& game_manager) {
 	return game_manager.get_define_manager().get_start_date();
 }
 
-Date OpenVic::_get_end_date(GameManager const& game_manager) {
+Date _HistoryMapHelperFuncs::_get_end_date(GameManager const& game_manager) {
 	return game_manager.get_define_manager().get_end_date();
 }
diff --git a/src/openvic-simulation/history/HistoryMap.hpp b/src/openvic-simulation/history/HistoryMap.hpp
index 07f54f0..576f00e 100644
--- a/src/openvic-simulation/history/HistoryMap.hpp
+++ b/src/openvic-simulation/history/HistoryMap.hpp
@@ -18,9 +18,11 @@ namespace OpenVic {
 
 	struct GameManager;
 
-	/* Helper functions to avoid cyclic dependency issues */
-	Date _get_start_date(GameManager const& game_manager);
-	Date _get_end_date(GameManager const& game_manager);
+	namespace _HistoryMapHelperFuncs {
+		/* Helper functions to avoid cyclic dependency issues */
+		Date _get_start_date(GameManager const& game_manager);
+		Date _get_end_date(GameManager const& game_manager);
+	}
 
 	template<std::derived_from<HistoryEntry> _Entry, typename... Args>
 	struct HistoryMap {
@@ -30,22 +32,12 @@ namespace OpenVic {
 		std::map<Date, std::unique_ptr<entry_type>> PROPERTY(entries);
 
 		bool _try_load_history_entry(GameManager const& game_manager, Args... args, Date date, ast::NodeCPtr root) {
-			const Date end_date = _get_end_date(game_manager);
-			if (date > end_date) {
-				Logger::error("History entry ", date, " defined after end date ", end_date);
+			entry_type *const entry = _get_or_make_entry(game_manager, date);
+			if (entry != nullptr) {
+				return _load_history_entry(game_manager, args..., *entry, root);
+			} else {
 				return false;
 			}
-			typename decltype(entries)::iterator it = entries.find(date);
-			if (it == entries.end()) {
-				const std::pair<typename decltype(entries)::iterator, bool> result = entries.emplace(date, _make_entry(date));
-				if (result.second) {
-					it = result.first;
-				} else {
-					Logger::error("Failed to create history entry at date ", date);
-					return false;
-				}
-			}
-			return _load_history_entry(game_manager, args..., *it->second, root);
 		}
 
 	protected:
@@ -58,7 +50,7 @@ namespace OpenVic {
 		) = 0;
 
 		bool _load_history_file(GameManager const& game_manager, Args... args, ast::NodeCPtr root) {
-			return _try_load_history_entry(game_manager, args..., _get_start_date(game_manager), root);
+			return _try_load_history_entry(game_manager, args..., _HistoryMapHelperFuncs::_get_start_date(game_manager), root);
 		}
 
 		bool _load_history_sub_entry_callback(
@@ -87,6 +79,26 @@ namespace OpenVic {
 			return default_callback(key, value);
 		}
 
+		/* Returns history entry at specific date, if date doesn't have an entry creates one, if that fails returns nullptr. */
+		entry_type* _get_or_make_entry(GameManager const& game_manager, Date date) {
+			const Date end_date = _HistoryMapHelperFuncs::_get_end_date(game_manager);
+			if (date > end_date) {
+				Logger::error("History entry ", date, " defined after end date ", end_date);
+				return nullptr;
+			}
+			typename decltype(entries)::iterator it = entries.find(date);
+			if (it == entries.end()) {
+				const std::pair<typename decltype(entries)::iterator, bool> result = entries.emplace(date, _make_entry(date));
+				if (result.second) {
+					it = result.first;
+				} else {
+					Logger::error("Failed to create history entry at date ", date);
+					return nullptr;
+				}
+			}
+			return it->second.get();
+		}
+
 	public:
 		/* Returns history entry at specific date, if date doesn't have an entry returns nullptr. */
 		entry_type const* get_entry(Date date) const {
diff --git a/src/openvic-simulation/history/ProvinceHistory.cpp b/src/openvic-simulation/history/ProvinceHistory.cpp
index d9d3ef6..4e92300 100644
--- a/src/openvic-simulation/history/ProvinceHistory.cpp
+++ b/src/openvic-simulation/history/ProvinceHistory.cpp
@@ -135,6 +135,21 @@ ProvinceHistoryMap const* ProvinceHistoryManager::get_province_history(Province
 	}
 }
 
+ProvinceHistoryMap* ProvinceHistoryManager::_get_or_make_province_history(Province const& province) {
+	decltype(province_histories)::iterator it = province_histories.find(&province);
+	if (it == province_histories.end()) {
+		const std::pair<decltype(province_histories)::iterator, bool> result =
+			province_histories.emplace(&province, ProvinceHistoryMap { province });
+		if (result.second) {
+			it = result.first;
+		} else {
+			Logger::error("Failed to create province history map for province ", province.get_identifier());
+			return nullptr;
+		}
+	}
+	return &it->second;
+}
+
 bool ProvinceHistoryManager::load_province_history_file(
 	GameManager const& game_manager, Province const& province, ast::NodeCPtr root
 ) {
@@ -146,18 +161,52 @@ bool ProvinceHistoryManager::load_province_history_file(
 		return false;
 	}
 
-	decltype(province_histories)::iterator it = province_histories.find(&province);
-	if (it == province_histories.end()) {
-		const std::pair<decltype(province_histories)::iterator, bool> result =
-			province_histories.emplace(&province, ProvinceHistoryMap { province });
-		if (result.second) {
-			it = result.first;
-		} else {
-			Logger::error("Failed to create province history map for province ", province.get_identifier());
-			return false;
+	ProvinceHistoryMap* province_history = _get_or_make_province_history(province);
+	if (province_history != nullptr) {
+		return province_history->_load_history_file(game_manager, root);
+	} else {
+		return false;
+	}
+}
+
+bool ProvinceHistoryEntry::_load_province_pop_history(
+	GameManager const& game_manager, ast::NodeCPtr root, bool *non_integer_size
+) {
+	PopManager const& pop_manager = game_manager.get_pop_manager();
+	RebelManager const& rebel_manager = game_manager.get_politics_manager().get_rebel_manager();
+	return pop_manager.expect_pop_type_dictionary(
+		[this, &pop_manager, &rebel_manager, non_integer_size](PopType const& pop_type, ast::NodeCPtr pop_node) -> bool {
+			return pop_manager.load_pop_into_vector(rebel_manager, pops, pop_type, pop_node, non_integer_size);
 		}
+	)(root);
+}
+
+bool ProvinceHistoryMap::_load_province_pop_history(
+	GameManager const& game_manager, Date date, ast::NodeCPtr root, bool *non_integer_size
+) {
+	ProvinceHistoryEntry* entry = _get_or_make_entry(game_manager, date);
+	if (entry != nullptr) {
+		return entry->_load_province_pop_history(game_manager, root, non_integer_size);
+	} else {
+		return false;
 	}
-	ProvinceHistoryMap& province_history = it->second;
+}
 
-	return province_history._load_history_file(game_manager, root);
+bool ProvinceHistoryManager::load_pop_history_file(
+	GameManager const& game_manager, Date date, ast::NodeCPtr root, bool *non_integer_size
+) {
+	if (locked) {
+		Logger::error("Attempted to load pop history file after province history registry was locked!");
+		return false;
+	}
+	return game_manager.get_map().expect_province_dictionary(
+		[this, &game_manager, date, non_integer_size](Province const& province, ast::NodeCPtr node) -> bool {
+			ProvinceHistoryMap* province_history = _get_or_make_province_history(province);
+			if (province_history != nullptr) {
+				return province_history->_load_province_pop_history(game_manager, date, node, non_integer_size);
+			} else {
+				return false;
+			}
+		}
+	)(root);
 }
diff --git a/src/openvic-simulation/history/ProvinceHistory.hpp b/src/openvic-simulation/history/ProvinceHistory.hpp
index e4adc08..c3c8e67 100644
--- a/src/openvic-simulation/history/ProvinceHistory.hpp
+++ b/src/openvic-simulation/history/ProvinceHistory.hpp
@@ -33,8 +33,11 @@ namespace OpenVic {
 		std::map<BuildingType const*, BuildingType::level_t> PROPERTY(province_buildings);
 		std::map<BuildingType const*, BuildingType::level_t> PROPERTY(state_buildings);
 		fixed_point_map_t<Ideology const*> PROPERTY(party_loyalties);
+		std::vector<Pop> PROPERTY(pops);
 
 		ProvinceHistoryEntry(Province const& new_province, Date new_date);
+
+		bool _load_province_pop_history(GameManager const& game_manager, ast::NodeCPtr root, bool *non_integer_size);
 	};
 
 	struct ProvinceHistoryManager;
@@ -50,6 +53,11 @@ namespace OpenVic {
 
 		std::unique_ptr<ProvinceHistoryEntry> _make_entry(Date date) const override;
 		bool _load_history_entry(GameManager const& game_manager, ProvinceHistoryEntry& entry, ast::NodeCPtr root) override;
+
+	private:
+		bool _load_province_pop_history(
+			GameManager const& game_manager, Date date, ast::NodeCPtr root, bool *non_integer_size
+		);
 	};
 
 	struct ProvinceHistoryManager {
@@ -57,6 +65,8 @@ namespace OpenVic {
 		std::map<Province const*, ProvinceHistoryMap> PROPERTY(province_histories);
 		bool locked = false;
 
+		ProvinceHistoryMap* _get_or_make_province_history(Province const& province);
+
 	public:
 		ProvinceHistoryManager() = default;
 
@@ -66,5 +76,6 @@ namespace OpenVic {
 		ProvinceHistoryMap const* get_province_history(Province const* province) const;
 
 		bool load_province_history_file(GameManager const& game_manager, Province const& province, ast::NodeCPtr root);
+		bool load_pop_history_file(GameManager const& game_manager, Date date, ast::NodeCPtr root, bool *non_integer_size);
 	};
 } // namespace OpenVic
diff --git a/src/openvic-simulation/interface/UI.hpp b/src/openvic-simulation/interface/UI.hpp
index 12eadb5..ce9336c 100644
--- a/src/openvic-simulation/interface/UI.hpp
+++ b/src/openvic-simulation/interface/UI.hpp
@@ -4,7 +4,7 @@
 
 namespace OpenVic {
 
-	class UIManager { 
+	class UIManager {
 		NamedInstanceRegistry<GFX::Sprite> IDENTIFIER_REGISTRY(sprite);
 		NamedInstanceRegistry<GUI::Scene, UIManager const&> IDENTIFIER_REGISTRY(scene);
 		IdentifierRegistry<GFX::Font> IDENTIFIER_REGISTRY(font);
diff --git a/src/openvic-simulation/map/Map.cpp b/src/openvic-simulation/map/Map.cpp
index 44443a4..b272f1f 100644
--- a/src/openvic-simulation/map/Map.cpp
+++ b/src/openvic-simulation/map/Map.cpp
@@ -294,8 +294,15 @@ bool Map::apply_history_to_provinces(ProvinceHistoryManager const& history_manag
 		if (!province.get_water()) {
 			ProvinceHistoryMap const* history_map = history_manager.get_province_history(&province);
 			if (history_map != nullptr) {
+				ProvinceHistoryEntry const* pop_history_entry = nullptr;
 				for (ProvinceHistoryEntry const* entry : history_map->get_entries_up_to(date)) {
 					province.apply_history_to_province(entry);
+					if (!entry->get_pops().empty()) {
+						pop_history_entry = entry;
+					}
+				}
+				if (pop_history_entry != nullptr) {
+					province.add_pop_vec(pop_history_entry->get_pops());
 				}
 			}
 		}
diff --git a/src/openvic-simulation/map/Province.cpp b/src/openvic-simulation/map/Province.cpp
index 363be6c..8094463 100644
--- a/src/openvic-simulation/map/Province.cpp
+++ b/src/openvic-simulation/map/Province.cpp
@@ -58,14 +58,6 @@ bool Province::expand_building(std::string_view building_type_identifier) {
 	return building->expand();
 }
 
-bool Province::load_pop_list(PopManager const& pop_manager, ast::NodeCPtr root) {
-	return expect_dictionary_reserve_length(pops,
-		[this, &pop_manager](std::string_view pop_type_identifier, ast::NodeCPtr pop_node) -> bool {
-			return pop_manager.load_pop_into_province(*this, pop_type_identifier, pop_node);
-		}
-	)(root);
-}
-
 bool Province::add_pop(Pop&& pop) {
 	if (!get_water()) {
 		pops.push_back(std::move(pop));
@@ -76,6 +68,19 @@ bool Province::add_pop(Pop&& pop) {
 	}
 }
 
+bool Province::add_pop_vec(std::vector<Pop> const& pop_vec) {
+	if (!get_water()) {
+		pops.reserve(pops.size() + pop_vec.size());
+		for (Pop const& pop : pop_vec) {
+			pops.push_back(pop);
+		}
+		return true;
+	} else {
+		Logger::error("Trying to add pop vector to water province ", get_identifier());
+		return false;
+	}
+}
+
 size_t Province::get_pop_count() const {
 	return pops.size();
 }
diff --git a/src/openvic-simulation/map/Province.hpp b/src/openvic-simulation/map/Province.hpp
index e73dec3..37ecdaa 100644
--- a/src/openvic-simulation/map/Province.hpp
+++ b/src/openvic-simulation/map/Province.hpp
@@ -101,8 +101,8 @@ namespace OpenVic {
 
 		bool expand_building(std::string_view building_type_identifier);
 
-		bool load_pop_list(PopManager const& pop_manager, ast::NodeCPtr root);
 		bool add_pop(Pop&& pop);
+		bool add_pop_vec(std::vector<Pop> const& pop_vec);
 		size_t get_pop_count() const;
 		void update_pops();
 
diff --git a/src/openvic-simulation/misc/Decision.cpp b/src/openvic-simulation/misc/Decision.cpp
index 9200688..f1f2b6c 100644
--- a/src/openvic-simulation/misc/Decision.cpp
+++ b/src/openvic-simulation/misc/Decision.cpp
@@ -34,7 +34,7 @@ bool DecisionManager::add_decision(
 		}
 	}
 
-	return decisions.add_item({ 
+	return decisions.add_item({
 		identifier,
 		alert,
 		news,
@@ -42,7 +42,7 @@ bool DecisionManager::add_decision(
 		news_desc_long,
 		news_desc_medium,
 		news_desc_short,
-		picture 
+		picture
 	}, duplicate_warning_callback);
 }
 
diff --git a/src/openvic-simulation/politics/Government.cpp b/src/openvic-simulation/politics/Government.cpp
index a226518..fc93cc3 100644
--- a/src/openvic-simulation/politics/Government.cpp
+++ b/src/openvic-simulation/politics/Government.cpp
@@ -111,3 +111,9 @@ bool GovernmentTypeManager::load_government_types_file(IdeologyManager const& id
 
 	return ret;
 }
+
+bool GovernmentTypeManager::is_valid_flag_type(std::string_view type) const {
+	return std::any_of(flag_types.begin(), flag_types.end(), [type](std::string const& flag_type) -> bool {
+		return flag_type == type;
+	});
+}
diff --git a/src/openvic-simulation/politics/Government.hpp b/src/openvic-simulation/politics/Government.hpp
index 4682bbf..34a9194 100644
--- a/src/openvic-simulation/politics/Government.hpp
+++ b/src/openvic-simulation/politics/Government.hpp
@@ -38,5 +38,7 @@ namespace OpenVic {
 		);
 
 		bool load_government_types_file(IdeologyManager const& ideology_manager, ast::NodeCPtr root);
+
+		bool is_valid_flag_type(std::string_view type) const;
 	};
 } // namespace OpenVic
diff --git a/src/openvic-simulation/pop/Pop.cpp b/src/openvic-simulation/pop/Pop.cpp
index cf54c2b..2d85dce 100644
--- a/src/openvic-simulation/pop/Pop.cpp
+++ b/src/openvic-simulation/pop/Pop.cpp
@@ -4,14 +4,18 @@
 
 #include "openvic-simulation/dataloader/NodeTools.hpp"
 #include "openvic-simulation/map/Province.hpp"
+#include "openvic-simulation/politics/Rebel.hpp"
 #include "openvic-simulation/utility/Logger.hpp"
 
 using namespace OpenVic;
 using namespace OpenVic::NodeTools;
 
 Pop::Pop(
-	PopType const& new_type, Culture const& new_culture, Religion const& new_religion, pop_size_t new_size
-) : type { new_type }, culture { new_culture }, religion { new_religion }, size { new_size } {
+	PopType const& new_type, Culture const& new_culture, Religion const& new_religion, pop_size_t new_size,
+	fixed_point_t new_militancy, fixed_point_t new_consciousness, RebelType const* new_rebel_type
+) : type { new_type }, culture { new_culture }, religion { new_religion }, size { new_size }, num_promoted { 0 },
+	num_demoted { 0 }, num_migrated { 0 }, militancy { new_militancy }, consciousness { new_consciousness },
+	rebel_type { new_rebel_type } {
 	assert(size > 0);
 }
 
@@ -157,26 +161,34 @@ bool PopManager::load_pop_type_file(
 	return ret;
 }
 
-bool PopManager::load_pop_into_province(Province& province, std::string_view pop_type_identifier, ast::NodeCPtr pop_node)
-	const {
-	PopType const* type = get_pop_type_by_identifier(pop_type_identifier);
+bool PopManager::load_pop_into_vector(
+	RebelManager const& rebel_manager, std::vector<Pop>& vec, PopType const& type, ast::NodeCPtr pop_node,
+	bool *non_integer_size
+) const {
 	Culture const* culture = nullptr;
 	Religion const* religion = nullptr;
-	Pop::pop_size_t size = 0;
+	fixed_point_t size = 0; /* Some genius filled later start dates with non-integer sized pops */
+	fixed_point_t militancy = 0, consciousness = 0;
+	RebelType const* rebel_type = nullptr;
+
 	bool ret = expect_dictionary_keys(
 		"culture", ONE_EXACTLY, culture_manager.expect_culture_identifier(assign_variable_callback_pointer(culture)),
 		"religion", ONE_EXACTLY, religion_manager.expect_religion_identifier(assign_variable_callback_pointer(religion)),
-		"size", ONE_EXACTLY, expect_uint(assign_variable_callback(size)),
-		"militancy", ZERO_OR_ONE, success_callback,
-		"rebel_type", ZERO_OR_ONE, success_callback
+		"size", ONE_EXACTLY, expect_fixed_point(assign_variable_callback(size)),
+		"militancy", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(militancy)),
+		"consciousness", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(consciousness)),
+		"rebel_type", ZERO_OR_ONE, rebel_manager.expect_rebel_type_identifier(assign_variable_callback_pointer(rebel_type))
 	)(pop_node);
 
-	if (type != nullptr && culture != nullptr && religion != nullptr && size > 0) {
-		ret &= province.add_pop({ *type, *culture, *religion, size });
+	if (non_integer_size != nullptr && !size.is_integer()) {
+		*non_integer_size = true;
+	}
+
+	if (culture != nullptr && religion != nullptr && size >= 1) {
+		vec.emplace_back(Pop { type, *culture, *religion, size.to_int64_t(), militancy, consciousness, rebel_type });
 	} else {
 		Logger::warning(
-			"Some pop arguments are invalid: province = ", province, ", type = ", type, ", culture = ", culture,
-			", religion = ", religion, ", size = ", size
+			"Some pop arguments are invalid: culture = ", culture, ", religion = ", religion, ", size = ", size
 		);
 	}
 	return ret;
diff --git a/src/openvic-simulation/pop/Pop.hpp b/src/openvic-simulation/pop/Pop.hpp
index 736b77d..6302a58 100644
--- a/src/openvic-simulation/pop/Pop.hpp
+++ b/src/openvic-simulation/pop/Pop.hpp
@@ -9,6 +9,8 @@ namespace OpenVic {
 
 	struct PopManager;
 	struct PopType;
+	struct RebelType;
+	struct RebelManager;
 
 	/* REQUIREMENTS:
 	 * POP-18, POP-19, POP-20, POP-21, POP-34, POP-35, POP-36, POP-37
@@ -27,10 +29,17 @@ namespace OpenVic {
 		pop_size_t PROPERTY(num_demoted);
 		pop_size_t PROPERTY(num_migrated);
 
-		Pop(PopType const& new_type, Culture const& new_culture, Religion const& new_religion, pop_size_t new_size);
+		fixed_point_t PROPERTY(militancy);
+		fixed_point_t PROPERTY(consciousness);
+		RebelType const* PROPERTY(rebel_type);
+
+		Pop(
+			PopType const& new_type, Culture const& new_culture, Religion const& new_religion, pop_size_t new_size,
+			fixed_point_t new_militancy, fixed_point_t new_consciousness, RebelType const* new_rebel_type
+		);
 
 	public:
-		Pop(Pop const&) = delete;
+		Pop(Pop const&) = default;
 		Pop(Pop&&) = default;
 		Pop& operator=(Pop const&) = delete;
 		Pop& operator=(Pop&&) = delete;
@@ -111,6 +120,9 @@ namespace OpenVic {
 		bool load_pop_type_file(
 			std::string_view filestem, UnitManager const& unit_manager, GoodManager const& good_manager, ast::NodeCPtr root
 		);
-		bool load_pop_into_province(Province& province, std::string_view pop_type_identifier, ast::NodeCPtr pop_node) const;
+		bool load_pop_into_vector(
+			RebelManager const& rebel_manager, std::vector<Pop>& vec, PopType const& type, ast::NodeCPtr pop_node,
+			bool *non_integer_size
+		) const;
 	};
 }
diff --git a/src/openvic-simulation/types/fixed_point/FixedPoint.hpp b/src/openvic-simulation/types/fixed_point/FixedPoint.hpp
index ba6790f..75abefb 100644
--- a/src/openvic-simulation/types/fixed_point/FixedPoint.hpp
+++ b/src/openvic-simulation/types/fixed_point/FixedPoint.hpp
@@ -214,6 +214,10 @@ namespace OpenVic {
 			return value & (ONE - 1);
 		}
 
+		constexpr bool is_integer() const {
+			return get_frac() == 0;
+		}
+
 		constexpr int64_t to_int64_t() const {
 			return value >> PRECISION;
 		}