Province pop history + country government flag override refactors

20 files changed, 307 insertions, 131 deletions

diff --git a/src/openvic-simulation/dataloader/Dataloader.cpp b/src/openvic-simulation/dataloader/Dataloader.cpp
index 382c7aa..4687925 100644
--- a/src/openvic-simulation/dataloader/Dataloader.cpp
+++ b/src/openvic-simulation/dataloader/Dataloader.cpp
@@ -1,6 +1,5 @@
 #include "Dataloader.hpp"
 
-#include <filesystem>
 #include <openvic-dataloader/csv/Parser.hpp>
 #include <openvic-dataloader/detail/CallbackOStream.hpp>
 #include <openvic-dataloader/v2script/Parser.hpp>
@@ -26,6 +25,15 @@ using StringUtils::append_string_views;
 #define FILESYSTEM_NEEDS_FORWARD_SLASHES
 #endif
 
+static fs::path ensure_forward_slash_path(std::string_view path) {
+#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES)
+	/* Back-slashes need to be converted into forward-slashes */
+	return StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path));
+#else
+	return path;
+#endif
+}
+
 static constexpr bool path_equals_case_insensitive(std::string_view lhs, std::string_view rhs) {
 	constexpr auto ichar_equals = [](unsigned char l, unsigned char r) {
 		return std::tolower(l) == std::tolower(r);
@@ -61,13 +69,7 @@ bool Dataloader::set_roots(path_vector_t const& new_roots) {
 }
 
 fs::path Dataloader::lookup_file(std::string_view path, bool print_error) const {
-#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES)
-	/* Back-slashes need to be converted into forward-slashes */
-	const std::string forward_slash_path { StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path)) };
-	path = forward_slash_path;
-#endif
-
-	const fs::path filepath { path };
+	const fs::path filepath { ensure_forward_slash_path(path) };
 
 #if defined(FILESYSTEM_CASE_INSENSITIVE)
 	/* Case-insensitive filesystem */
@@ -121,12 +123,7 @@ requires requires (_UniqueKey const& unique_key, std::string_view path) {
 Dataloader::path_vector_t Dataloader::_lookup_files_in_dir(
 	std::string_view path, fs::path const& extension, _UniqueKey const& unique_key
 ) const {
-#if defined(FILESYSTEM_NEEDS_FORWARD_SLASHES)
-	/* Back-slashes need to be converted into forward-slashes */
-	const std::string forward_slash_path { StringUtils::make_forward_slash_path(StringUtils::remove_leading_slashes(path)) };
-	path = forward_slash_path;
-#endif
-	const fs::path dirpath { path };
+	const fs::path dirpath { ensure_forward_slash_path(path) };
 	path_vector_t ret;
 	struct file_entry_t {
 		fs::path file;
@@ -135,9 +132,8 @@ Dataloader::path_vector_t Dataloader::_lookup_files_in_dir(
 	string_map_t<file_entry_t> found_files;
 	for (fs::path const& root : roots) {
 		const size_t root_len = root.string().size();
-		const fs::path composed = root / dirpath;
 		std::error_code ec;
-		for (fs::directory_entry const& entry : _DirIterator { composed, ec }) {
+		for (fs::directory_entry const& entry : _DirIterator { root / dirpath, ec }) {
 			if (entry.is_regular_file()) {
 				fs::path file = entry;
 				if ((extension.empty() || file.extension() == extension)) {
@@ -200,6 +196,20 @@ bool Dataloader::apply_to_files(path_vector_t const& files, callback_t<fs::path
 	return ret;
 }
 
+string_set_t Dataloader::lookup_dirs_in_dir(std::string_view path) const {
+	const fs::path dirpath { ensure_forward_slash_path(path) };
+	string_set_t ret;
+	for (fs::path const& root : roots) {
+		std::error_code ec;
+		for (fs::directory_entry const& entry : fs::directory_iterator { root / dirpath, ec }) {
+			if (entry.is_directory()) {
+				ret.emplace(entry.path().filename().string());
+			}
+		}
+	}
+	return ret;
+}
+
 template<std::derived_from<detail::BasicParser> Parser, bool (*parse_func)(Parser&)>
 static Parser _run_ovdl_parser(fs::path const& path) {
 	Parser parser;
@@ -501,20 +511,25 @@ bool Dataloader::_load_history(GameManager& game_manager, bool unused_history_fi
 	);
 
 	/* Pop History */
-	static constexpr std::string_view pop_history_directory = "history/pops";
-	for (fs::path root : roots) {
-		fs::path concat = root / pop_history_directory;
-		for (fs::path dir : fs::directory_iterator(concat)) {
-			const Date date = Date::from_string(dir.filename().string());
+	static constexpr std::string_view pop_history_directory = "history/pops/";
+	const string_set_t pop_history_dirs = lookup_dirs_in_dir(pop_history_directory);
+	const Date last_bookmark_date = game_manager.get_history_manager().get_bookmark_manager().get_last_bookmark_date();
+	for (std::string const& dir : pop_history_dirs) {
+		bool successful = false;
+		const Date date = Date::from_string(dir, &successful);
+		if (successful && date <= last_bookmark_date) {
+			bool non_integer_size = false;
 			ret &= apply_to_files(
-				lookup_basic_indentifier_prefixed_files_in_dir_recursive(dir.string(), ".txt"),
-				[this, &game_manager, &date](fs::path const& file) -> bool {
-					const std::string filename = file.stem().string();
+				lookup_files_in_dir(StringUtils::append_string_views(pop_history_directory, dir), ".txt"),
+				[this, &game_manager, date, &non_integer_size](fs::path const& file) -> bool {
 					return game_manager.get_history_manager().get_province_manager().load_pop_history_file(
-						game_manager, date, parse_defines(file).get_file_node()
+						game_manager, date, parse_defines(file).get_file_node(), &non_integer_size
 					);
 				}
 			);
+			if (non_integer_size) {
+				Logger::warning("Non-integer pop sizes in pop history files for ", date);
+			}
 		}
 	}
 
diff --git a/src/openvic-simulation/dataloader/Dataloader.hpp b/src/openvic-simulation/dataloader/Dataloader.hpp
index 7f5d0aa..3512e59 100644
--- a/src/openvic-simulation/dataloader/Dataloader.hpp
+++ b/src/openvic-simulation/dataloader/Dataloader.hpp
@@ -94,6 +94,8 @@ namespace OpenVic {
 		) const;
 		bool apply_to_files(path_vector_t const& files, NodeTools::callback_t<fs::path const&> callback) const;
 
+		string_set_t lookup_dirs_in_dir(std::string_view path) const;
+
 		bool load_defines(GameManager& game_manager) const;
 
 		enum locale_t : size_t {
diff --git a/src/openvic-simulation/history/Bookmark.cpp b/src/openvic-simulation/history/Bookmark.cpp
index 2927950..b758867 100644
--- a/src/openvic-simulation/history/Bookmark.cpp
+++ b/src/openvic-simulation/history/Bookmark.cpp
@@ -48,3 +48,13 @@ bool BookmarkManager::load_bookmark_file(ast::NodeCPtr root) {
 
 	return ret;
 }
+
+Date BookmarkManager::get_last_bookmark_date() const {
+	Date ret {};
+	for (Bookmark const& bookmark : get_bookmarks()) {
+		if (bookmark.get_date() > ret) {
+			ret = bookmark.get_date();
+		}
+	}
+	return ret;
+}
diff --git a/src/openvic-simulation/history/Bookmark.hpp b/src/openvic-simulation/history/Bookmark.hpp
index 7401105..6f30586 100644
--- a/src/openvic-simulation/history/Bookmark.hpp
+++ b/src/openvic-simulation/history/Bookmark.hpp
@@ -38,5 +38,7 @@ namespace OpenVic {
 			uint32_t initial_camera_y
 		);
 		bool load_bookmark_file(ast::NodeCPtr root);
+
+		Date get_last_bookmark_date() const;
 	};
 }
diff --git a/src/openvic-simulation/history/CountryHistory.cpp b/src/openvic-simulation/history/CountryHistory.cpp
index 559957f..00c88b8 100644
--- a/src/openvic-simulation/history/CountryHistory.cpp
+++ b/src/openvic-simulation/history/CountryHistory.cpp
@@ -28,7 +28,7 @@ bool CountryHistoryMap::_load_history_entry(
 	DecisionManager const& decision_manager = game_manager.get_decision_manager();
 
 	return expect_dictionary_keys_and_default(
-		[this, &game_manager, &dataloader, &deployment_manager, &issue_manager, 
+		[this, &game_manager, &dataloader, &deployment_manager, &issue_manager,
 			&technology_manager, &invention_manager, &country_manager, &entry](std::string_view key, ast::NodeCPtr value) -> bool {
 			ReformGroup const* reform_group = issue_manager.get_reform_group_by_identifier(key);
 			if (reform_group != nullptr) {
@@ -67,7 +67,7 @@ bool CountryHistoryMap::_load_history_entry(
 					return entry.inventions.emplace(invention, flag).second;
 				} else return false;
 			}
-			
+
 			return _load_history_sub_entry_callback(
 				game_manager, dataloader, deployment_manager, entry.get_date(), value, key, value
 			);
@@ -120,29 +120,70 @@ bool CountryHistoryMap::_load_history_entry(
 		"consciousness", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(entry.consciousness)),
 		"nonstate_consciousness", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(entry.nonstate_consciousness)),
 		"is_releasable_vassal", ZERO_OR_ONE, expect_bool(assign_variable_callback(entry.releasable_vassal)),
-		"decision", ZERO_OR_MORE, decision_manager.expect_decision_identifier([&entry](Decision const& decision) -> bool {
-			return entry.decisions.emplace(&decision).second;
-		}),
+		"decision", ZERO_OR_MORE, decision_manager.expect_decision_identifier(set_callback_pointer(entry.decisions)),
 		"govt_flag", ZERO_OR_ONE, [&entry, &politics_manager](ast::NodeCPtr value) -> bool {
 			GovernmentTypeManager const& government_type_manager = politics_manager.get_government_type_manager();
 			GovernmentType const* government_type = nullptr;
-			return expect_dictionary(
-				[&entry, &government_type_manager, &government_type](std::string_view id, ast::NodeCPtr node) -> bool {
-					bool ret = true;
+			bool flag_expected = false;
+			bool ret = expect_dictionary(
+				[&entry, &government_type_manager, &government_type, &flag_expected](std::string_view id,
+					ast::NodeCPtr node) -> bool {
 					if (id == "government") {
-						government_type = government_type_manager.get_government_type_by_identifier(id);
-						ret &= government_type != nullptr;
-					} else if (id == "flag") {
-						std::string_view flag;
-						ret &= expect_identifier_or_string(assign_variable_callback(flag))(node);
-						ret &= entry.government_flags.emplace(government_type, flag).second;
+						bool ret = true;
+						if (flag_expected) {
+							Logger::error(
+								"Government key found when expect flag type override for ", government_type,
+								" in history of ", entry.get_country().get_identifier()
+							);
+							ret = false;
+						}
+						flag_expected = true;
 						government_type = nullptr;
+						ret &= government_type_manager.expect_government_type_identifier(
+							assign_variable_callback_pointer(government_type)
+						)(node);
+						return ret;
+					} else if (id == "flag") {
+						if (flag_expected) {
+							flag_expected = false;
+							GovernmentType const* flag_override_government_type = nullptr;
+							bool ret = government_type_manager.expect_government_type_identifier(
+								assign_variable_callback_pointer(flag_override_government_type)
+							)(node);
+							/* If the first government type is null, the "government" section will have already output
+							 * an error, so no need to output another one here. */
+							if (government_type != nullptr && flag_override_government_type != nullptr) {
+								ret &= entry.government_flag_overrides.emplace(
+									government_type, flag_override_government_type
+								).second;
+							}
+							return ret;
+						} else {
+							Logger::error(
+								"Flag key found when expecting government type for flag type override in history of ",
+								entry.get_country().get_identifier()
+							);
+							return false;
+						}
+					} else {
+						Logger::error(
+							"Invalid key ", id, " in government flag overrides in history of ",
+							entry.get_country().get_identifier()
+						);
+						return false;
 					}
-					return ret;
 				}
 			)(value);
+			if (flag_expected) {
+				Logger::error(
+					"Missing flag type override for government type ", government_type, " in history of ",
+					entry.get_country().get_identifier()
+				);
+				ret = false;
+			}
+			return ret;
 		},
-		"colonial_points", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(entry.colonial_points)), 
+		"colonial_points", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(entry.colonial_points)),
 		"set_country_flag", ZERO_OR_MORE, expect_identifier_or_string([&entry](std::string_view flag) -> bool {
 			return entry.country_flags.emplace(flag).second;
 		}),
diff --git a/src/openvic-simulation/history/CountryHistory.hpp b/src/openvic-simulation/history/CountryHistory.hpp
index 3f0965f..106b1c3 100644
--- a/src/openvic-simulation/history/CountryHistory.hpp
+++ b/src/openvic-simulation/history/CountryHistory.hpp
@@ -55,7 +55,7 @@ namespace OpenVic {
 		std::optional<fixed_point_t> PROPERTY(colonial_points);
 		string_set_t PROPERTY(country_flags);
 		string_set_t PROPERTY(global_flags);
-		std::map<GovernmentType const*, std::string> PROPERTY(government_flags);
+		std::map<GovernmentType const*, GovernmentType const*> PROPERTY(government_flag_overrides);
 		std::set<Decision const*> decisions;
 
 		CountryHistoryEntry(Country const& new_country, Date new_date);
diff --git a/src/openvic-simulation/history/HistoryMap.cpp b/src/openvic-simulation/history/HistoryMap.cpp
index b669208..7b5353f 100644
--- a/src/openvic-simulation/history/HistoryMap.cpp
+++ b/src/openvic-simulation/history/HistoryMap.cpp
@@ -6,10 +6,10 @@ using namespace OpenVic;
 
 HistoryEntry::HistoryEntry(Date new_date) : date { new_date } {}
 
-Date OpenVic::_get_start_date(GameManager const& game_manager) {
+Date _HistoryMapHelperFuncs::_get_start_date(GameManager const& game_manager) {
 	return game_manager.get_define_manager().get_start_date();
 }
 
-Date OpenVic::_get_end_date(GameManager const& game_manager) {
+Date _HistoryMapHelperFuncs::_get_end_date(GameManager const& game_manager) {
 	return game_manager.get_define_manager().get_end_date();
 }
diff --git a/src/openvic-simulation/history/HistoryMap.hpp b/src/openvic-simulation/history/HistoryMap.hpp
index 4a79474..576f00e 100644
--- a/src/openvic-simulation/history/HistoryMap.hpp
+++ b/src/openvic-simulation/history/HistoryMap.hpp
@@ -18,37 +18,29 @@ namespace OpenVic {
 
 	struct GameManager;
 
-	/* Helper functions to avoid cyclic dependency issues */
-	Date _get_start_date(GameManager const& game_manager);
-	Date _get_end_date(GameManager const& game_manager);
+	namespace _HistoryMapHelperFuncs {
+		/* Helper functions to avoid cyclic dependency issues */
+		Date _get_start_date(GameManager const& game_manager);
+		Date _get_end_date(GameManager const& game_manager);
+	}
 
 	template<std::derived_from<HistoryEntry> _Entry, typename... Args>
 	struct HistoryMap {
 		using entry_type = _Entry;
 
 	private:
+		std::map<Date, std::unique_ptr<entry_type>> PROPERTY(entries);
+
 		bool _try_load_history_entry(GameManager const& game_manager, Args... args, Date date, ast::NodeCPtr root) {
-			const Date end_date = _get_end_date(game_manager);
-			if (date > end_date) {
-				Logger::error("History entry ", date, " defined after end date ", end_date);
+			entry_type *const entry = _get_or_make_entry(game_manager, date);
+			if (entry != nullptr) {
+				return _load_history_entry(game_manager, args..., *entry, root);
+			} else {
 				return false;
 			}
-			typename decltype(entries)::iterator it = entries.find(date);
-			if (it == entries.end()) {
-				const std::pair<typename decltype(entries)::iterator, bool> result = entries.emplace(date, _make_entry(date));
-				if (result.second) {
-					it = result.first;
-				} else {
-					Logger::error("Failed to create history entry at date ", date);
-					return false;
-				}
-			}
-			return _load_history_entry(game_manager, args..., *it->second, root);
 		}
 
 	protected:
-		std::map<Date, std::unique_ptr<entry_type>> PROPERTY_ACCESS(entries, protected);
-
 		HistoryMap() = default;
 
 		virtual std::unique_ptr<entry_type> _make_entry(Date date) const = 0;
@@ -58,7 +50,7 @@ namespace OpenVic {
 		) = 0;
 
 		bool _load_history_file(GameManager const& game_manager, Args... args, ast::NodeCPtr root) {
-			return _try_load_history_entry(game_manager, args..., _get_start_date(game_manager), root);
+			return _try_load_history_entry(game_manager, args..., _HistoryMapHelperFuncs::_get_start_date(game_manager), root);
 		}
 
 		bool _load_history_sub_entry_callback(
@@ -87,6 +79,26 @@ namespace OpenVic {
 			return default_callback(key, value);
 		}
 
+		/* Returns history entry at specific date, if date doesn't have an entry creates one, if that fails returns nullptr. */
+		entry_type* _get_or_make_entry(GameManager const& game_manager, Date date) {
+			const Date end_date = _HistoryMapHelperFuncs::_get_end_date(game_manager);
+			if (date > end_date) {
+				Logger::error("History entry ", date, " defined after end date ", end_date);
+				return nullptr;
+			}
+			typename decltype(entries)::iterator it = entries.find(date);
+			if (it == entries.end()) {
+				const std::pair<typename decltype(entries)::iterator, bool> result = entries.emplace(date, _make_entry(date));
+				if (result.second) {
+					it = result.first;
+				} else {
+					Logger::error("Failed to create history entry at date ", date);
+					return nullptr;
+				}
+			}
+			return it->second.get();
+		}
+
 	public:
 		/* Returns history entry at specific date, if date doesn't have an entry returns nullptr. */
 		entry_type const* get_entry(Date date) const {
diff --git a/src/openvic-simulation/history/ProvinceHistory.cpp b/src/openvic-simulation/history/ProvinceHistory.cpp
index 61638bc..4e92300 100644
--- a/src/openvic-simulation/history/ProvinceHistory.cpp
+++ b/src/openvic-simulation/history/ProvinceHistory.cpp
@@ -2,7 +2,6 @@
 
 #include "openvic-simulation/GameManager.hpp"
 #include "openvic-simulation/dataloader/NodeTools.hpp"
-#include <memory>
 
 using namespace OpenVic;
 using namespace OpenVic::NodeTools;
@@ -16,37 +15,6 @@ std::unique_ptr<ProvinceHistoryEntry> ProvinceHistoryMap::_make_entry(Date date)
 	return std::unique_ptr<ProvinceHistoryEntry> { new ProvinceHistoryEntry { province, date } };
 }
 
-ProvinceHistoryEntry* ProvinceHistoryMap::_get_entry(Date date) {
-	return entries.at(date).get();
-}
-
-bool ProvinceHistoryEntry::_load_province_pop_history(PopManager const& pop_manager, ast::NodeCPtr root) {
-	return pop_manager.expect_pop_type_dictionary([this, &pop_manager](PopType const& type, ast::NodeCPtr pop_node) -> bool {
-			CultureManager const& culture_manager = pop_manager.get_culture_manager();
-			ReligionManager const& religion_manager = pop_manager.get_religion_manager();
-			Culture const* culture = nullptr;
-			Religion const* religion = nullptr;
-			Pop::pop_size_t size = 0;
-			bool ret = expect_dictionary_keys(
-				"culture", ONE_EXACTLY, culture_manager.expect_culture_identifier(assign_variable_callback_pointer(culture)),
-				"religion", ONE_EXACTLY, religion_manager.expect_religion_identifier(assign_variable_callback_pointer(religion)),
-				"size", ONE_EXACTLY, expect_uint(assign_variable_callback(size)),
-				"militancy", ZERO_OR_ONE, success_callback,
-				"rebel_type", ZERO_OR_ONE, success_callback
-			)(pop_node);
-			if (culture != nullptr && religion != nullptr && size > 0) {
-				pops.emplace_back(Pop { type, *culture, *religion, size });
-			} else {
-				Logger::warning(
-					"Some pop arguments are invalid: province = ", province, ", type = ", type, ", culture = ", culture,
-					", religion = ", religion, ", size = ", size
-				);
-			}
-			return ret;
-		}
-	)(root);
-}
-
 bool ProvinceHistoryMap::_load_history_entry(
 	GameManager const& game_manager, ProvinceHistoryEntry& entry, ast::NodeCPtr root
 ) {
@@ -167,6 +135,21 @@ ProvinceHistoryMap const* ProvinceHistoryManager::get_province_history(Province
 	}
 }
 
+ProvinceHistoryMap* ProvinceHistoryManager::_get_or_make_province_history(Province const& province) {
+	decltype(province_histories)::iterator it = province_histories.find(&province);
+	if (it == province_histories.end()) {
+		const std::pair<decltype(province_histories)::iterator, bool> result =
+			province_histories.emplace(&province, ProvinceHistoryMap { province });
+		if (result.second) {
+			it = result.first;
+		} else {
+			Logger::error("Failed to create province history map for province ", province.get_identifier());
+			return nullptr;
+		}
+	}
+	return &it->second;
+}
+
 bool ProvinceHistoryManager::load_province_history_file(
 	GameManager const& game_manager, Province const& province, ast::NodeCPtr root
 ) {
@@ -178,30 +161,52 @@ bool ProvinceHistoryManager::load_province_history_file(
 		return false;
 	}
 
-	decltype(province_histories)::iterator it = province_histories.find(&province);
-	if (it == province_histories.end()) {
-		const std::pair<decltype(province_histories)::iterator, bool> result =
-			province_histories.emplace(&province, ProvinceHistoryMap { province });
-		if (result.second) {
-			it = result.first;
-		} else {
-			Logger::error("Failed to create province history map for province ", province.get_identifier());
-			return false;
-		}
+	ProvinceHistoryMap* province_history = _get_or_make_province_history(province);
+	if (province_history != nullptr) {
+		return province_history->_load_history_file(game_manager, root);
+	} else {
+		return false;
 	}
-	ProvinceHistoryMap& province_history = it->second;
-
-	return province_history._load_history_file(game_manager, root);
 }
 
-bool ProvinceHistoryManager::load_pop_history_file(GameManager const& game_manager, Date date, ast::NodeCPtr root) {
+bool ProvinceHistoryEntry::_load_province_pop_history(
+	GameManager const& game_manager, ast::NodeCPtr root, bool *non_integer_size
+) {
 	PopManager const& pop_manager = game_manager.get_pop_manager();
-	return game_manager.get_map().expect_province_dictionary([this, &pop_manager, date](Province const& province, ast::NodeCPtr node) -> bool {
-		if (province_histories.contains(&province)) {
-			ProvinceHistoryEntry* entry = province_histories.at(&province)._get_entry(date);
-			return entry != nullptr
-				? entry->_load_province_pop_history(pop_manager, node)
-				: false;
-		} else return false;
-	})(root);
-}
\ No newline at end of file
+	RebelManager const& rebel_manager = game_manager.get_politics_manager().get_rebel_manager();
+	return pop_manager.expect_pop_type_dictionary(
+		[this, &pop_manager, &rebel_manager, non_integer_size](PopType const& pop_type, ast::NodeCPtr pop_node) -> bool {
+			return pop_manager.load_pop_into_vector(rebel_manager, pops, pop_type, pop_node, non_integer_size);
+		}
+	)(root);
+}
+
+bool ProvinceHistoryMap::_load_province_pop_history(
+	GameManager const& game_manager, Date date, ast::NodeCPtr root, bool *non_integer_size
+) {
+	ProvinceHistoryEntry* entry = _get_or_make_entry(game_manager, date);
+	if (entry != nullptr) {
+		return entry->_load_province_pop_history(game_manager, root, non_integer_size);
+	} else {
+		return false;
+	}
+}
+
+bool ProvinceHistoryManager::load_pop_history_file(
+	GameManager const& game_manager, Date date, ast::NodeCPtr root, bool *non_integer_size
+) {
+	if (locked) {
+		Logger::error("Attempted to load pop history file after province history registry was locked!");
+		return false;
+	}
+	return game_manager.get_map().expect_province_dictionary(
+		[this, &game_manager, date, non_integer_size](Province const& province, ast::NodeCPtr node) -> bool {
+			ProvinceHistoryMap* province_history = _get_or_make_province_history(province);
+			if (province_history != nullptr) {
+				return province_history->_load_province_pop_history(game_manager, date, node, non_integer_size);
+			} else {
+				return false;
+			}
+		}
+	)(root);
+}
diff --git a/src/openvic-simulation/history/ProvinceHistory.hpp b/src/openvic-simulation/history/ProvinceHistory.hpp
index b69408e..c3c8e67 100644
--- a/src/openvic-simulation/history/ProvinceHistory.hpp
+++ b/src/openvic-simulation/history/ProvinceHistory.hpp
@@ -17,7 +17,6 @@ namespace OpenVic {
 
 	struct ProvinceHistoryEntry : HistoryEntry {
 		friend struct ProvinceHistoryMap;
-		friend struct ProvinceHistoryManager;
 
 	private:
 		Province const& PROPERTY(province);
@@ -37,7 +36,8 @@ namespace OpenVic {
 		std::vector<Pop> PROPERTY(pops);
 
 		ProvinceHistoryEntry(Province const& new_province, Date new_date);
-		bool _load_province_pop_history(PopManager const& pop_manager, ast::NodeCPtr root);
+
+		bool _load_province_pop_history(GameManager const& game_manager, ast::NodeCPtr root, bool *non_integer_size);
 	};
 
 	struct ProvinceHistoryManager;
@@ -48,13 +48,16 @@ namespace OpenVic {
 	private:
 		Province const& PROPERTY(province);
 
-		ProvinceHistoryEntry* _get_entry(Date date);
-
 	protected:
 		ProvinceHistoryMap(Province const& new_province);
 
 		std::unique_ptr<ProvinceHistoryEntry> _make_entry(Date date) const override;
 		bool _load_history_entry(GameManager const& game_manager, ProvinceHistoryEntry& entry, ast::NodeCPtr root) override;
+
+	private:
+		bool _load_province_pop_history(
+			GameManager const& game_manager, Date date, ast::NodeCPtr root, bool *non_integer_size
+		);
 	};
 
 	struct ProvinceHistoryManager {
@@ -62,6 +65,8 @@ namespace OpenVic {
 		std::map<Province const*, ProvinceHistoryMap> PROPERTY(province_histories);
 		bool locked = false;
 
+		ProvinceHistoryMap* _get_or_make_province_history(Province const& province);
+
 	public:
 		ProvinceHistoryManager() = default;
 
@@ -71,6 +76,6 @@ namespace OpenVic {
 		ProvinceHistoryMap const* get_province_history(Province const* province) const;
 
 		bool load_province_history_file(GameManager const& game_manager, Province const& province, ast::NodeCPtr root);
-		bool load_pop_history_file(GameManager const& game_manager, Date date, ast::NodeCPtr root);
+		bool load_pop_history_file(GameManager const& game_manager, Date date, ast::NodeCPtr root, bool *non_integer_size);
 	};
 } // namespace OpenVic
diff --git a/src/openvic-simulation/interface/UI.hpp b/src/openvic-simulation/interface/UI.hpp
index 12eadb5..ce9336c 100644
--- a/src/openvic-simulation/interface/UI.hpp
+++ b/src/openvic-simulation/interface/UI.hpp
@@ -4,7 +4,7 @@
 
 namespace OpenVic {
 
-	class UIManager { 
+	class UIManager {
 		NamedInstanceRegistry<GFX::Sprite> IDENTIFIER_REGISTRY(sprite);
 		NamedInstanceRegistry<GUI::Scene, UIManager const&> IDENTIFIER_REGISTRY(scene);
 		IdentifierRegistry<GFX::Font> IDENTIFIER_REGISTRY(font);
diff --git a/src/openvic-simulation/map/Map.cpp b/src/openvic-simulation/map/Map.cpp
index 18f72cf..5d8c6e2 100644
--- a/src/openvic-simulation/map/Map.cpp
+++ b/src/openvic-simulation/map/Map.cpp
@@ -294,8 +294,15 @@ bool Map::apply_history_to_provinces(ProvinceHistoryManager const& history_manag
 		if (!province.get_water()) {
 			ProvinceHistoryMap const* history_map = history_manager.get_province_history(&province);
 			if (history_map != nullptr) {
+				ProvinceHistoryEntry const* pop_history_entry = nullptr;
 				for (ProvinceHistoryEntry const* entry : history_map->get_entries_up_to(date)) {
 					province.apply_history_to_province(entry);
+					if (!entry->get_pops().empty()) {
+						pop_history_entry = entry;
+					}
+				}
+				if (pop_history_entry != nullptr) {
+					province.add_pop_vec(pop_history_entry->get_pops());
 				}
 			}
 		}
diff --git a/src/openvic-simulation/map/Province.cpp b/src/openvic-simulation/map/Province.cpp
index 19d06df..8094463 100644
--- a/src/openvic-simulation/map/Province.cpp
+++ b/src/openvic-simulation/map/Province.cpp
@@ -68,6 +68,19 @@ bool Province::add_pop(Pop&& pop) {
 	}
 }
 
+bool Province::add_pop_vec(std::vector<Pop> const& pop_vec) {
+	if (!get_water()) {
+		pops.reserve(pops.size() + pop_vec.size());
+		for (Pop const& pop : pop_vec) {
+			pops.push_back(pop);
+		}
+		return true;
+	} else {
+		Logger::error("Trying to add pop vector to water province ", get_identifier());
+		return false;
+	}
+}
+
 size_t Province::get_pop_count() const {
 	return pops.size();
 }
diff --git a/src/openvic-simulation/map/Province.hpp b/src/openvic-simulation/map/Province.hpp
index dc82deb..1eea458 100644
--- a/src/openvic-simulation/map/Province.hpp
+++ b/src/openvic-simulation/map/Province.hpp
@@ -101,6 +101,7 @@ namespace OpenVic {
 		bool expand_building(std::string_view building_type_identifier);
 
 		bool add_pop(Pop&& pop);
+		bool add_pop_vec(std::vector<Pop> const& pop_vec);
 		size_t get_pop_count() const;
 		void update_pops();
 
diff --git a/src/openvic-simulation/misc/Decision.cpp b/src/openvic-simulation/misc/Decision.cpp
index 9200688..f1f2b6c 100644
--- a/src/openvic-simulation/misc/Decision.cpp
+++ b/src/openvic-simulation/misc/Decision.cpp
@@ -34,7 +34,7 @@ bool DecisionManager::add_decision(
 		}
 	}
 
-	return decisions.add_item({ 
+	return decisions.add_item({
 		identifier,
 		alert,
 		news,
@@ -42,7 +42,7 @@ bool DecisionManager::add_decision(
 		news_desc_long,
 		news_desc_medium,
 		news_desc_short,
-		picture 
+		picture
 	}, duplicate_warning_callback);
 }
 
diff --git a/src/openvic-simulation/politics/Government.cpp b/src/openvic-simulation/politics/Government.cpp
index a226518..fc93cc3 100644
--- a/src/openvic-simulation/politics/Government.cpp
+++ b/src/openvic-simulation/politics/Government.cpp
@@ -111,3 +111,9 @@ bool GovernmentTypeManager::load_government_types_file(IdeologyManager const& id
 
 	return ret;
 }
+
+bool GovernmentTypeManager::is_valid_flag_type(std::string_view type) const {
+	return std::any_of(flag_types.begin(), flag_types.end(), [type](std::string const& flag_type) -> bool {
+		return flag_type == type;
+	});
+}
diff --git a/src/openvic-simulation/politics/Government.hpp b/src/openvic-simulation/politics/Government.hpp
index 4682bbf..34a9194 100644
--- a/src/openvic-simulation/politics/Government.hpp
+++ b/src/openvic-simulation/politics/Government.hpp
@@ -38,5 +38,7 @@ namespace OpenVic {
 		);
 
 		bool load_government_types_file(IdeologyManager const& ideology_manager, ast::NodeCPtr root);
+
+		bool is_valid_flag_type(std::string_view type) const;
 	};
 } // namespace OpenVic
diff --git a/src/openvic-simulation/pop/Pop.cpp b/src/openvic-simulation/pop/Pop.cpp
index eae2e52..2d85dce 100644
--- a/src/openvic-simulation/pop/Pop.cpp
+++ b/src/openvic-simulation/pop/Pop.cpp
@@ -4,14 +4,18 @@
 
 #include "openvic-simulation/dataloader/NodeTools.hpp"
 #include "openvic-simulation/map/Province.hpp"
+#include "openvic-simulation/politics/Rebel.hpp"
 #include "openvic-simulation/utility/Logger.hpp"
 
 using namespace OpenVic;
 using namespace OpenVic::NodeTools;
 
 Pop::Pop(
-	PopType const& new_type, Culture const& new_culture, Religion const& new_religion, pop_size_t new_size
-) : type { new_type }, culture { new_culture }, religion { new_religion }, size { new_size } {
+	PopType const& new_type, Culture const& new_culture, Religion const& new_religion, pop_size_t new_size,
+	fixed_point_t new_militancy, fixed_point_t new_consciousness, RebelType const* new_rebel_type
+) : type { new_type }, culture { new_culture }, religion { new_religion }, size { new_size }, num_promoted { 0 },
+	num_demoted { 0 }, num_migrated { 0 }, militancy { new_militancy }, consciousness { new_consciousness },
+	rebel_type { new_rebel_type } {
 	assert(size > 0);
 }
 
@@ -156,3 +160,36 @@ bool PopManager::load_pop_type_file(
 	);
 	return ret;
 }
+
+bool PopManager::load_pop_into_vector(
+	RebelManager const& rebel_manager, std::vector<Pop>& vec, PopType const& type, ast::NodeCPtr pop_node,
+	bool *non_integer_size
+) const {
+	Culture const* culture = nullptr;
+	Religion const* religion = nullptr;
+	fixed_point_t size = 0; /* Some genius filled later start dates with non-integer sized pops */
+	fixed_point_t militancy = 0, consciousness = 0;
+	RebelType const* rebel_type = nullptr;
+
+	bool ret = expect_dictionary_keys(
+		"culture", ONE_EXACTLY, culture_manager.expect_culture_identifier(assign_variable_callback_pointer(culture)),
+		"religion", ONE_EXACTLY, religion_manager.expect_religion_identifier(assign_variable_callback_pointer(religion)),
+		"size", ONE_EXACTLY, expect_fixed_point(assign_variable_callback(size)),
+		"militancy", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(militancy)),
+		"consciousness", ZERO_OR_ONE, expect_fixed_point(assign_variable_callback(consciousness)),
+		"rebel_type", ZERO_OR_ONE, rebel_manager.expect_rebel_type_identifier(assign_variable_callback_pointer(rebel_type))
+	)(pop_node);
+
+	if (non_integer_size != nullptr && !size.is_integer()) {
+		*non_integer_size = true;
+	}
+
+	if (culture != nullptr && religion != nullptr && size >= 1) {
+		vec.emplace_back(Pop { type, *culture, *religion, size.to_int64_t(), militancy, consciousness, rebel_type });
+	} else {
+		Logger::warning(
+			"Some pop arguments are invalid: culture = ", culture, ", religion = ", religion, ", size = ", size
+		);
+	}
+	return ret;
+}
diff --git a/src/openvic-simulation/pop/Pop.hpp b/src/openvic-simulation/pop/Pop.hpp
index 12daaf6..6302a58 100644
--- a/src/openvic-simulation/pop/Pop.hpp
+++ b/src/openvic-simulation/pop/Pop.hpp
@@ -9,6 +9,8 @@ namespace OpenVic {
 
 	struct PopManager;
 	struct PopType;
+	struct RebelType;
+	struct RebelManager;
 
 	/* REQUIREMENTS:
 	 * POP-18, POP-19, POP-20, POP-21, POP-34, POP-35, POP-36, POP-37
@@ -27,9 +29,17 @@ namespace OpenVic {
 		pop_size_t PROPERTY(num_demoted);
 		pop_size_t PROPERTY(num_migrated);
 
+		fixed_point_t PROPERTY(militancy);
+		fixed_point_t PROPERTY(consciousness);
+		RebelType const* PROPERTY(rebel_type);
+
+		Pop(
+			PopType const& new_type, Culture const& new_culture, Religion const& new_religion, pop_size_t new_size,
+			fixed_point_t new_militancy, fixed_point_t new_consciousness, RebelType const* new_rebel_type
+		);
+
 	public:
-		Pop(PopType const& new_type, Culture const& new_culture, Religion const& new_religion, pop_size_t new_size);
-		Pop(Pop const&) = delete;
+		Pop(Pop const&) = default;
 		Pop(Pop&&) = default;
 		Pop& operator=(Pop const&) = delete;
 		Pop& operator=(Pop&&) = delete;
@@ -110,5 +120,9 @@ namespace OpenVic {
 		bool load_pop_type_file(
 			std::string_view filestem, UnitManager const& unit_manager, GoodManager const& good_manager, ast::NodeCPtr root
 		);
+		bool load_pop_into_vector(
+			RebelManager const& rebel_manager, std::vector<Pop>& vec, PopType const& type, ast::NodeCPtr pop_node,
+			bool *non_integer_size
+		) const;
 	};
 }
diff --git a/src/openvic-simulation/types/fixed_point/FixedPoint.hpp b/src/openvic-simulation/types/fixed_point/FixedPoint.hpp
index ba6790f..75abefb 100644
--- a/src/openvic-simulation/types/fixed_point/FixedPoint.hpp
+++ b/src/openvic-simulation/types/fixed_point/FixedPoint.hpp
@@ -214,6 +214,10 @@ namespace OpenVic {
 			return value & (ONE - 1);
 		}
 
+		constexpr bool is_integer() const {
+			return get_frac() == 0;
+		}
+
 		constexpr int64_t to_int64_t() const {
 			return value >> PRECISION;
 		}