From b0c3ba3f91926b0c95625bdbf4aab69269130b13 Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Thu, 9 May 2024 10:06:02 -0400 Subject: Add runtime encoding detection and conversion Win-1251/1252 detection is a reduced C++ version of https://github.com/hsivonen/chardetng Add manually-specified encoding fallback Add default system encoding fallback Add error recovery to v2script Add unknown encoding detection warning Remove csv::Parser templating Fix lua files dropping data Update lexy to foonathan/lexy@1e5d99fa3826b1c3c8628d3a11117fb4fb4cc0d0 Remove exclusive reliance on lexy::default_encoding for v2script Move internal concepts to src/openvic-detail/InternalConcepts.hpp Move contents of DetectUtf8.hpp to src/detail/Detect.hpp Move openvic-dataloader/AbstractSyntaxTree.hpp to src Move DiagnosticLogger.hpp to src Move File.hpp to src Move openvic-dataloader/detail/utlity files to openvic-dataloader/detail Add ovdl::utility::type_concat Add ovdl::utility::type_prepend Add ovdl::utility::is_instance_of Overhaul parse error messages --- include/openvic-dataloader/v2script/Parser.hpp | 30 ++++++++++++++------------ 1 file changed, 16 insertions(+), 14 deletions(-) (limited to 'include/openvic-dataloader/v2script/Parser.hpp') diff --git a/include/openvic-dataloader/v2script/Parser.hpp b/include/openvic-dataloader/v2script/Parser.hpp index f9f0ce8..1f6b158 100644 --- a/include/openvic-dataloader/v2script/Parser.hpp +++ b/include/openvic-dataloader/v2script/Parser.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -10,8 +11,9 @@ #include #include #include -#include -#include +#include +#include +#include #include #include @@ -25,20 +27,20 @@ namespace ovdl::v2script { Parser(); Parser(std::basic_ostream& error_stream); - static Parser from_buffer(const char* data, std::size_t size); - static Parser from_buffer(const char* start, const char* end); - static Parser from_string(const std::string_view string); - static Parser from_file(const char* path); - static Parser from_file(const std::filesystem::path& path); + static Parser from_buffer(const char* data, std::size_t size, std::optional encoding_fallback = std::nullopt); + static Parser from_buffer(const char* start, const char* end, std::optional encoding_fallback = std::nullopt); + static Parser from_string(const std::string_view string, std::optional encoding_fallback = std::nullopt); + static Parser from_file(const char* path, std::optional encoding_fallback = std::nullopt); + static Parser from_file(const std::filesystem::path& path, std::optional encoding_fallback = std::nullopt); - constexpr Parser& load_from_buffer(const char* data, std::size_t size); - constexpr Parser& load_from_buffer(const char* start, const char* end); - constexpr Parser& load_from_string(const std::string_view string); - Parser& load_from_file(const char* path); - Parser& load_from_file(const std::filesystem::path& path); + constexpr Parser& load_from_buffer(const char* data, std::size_t size, std::optional encoding_fallback = std::nullopt); + constexpr Parser& load_from_buffer(const char* start, const char* end, std::optional encoding_fallback = std::nullopt); + constexpr Parser& load_from_string(const std::string_view string, std::optional encoding_fallback = std::nullopt); + Parser& load_from_file(const char* path, std::optional encoding_fallback = std::nullopt); + Parser& load_from_file(const std::filesystem::path& path, std::optional encoding_fallback = std::nullopt); - constexpr Parser& load_from_file(const detail::HasCstr auto& path) { - return load_from_file(path.c_str()); + constexpr Parser& load_from_file(const detail::HasCstr auto& path, std::optional encoding_fallback = std::nullopt) { + return load_from_file(path.c_str(), encoding_fallback); } bool simple_parse(); -- cgit v1.2.3-56-ga3b1