From b0c3ba3f91926b0c95625bdbf4aab69269130b13 Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Thu, 9 May 2024 10:06:02 -0400 Subject: Add runtime encoding detection and conversion Win-1251/1252 detection is a reduced C++ version of https://github.com/hsivonen/chardetng Add manually-specified encoding fallback Add default system encoding fallback Add error recovery to v2script Add unknown encoding detection warning Remove csv::Parser templating Fix lua files dropping data Update lexy to foonathan/lexy@1e5d99fa3826b1c3c8628d3a11117fb4fb4cc0d0 Remove exclusive reliance on lexy::default_encoding for v2script Move internal concepts to src/openvic-detail/InternalConcepts.hpp Move contents of DetectUtf8.hpp to src/detail/Detect.hpp Move openvic-dataloader/AbstractSyntaxTree.hpp to src Move DiagnosticLogger.hpp to src Move File.hpp to src Move openvic-dataloader/detail/utlity files to openvic-dataloader/detail Add ovdl::utility::type_concat Add ovdl::utility::type_prepend Add ovdl::utility::is_instance_of Overhaul parse error messages --- src/openvic-dataloader/csv/CsvGrammar.hpp | 244 +++++++++++++++------------ src/openvic-dataloader/csv/CsvParseState.hpp | 26 +-- src/openvic-dataloader/csv/Parser.cpp | 182 +++++++++----------- 3 files changed, 222 insertions(+), 230 deletions(-) (limited to 'src/openvic-dataloader/csv') diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp index 5451f26..19aee54 100644 --- a/src/openvic-dataloader/csv/CsvGrammar.hpp +++ b/src/openvic-dataloader/csv/CsvGrammar.hpp @@ -9,22 +9,20 @@ #include #include +#include #include +#include #include +#include +#include +#include +#include "detail/Convert.hpp" +#include "detail/InternalConcepts.hpp" #include "detail/dsl.hpp" // Grammar Definitions // namespace ovdl::csv::grammar { - using EncodingType = ovdl::csv::EncodingType; - - template - concept ParseChars = requires() { - { T::character }; - { T::control }; - }; - - template struct ParseOptions { /// @brief Seperator character char SepChar; @@ -33,12 +31,34 @@ namespace ovdl::csv::grammar { /// @brief Paradox-style localization escape characters /// @note Is ignored if SupportStrings is true char EscapeChar; + }; - static constexpr auto parse_chars = T {}; - static constexpr auto character = parse_chars.character; - static constexpr auto control = parse_chars.control; + struct ConvertErrorHandler { + static constexpr void on_invalid_character(detail::IsStateType auto& state, auto reader) { + state.logger().warning("invalid character value '{}' found", static_cast(reader.peek())) // + .primary(BasicNodeLocation { reader.position() }, "here") + .finish(); + } }; + constexpr bool IsUtf8(auto encoding) { + return std::same_as, lexy::utf8_char_encoding>; + } + + template + constexpr auto convert_as_string = convert::convert_as_string< + String, + ConvertErrorHandler>; + + constexpr auto ansi_character = lexy::dsl::ascii::character / dsl::lit_b_range<0x80, 0xFF>; + constexpr auto ansi_control = + lexy::dsl::ascii::control / + lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> / + lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>; + + constexpr auto utf_character = lexy::dsl::unicode::character; + constexpr auto utf_control = lexy::dsl::unicode::control; + constexpr auto escaped_symbols = lexy::symbol_table // .map<'"'>('"') .map<'\''>('\'') @@ -55,38 +75,95 @@ namespace ovdl::csv::grammar { template struct CsvGrammar { - struct StringValue { - static constexpr auto rule = [] { - // Arbitrary code points - auto c = Options.character - Options.control; + struct StringValue : lexy::scan_production, + lexy::token_production { + + template + static constexpr scan_result scan(lexy::rule_scanner& scanner, detail::IsFileParseState auto& state) { + using encoding = typename Reader::encoding; + + constexpr auto rule = [] { + // Arbitrary code points + auto c = [] { + if constexpr (std::same_as || std::same_as) { + return ansi_character - ansi_control; + } else { + return utf_character - utf_control; + } + }(); - auto back_escape = lexy::dsl::backslash_escape // - .symbol(); + auto back_escape = lexy::dsl::backslash_escape // + .symbol(); - auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // - .template symbol(); + auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + .template symbol(); - return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape); - }(); + return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape); + }(); + + lexy::scan_result str_result = scanner.template parse(rule); + if (!scanner || !str_result) + return lexy::scan_failed; + return str_result.value(); + } - static constexpr auto value = lexy::as_string; + static constexpr auto rule = lexy::dsl::peek(lexy::dsl::lit_c<'"'>) >> lexy::dsl::scan; + + static constexpr auto value = convert_as_string >> lexy::forward; }; - struct PlainValue { - static constexpr auto rule = [] { + struct PlainValue : lexy::scan_production, + lexy::token_production { + + template + static constexpr auto _escape_check = character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline); + + template + static constexpr scan_result scan(lexy::rule_scanner& scanner, detail::IsFileParseState auto& state) { + using encoding = typename Reader::encoding; + + constexpr auto rule = [] { + constexpr auto character = [] { + if constexpr (std::same_as || std::same_as) { + return ansi_character; + } else { + return utf_character; + } + }(); + + if constexpr (Options.SupportStrings) { + return lexy::dsl::identifier(character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline)); + } else { + auto escape_check_char = _escape_check; + auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>; + auto id_segment = lexy::dsl::identifier(id_check_char); + auto escape_segement = lexy::dsl::token(escape_check_char); + auto escape_sym = lexy::dsl::symbol(escape_segement); + auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym; + return lexy::dsl::list(id_segment | escape_rule); + } + }(); + if constexpr (Options.SupportStrings) { - return lexy::dsl::identifier(Options.character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline)); + auto lexeme_result = scanner.template parse>(rule); + if (!scanner || !lexeme_result) + return lexy::scan_failed; + return std::string { lexeme_result.value().begin(), lexeme_result.value().end() }; } else { - auto escape_check_char = Options.character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline); - auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>; - auto id_segment = lexy::dsl::identifier(id_check_char); - auto escape_segement = lexy::dsl::token(escape_check_char); - auto escape_sym = lexy::dsl::symbol(escape_segement); - auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym; - return lexy::dsl::list(id_segment | escape_rule); + lexy::scan_result str_result = scanner.template parse(rule); + if (!scanner || !str_result) + return lexy::scan_failed; + return str_result.value(); } - }(); - static constexpr auto value = lexy::as_string; + } + + static constexpr auto rule = + dsl::peek( + _escape_check, + _escape_check) >> + lexy::dsl::scan; + + static constexpr auto value = convert_as_string >> lexy::forward; }; struct Value { @@ -114,17 +191,17 @@ namespace ovdl::csv::grammar { static constexpr auto rule = lexy::dsl::list(lexy::dsl::p, lexy::dsl::trailing_sep(lexy::dsl::p)); static constexpr auto value = lexy::fold_inplace( std::initializer_list {}, - [](ovdl::csv::LineObject& result, auto&& arg) { - if constexpr (std::is_same_v, std::size_t>) { - // Count seperators, adds to previous value, making it a position - using position_type = ovdl::csv::LineObject::position_type; - result.emplace_back(static_cast(arg + result.back().first), ""); + [](ovdl::csv::LineObject& result, std::size_t&& arg) { + // Count seperators, adds to previous value, making it a position + using position_type = ovdl::csv::LineObject::position_type; + result.emplace_back(static_cast(arg + result.back().first), ""); + }, + [](ovdl::csv::LineObject& result, std::string&& arg) { + if (result.empty()) { + result.emplace_back(0u, LEXY_MOV(arg)); } else { - if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); - else { - auto& [pos, value] = result.back(); - value = arg; - } + auto& [pos, value] = result.back(); + value = LEXY_MOV(arg); } }); }; @@ -169,74 +246,17 @@ namespace ovdl::csv::grammar { static constexpr auto value = lexy::as_list>; }; - template - using CommaFile = File { ',', false, '$' }>; - template - using ColonFile = File { ':', false, '$' }>; - template - using SemiColonFile = File { ';', false, '$' }>; - template - using TabFile = File { '\t', false, '$' }>; - template - using BarFile = File { '|', false, '$' }>; - - namespace strings { - template - using CommaFile = File { ',', true, '$' }>; - template - using ColonFile = File { ':', true, '$' }>; - template - using SemiColonFile = File { ';', true, '$' }>; - template - using TabFile = File { '\t', true, '$' }>; - template - using BarFile = File { '|', true, '$' }>; - } -} - -namespace ovdl::csv::grammar::windows1252 { - struct windows1252_t { - static constexpr auto character = dsl::make_range<0x01, 0xFF>(); - static constexpr auto control = - lexy::dsl::ascii::control / - lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> / - lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>; - }; - - using CommaFile = CommaFile; - using ColonFile = ColonFile; - using SemiColonFile = SemiColonFile; - using TabFile = TabFile; - using BarFile = BarFile; - - namespace strings { - using CommaFile = grammar::strings::CommaFile; - using ColonFile = grammar::strings::ColonFile; - using SemiColonFile = grammar::strings::SemiColonFile; - using TabFile = grammar::strings::TabFile; - using BarFile = grammar::strings::BarFile; - - } -} - -namespace ovdl::csv::grammar::utf8 { - struct unicode_t { - static constexpr auto character = lexy::dsl::unicode::character; - static constexpr auto control = lexy::dsl::unicode::control; - }; - - using CommaFile = CommaFile; - using ColonFile = ColonFile; - using SemiColonFile = SemiColonFile; - using TabFile = TabFile; - using BarFile = BarFile; + using CommaFile = File; + using ColonFile = File; + using SemiColonFile = File; + using TabFile = File; + using BarFile = File; namespace strings { - using CommaFile = grammar::strings::CommaFile; - using ColonFile = grammar::strings::ColonFile; - using SemiColonFile = grammar::strings::SemiColonFile; - using TabFile = grammar::strings::TabFile; - using BarFile = grammar::strings::BarFile; - + using CommaFile = File; + using ColonFile = File; + using SemiColonFile = File; + using TabFile = File; + using BarFile = File; } } \ No newline at end of file diff --git a/src/openvic-dataloader/csv/CsvParseState.hpp b/src/openvic-dataloader/csv/CsvParseState.hpp index 2390453..ee60c34 100644 --- a/src/openvic-dataloader/csv/CsvParseState.hpp +++ b/src/openvic-dataloader/csv/CsvParseState.hpp @@ -1,28 +1,16 @@ #pragma once -#include -#include #include #include #include -template -struct LexyEncodingFrom { -}; +#include "File.hpp" +#include "ParseState.hpp" +#include "detail/InternalConcepts.hpp" -template<> -struct LexyEncodingFrom { - using encoding = lexy::default_encoding; -}; +namespace ovdl::csv { + using CsvParseState = ovdl::FileParseState>>; -template<> -struct LexyEncodingFrom { - using encoding = lexy::utf8_char_encoding; -}; - -template -using CsvFile = ovdl::BasicFile::encoding, std::vector>; - -template -using CsvParseState = ovdl::FileParseState>; \ No newline at end of file + static_assert(detail::IsFileParseState, "CsvParseState failed IsFileParseState concept"); +} \ No newline at end of file diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp index 361f6ad..5dbee32 100644 --- a/src/openvic-dataloader/csv/Parser.cpp +++ b/src/openvic-dataloader/csv/Parser.cpp @@ -1,11 +1,14 @@ +#include +#include +#include #include -#include +#include #include #include -#include +#include #include -#include +#include #include #include @@ -22,15 +25,27 @@ using namespace ovdl::csv; /// ParseHandler /// -template -struct Parser::ParseHandler final : detail::BasicFileParseHandler> { +struct Parser::ParseHandler final : detail::BasicFileParseHandler { template std::optional parse() { - auto result = lexy::parse(this->buffer(), *this->_parse_state, this->_parse_state->logger().error_callback()); + auto result = [&] { + switch (parse_state().encoding()) { + using enum detail::Encoding; + case Ascii: + case Utf8: + return lexy::parse(buffer(), parse_state(), parse_state().logger().error_callback()); + case Unknown: + case Windows1251: + case Windows1252: + return lexy::parse(buffer(), parse_state(), parse_state().logger().error_callback()); + default: + ovdl::detail::unreachable(); + } + }(); if (!result) { - return this->_parse_state->logger().get_errors(); + return this->parse_state().logger().get_errors(); } - _lines = std::move(result.value()); + _lines = LEXY_MOV(result).value(); return std::nullopt; } @@ -42,55 +57,45 @@ private: std::vector _lines; }; -/// BufferHandler /// +/// ParserHandler /// -template -Parser::Parser() +Parser::Parser() : _parse_handler(std::make_unique()) { set_error_log_to_null(); } -template -Parser::Parser(std::basic_ostream& error_stream) +Parser::Parser(std::basic_ostream& error_stream) : _parse_handler(std::make_unique()) { set_error_log_to(error_stream); } -template -Parser::Parser(Parser&&) = default; -template -Parser& Parser::operator=(Parser&&) = default; -template -Parser::~Parser() = default; +Parser::Parser(Parser&&) = default; +Parser& Parser::operator=(Parser&&) = default; +Parser::~Parser() = default; -template -Parser Parser::from_buffer(const char* data, std::size_t size) { +Parser Parser::from_buffer(const char* data, std::size_t size, std::optional encoding_fallback) { Parser result; - return std::move(result.load_from_buffer(data, size)); + return std::move(result.load_from_buffer(data, size, encoding_fallback)); } -template -Parser Parser::from_buffer(const char* start, const char* end) { +Parser Parser::from_buffer(const char* start, const char* end, std::optional encoding_fallback) { Parser result; - return std::move(result.load_from_buffer(start, end)); + return std::move(result.load_from_buffer(start, end, encoding_fallback)); } -template -Parser Parser::from_string(const std::string_view string) { +Parser Parser::from_string(const std::string_view string, std::optional encoding_fallback) { Parser result; - return std::move(result.load_from_string(string)); + return std::move(result.load_from_string(string, encoding_fallback)); } -template -Parser Parser::from_file(const char* path) { +Parser Parser::from_file(const char* path, std::optional encoding_fallback) { Parser result; - return std::move(result.load_from_file(path)); + return std::move(result.load_from_file(path, encoding_fallback)); } -template -Parser Parser::from_file(const std::filesystem::path& path) { +Parser Parser::from_file(const std::filesystem::path& path, std::optional encoding_fallback) { Parser result; - return std::move(result.load_from_file(path)); + return std::move(result.load_from_file(path, encoding_fallback)); } /// @@ -106,9 +111,8 @@ Parser Parser::from_file(const std::filesystem::path& path) /// @param func /// @param args /// -template template -constexpr void Parser::_run_load_func(detail::LoadCallback auto func, Args... args) { +constexpr void Parser::_run_load_func(detail::LoadCallback auto func, Args... args) { _has_fatal_error = false; auto error = func(_parse_handler.get(), std::forward(args)...); auto error_message = _parse_handler->make_error_from(error); @@ -122,82 +126,66 @@ constexpr void Parser::_run_load_func(detail::LoadCallback -constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size) { +constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size, std::optional encoding_fallback) { // Type can't be deduced? - _run_load_func(std::mem_fn(&ParseHandler::load_buffer_size), data, size); + _run_load_func(std::mem_fn(&ParseHandler::load_buffer_size), data, size, encoding_fallback); return *this; } -template -constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) { +constexpr Parser& Parser::load_from_buffer(const char* start, const char* end, std::optional encoding_fallback) { // Type can't be deduced? - _run_load_func(std::mem_fn(&ParseHandler::load_buffer), start, end); + _run_load_func(std::mem_fn(&ParseHandler::load_buffer), start, end, encoding_fallback); return *this; } -template -constexpr Parser& Parser::load_from_string(const std::string_view string) { - return load_from_buffer(string.data(), string.size()); +constexpr Parser& Parser::load_from_string(const std::string_view string, std::optional encoding_fallback) { + return load_from_buffer(string.data(), string.size(), encoding_fallback); } -template -Parser& Parser::load_from_file(const char* path) { +Parser& Parser::load_from_file(const char* path, std::optional encoding_fallback) { set_file_path(path); // Type can be deduced?? - _run_load_func(std::mem_fn(&ParseHandler::load_file), path); + _run_load_func(std::mem_fn(&ParseHandler::load_file), get_file_path().data(), encoding_fallback); return *this; } -template -Parser& Parser::load_from_file(const std::filesystem::path& path) { - return load_from_file(path.string().c_str()); +Parser& Parser::load_from_file(const std::filesystem::path& path, std::optional encoding_fallback) { + return load_from_file(path.string().c_str(), encoding_fallback); } -template -bool Parser::parse_csv(bool handle_strings) { +bool Parser::parse_csv(bool handle_strings) { if (!_parse_handler->is_valid()) { return false; } - std::optional::error_range> errors; - // auto report_error = ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }); - if constexpr (Encoding == EncodingType::Windows1252) { + std::optional errors = [&] { if (handle_strings) - errors = _parse_handler->template parse(); + return _parse_handler->template parse(); else - errors = _parse_handler->template parse(); - } else { - if (handle_strings) - errors = _parse_handler->template parse(); - else - errors = _parse_handler->template parse(); - } + return _parse_handler->template parse(); + }(); _has_error = _parse_handler->parse_state().logger().errored(); _has_warning = _parse_handler->parse_state().logger().warned(); if (!errors->empty()) { + _has_error = true; _has_fatal_error = true; if (&_error_stream.get() != &detail::cnull) { print_errors_to(_error_stream); } return false; } - _lines = std::move(_parse_handler->get_lines()); return true; } -template -const std::vector& Parser::get_lines() const { - return _lines; +const std::vector& Parser::get_lines() const { + return _parse_handler->get_lines(); } -template -typename Parser::error_range Parser::get_errors() const { +typename Parser::error_range Parser::get_errors() const { return _parse_handler->parse_state().logger().get_errors(); } -template -const FilePosition Parser::get_error_position(const error::Error* error) const { +const FilePosition Parser::get_error_position(const error::Error* error) const { if (!error || !error->is_linked_in_tree()) { return {}; } @@ -206,18 +194,27 @@ const FilePosition Parser::get_error_position(const error::Error* erro return {}; } - auto loc_begin = lexy::get_input_location(_parse_handler->buffer(), err_location.begin()); - FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() }; - if (err_location.begin() < err_location.end()) { - auto loc_end = lexy::get_input_location(_parse_handler->buffer(), err_location.end(), loc_begin.anchor()); - result.end_line = loc_end.line_nr(); - result.end_column = loc_end.column_nr(); - } - return result; +// TODO: Remove reinterpret_cast +// WARNING: This almost certainly breaks on utf16 and utf32 encodings, luckily we don't parse in that format +// This is purely to silence the node_location errors because char8_t is useless +#define REINTERPRET_IT(IT) reinterpret_cast::encoding::char_type*>((IT)) + + return _parse_handler->parse_state().file().visit_buffer( + [&](auto&& buffer) -> FilePosition { + auto loc_begin = lexy::get_input_location(buffer, REINTERPRET_IT(err_location.begin())); + FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() }; + if (err_location.begin() < err_location.end()) { + auto loc_end = lexy::get_input_location(buffer, REINTERPRET_IT(err_location.end()), loc_begin.anchor()); + result.end_line = loc_end.line_nr(); + result.end_column = loc_end.column_nr(); + } + return result; + }); + +#undef REINTERPRET_IT } -template -void Parser::print_errors_to(std::basic_ostream& stream) const { +void Parser::print_errors_to(std::basic_ostream& stream) const { auto errors = get_errors(); if (errors.empty()) return; for (const auto error : errors) { @@ -226,19 +223,9 @@ void Parser::print_errors_to(std::basic_ostream& stream) const { [&](const error::BufferError* buffer_error) { stream << "buffer error: " << buffer_error->message() << '\n'; }, - [&](const error::ParseError* parse_error) { - auto position = get_error_position(parse_error); - std::string pos_str = fmt::format(":{}:{}: ", position.start_line, position.start_column); - stream << _file_path << pos_str << "parse error for '" << parse_error->production_name() << "': " << parse_error->message() << '\n'; - }, - [&](dryad::child_visitor visitor, const error::Semantic* semantic) { - auto position = get_error_position(semantic); - std::string pos_str = ": "; - if (!position.is_empty()) { - pos_str = fmt::format(":{}:{}: ", position.start_line, position.start_column); - } - stream << _file_path << pos_str << semantic->message() << '\n'; - auto annotations = semantic->annotations(); + [&](dryad::child_visitor visitor, const error::AnnotatedError* annotated_error) { + stream << annotated_error->message() << '\n'; + auto annotations = annotated_error->annotations(); for (auto annotation : annotations) { visitor(annotation); } @@ -250,7 +237,4 @@ void Parser::print_errors_to(std::basic_ostream& stream) const { stream << secondary->message() << '\n'; }); } -} - -template class ovdl::csv::Parser; -template class ovdl::csv::Parser; \ No newline at end of file +} \ No newline at end of file -- cgit v1.2.3-56-ga3b1