From fed40d5dd5203bae678be7aea6fd91c73b43e298 Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Mon, 11 Sep 2023 19:12:18 -0400 Subject: Overhaul CSV parsing and align with Victoria 2 CSVs Fix "" not being converted into " in Csv files Move duplicated Grammar in CsvGrammar.hpp to Grammar.inc Add ParseOptions to simplify template options Add StringValue disable option: Inlines value escapes Add strings namespace to grammars for `ParseOptions.SupportStrings = true` Add string parse handling to parse_csv via argument --- src/openvic-dataloader/csv/CsvGrammar.hpp | 240 +----------------------------- src/openvic-dataloader/csv/Grammar.inc | 171 +++++++++++++++++++++ src/openvic-dataloader/csv/Parser.cpp | 13 +- 3 files changed, 183 insertions(+), 241 deletions(-) create mode 100644 src/openvic-dataloader/csv/Grammar.inc (limited to 'src') diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp index 4765112..8f8da82 100644 --- a/src/openvic-dataloader/csv/CsvGrammar.hpp +++ b/src/openvic-dataloader/csv/CsvGrammar.hpp @@ -21,248 +21,12 @@ namespace ovdl::csv::grammar::windows1252 { lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> / lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>; - struct StringValue { - static constexpr auto escaped_symbols = lexy::symbol_table // - .map<'"'>('"') - .map<'\''>('\'') - .map<'\\'>('\\') - .map<'/'>('/') - .map<'b'>('\b') - .map<'f'>('\f') - .map<'n'>('\n') - .map<'r'>('\r') - .map<'t'>('\t'); - /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" - // static constexpr auto escaped_quote = lexy::symbol_table // - // .map<'"'>('"'); - static constexpr auto rule = [] { - // Arbitrary code points - auto c = character - (control / lexy::dsl::lit_b<'"'>); - - auto back_escape = lexy::dsl::backslash_escape // - .symbol(); - - // auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // - // .symbol(); - - return lexy::dsl::quoted(c, back_escape); - }(); - - static constexpr auto value = lexy::as_string; - }; - - template - struct PlainValue { - static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline)); - static constexpr auto value = lexy::as_string; - }; - - template - struct Value { - static constexpr auto rule = lexy::dsl::p | lexy::dsl::p>; - static constexpr auto value = lexy::forward; - }; - - template - struct SepConst { - static constexpr auto rule = lexy::dsl::lit_b; - static constexpr auto value = lexy::constant(1); - }; - - template - struct Seperator { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>); - static constexpr auto value = lexy::count; - }; - - template - struct LineEnd { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::p>)); - static constexpr auto value = lexy::fold_inplace( - std::initializer_list {}, - [](csv::LineObject& result, auto&& arg) { - if constexpr (std::is_same_v, std::size_t>) { - // Count seperators, adds to previous value, making it a position - using position_type = csv::LineObject::position_type; - result.emplace_back(static_cast(arg + result.back().first), ""); - } else { - if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); - else { - auto& [pos, value] = result.back(); - value = arg; - } - } - }); - }; - - template - struct Line { - - static constexpr auto suffix_setter(csv::LineObject& line) { - auto& [position, value] = line.back(); - if (value.empty()) { - line.set_suffix_end(position); - line.pop_back(); - } else { - line.set_suffix_end(position + 1); - } - }; - - static constexpr auto rule = lexy::dsl::p> | lexy::dsl::p> >> lexy::dsl::p>; - static constexpr auto value = - lexy::callback( - [](csv::LineObject&& line) { - suffix_setter(line); - return LEXY_MOV(line); - }, - [](std::size_t prefix_count, csv::LineObject&& line) { - line.set_prefix_end(prefix_count); - // position needs to be adjusted to prefix - for (auto& [position, value] : line) { - position += prefix_count; - } - suffix_setter(line); - return LEXY_MOV(line); - }); - }; - - template - struct File { - static constexpr auto rule = - lexy::dsl::whitespace(lexy::dsl::newline) + - lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::eol))); - - static constexpr auto value = lexy::as_list>; - }; - - using CommaFile = File<','>; - using ColonFile = File<':'>; - using SemiColonFile = File<';'>; - using TabFile = File<'\t'>; - using BarFile = File<'|'>; +#include "Grammar.inc" } namespace ovdl::csv::grammar::utf8 { constexpr auto character = lexy::dsl::unicode::character; constexpr auto control = lexy::dsl::unicode::control; - struct StringValue { - static constexpr auto escaped_symbols = lexy::symbol_table // - .map<'"'>('"') - .map<'\''>('\'') - .map<'\\'>('\\') - .map<'/'>('/') - .map<'b'>('\b') - .map<'f'>('\f') - .map<'n'>('\n') - .map<'r'>('\r') - .map<'t'>('\t'); - /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" - // static constexpr auto escaped_quote = lexy::symbol_table // - // .map<'"'>('"'); - static constexpr auto rule = [] { - // Arbitrary code points - auto c = character - (control / lexy::dsl::lit_b<'"'>); - - auto back_escape = lexy::dsl::backslash_escape // - .symbol(); - - // auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // - // .symbol(); - - return lexy::dsl::quoted(c, back_escape); - }(); - - static constexpr auto value = lexy::as_string; - }; - - template - struct PlainValue { - static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline)); - static constexpr auto value = lexy::as_string; - }; - - template - struct Value { - static constexpr auto rule = lexy::dsl::p | lexy::dsl::p>; - static constexpr auto value = lexy::forward; - }; - - template - struct SepConst { - static constexpr auto rule = lexy::dsl::lit_b; - static constexpr auto value = lexy::constant(1); - }; - - template - struct Seperator { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>); - static constexpr auto value = lexy::count; - }; - - template - struct LineEnd { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::p>)); - static constexpr auto value = lexy::fold_inplace( - std::initializer_list {}, - [](csv::LineObject& result, auto&& arg) { - if constexpr (std::is_same_v, std::size_t>) { - // Count seperators, adds to previous value, making it a position - using position_type = csv::LineObject::position_type; - result.emplace_back(static_cast(arg + result.back().first), ""); - } else { - if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); - else { - auto& [pos, value] = result.back(); - value = arg; - } - } - }); - }; - - template - struct Line { - - static constexpr auto suffix_setter(csv::LineObject& line) { - auto& [position, value] = line.back(); - if (value.empty()) { - line.set_suffix_end(position); - line.pop_back(); - } else { - line.set_suffix_end(position + 1); - } - }; - - static constexpr auto rule = lexy::dsl::p> | lexy::dsl::p> >> lexy::dsl::p>; - static constexpr auto value = - lexy::callback( - [](csv::LineObject&& line) { - suffix_setter(line); - return LEXY_MOV(line); - }, - [](std::size_t prefix_count, csv::LineObject&& line) { - line.set_prefix_end(prefix_count); - // position needs to be adjusted to prefix - for (auto& [position, value] : line) { - position += prefix_count; - } - suffix_setter(line); - return LEXY_MOV(line); - }); - }; - - template - struct File { - static constexpr auto rule = - lexy::dsl::whitespace(lexy::dsl::newline) + - lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::eol))); - - static constexpr auto value = lexy::as_list>; - }; - - using CommaFile = File<','>; - using ColonFile = File<':'>; - using SemiColonFile = File<';'>; - using TabFile = File<'\t'>; - using BarFile = File<'|'>; +#include "Grammar.inc" } \ No newline at end of file diff --git a/src/openvic-dataloader/csv/Grammar.inc b/src/openvic-dataloader/csv/Grammar.inc new file mode 100644 index 0000000..ed7f455 --- /dev/null +++ b/src/openvic-dataloader/csv/Grammar.inc @@ -0,0 +1,171 @@ +// This is designed to be reused in CsvGrammar.hpp multiple times +// Please ignore undeclared variable use for character and control, this is intended +// Sorry was the cleanest way I could think of to prevent code duplication + +// Includes to keep file errors small +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "detail/LexyLitRange.hpp" + +struct ParseOptions { + /// @brief Seperator character + char SepChar; + /// @brief Determines whether StringValue is supported + bool SupportStrings; +}; + +struct StringValue { + static constexpr auto escaped_symbols = lexy::symbol_table // + .map<'"'>('"') + .map<'\''>('\'') + .map<'\\'>('\\') + .map<'/'>('/') + .map<'b'>('\b') + .map<'f'>('\f') + .map<'n'>('\n') + .map<'r'>('\r') + .map<'t'>('\t'); + /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" + static constexpr auto escaped_quote = lexy::symbol_table // + .map<'"'>('"'); + + static constexpr auto rule = [] { + // Arbitrary code points + auto c = character - control; + + auto back_escape = lexy::dsl::backslash_escape // + .symbol(); + + auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + .symbol(); + + return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape); + }(); + + static constexpr auto value = lexy::as_string; +}; + +template +struct PlainValue { + static constexpr auto rule = [] { + if constexpr (Options.SupportStrings) { + return lexy::dsl::identifier(character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline)); + } else { + auto escape_check_char = character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline); + auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>; + auto id_segment = lexy::dsl::identifier(id_check_char); + auto escape_segement = lexy::dsl::token(escape_check_char); + auto escape_sym = lexy::dsl::symbol(escape_segement); + auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym; + return lexy::dsl::list(id_segment | escape_rule); + } + }(); + static constexpr auto value = lexy::as_string; +}; + +template +struct Value { + static constexpr auto rule = [] { + if constexpr (Options.SupportStrings) { + return lexy::dsl::p | lexy::dsl::p>; + } else { + return lexy::dsl::p>; + } + }(); + static constexpr auto value = lexy::forward; +}; + +template +struct SepConst { + static constexpr auto rule = lexy::dsl::lit_b; + static constexpr auto value = lexy::constant(1); +}; + +template +struct Seperator { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>); + static constexpr auto value = lexy::count; +}; + +template +struct LineEnd { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::p>)); + static constexpr auto value = lexy::fold_inplace( + std::initializer_list {}, + [](ovdl::csv::LineObject& result, auto&& arg) { + if constexpr (std::is_same_v, std::size_t>) { + // Count seperators, adds to previous value, making it a position + using position_type = ovdl::csv::LineObject::position_type; + result.emplace_back(static_cast(arg + result.back().first), ""); + } else { + if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); + else { + auto& [pos, value] = result.back(); + value = arg; + } + } + }); +}; + +template +struct Line { + + static constexpr auto suffix_setter(ovdl::csv::LineObject& line) { + auto& [position, value] = line.back(); + if (value.empty()) { + line.set_suffix_end(position); + line.pop_back(); + } else { + line.set_suffix_end(position + 1); + } + }; + + static constexpr auto rule = lexy::dsl::p> | lexy::dsl::p> >> lexy::dsl::p>; + static constexpr auto value = + lexy::callback( + [](ovdl::csv::LineObject&& line) { + suffix_setter(line); + return LEXY_MOV(line); + }, + [](std::size_t prefix_count, ovdl::csv::LineObject&& line) { + line.set_prefix_end(prefix_count); + // position needs to be adjusted to prefix + for (auto& [position, value] : line) { + position += prefix_count; + } + suffix_setter(line); + return LEXY_MOV(line); + }); +}; + +template +struct File { + static constexpr auto rule = + lexy::dsl::whitespace(lexy::dsl::newline) + + lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::eol))); + + static constexpr auto value = lexy::as_list>; +}; + +using CommaFile = File; +using ColonFile = File; +using SemiColonFile = File; +using TabFile = File; +using BarFile = File; + +namespace strings { + using CommaFile = File; + using ColonFile = File; + using SemiColonFile = File; + using TabFile = File; + using BarFile = File; +} \ No newline at end of file diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp index 86ad02e..14ef553 100644 --- a/src/openvic-dataloader/csv/Parser.cpp +++ b/src/openvic-dataloader/csv/Parser.cpp @@ -165,16 +165,23 @@ constexpr Parser& Parser::load_from_file(const detail::Has_c } template -bool Parser::parse_csv() { +bool Parser::parse_csv(bool handle_strings) { if (!_buffer_handler->is_valid()) { return false; } std::optional> errors; + auto report_error = ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }); if constexpr (Encoding == EncodingType::Windows1252) { - errors = _buffer_handler->template parse(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + if (handle_strings) + errors = _buffer_handler->template parse(report_error); + else + errors = _buffer_handler->template parse(report_error); } else { - errors = _buffer_handler->template parse(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + if (handle_strings) + errors = _buffer_handler->template parse(report_error); + else + errors = _buffer_handler->template parse(report_error); } if (errors) { _errors.reserve(errors->size()); -- cgit v1.2.3-56-ga3b1