From 73caff98c16e7a52c13fc26621ad75345142fd4d Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Fri, 13 Oct 2023 06:18:55 -0400 Subject: Simplify csv grammar to one file Remove hacky include behavior for CSVGrammar --- src/openvic-dataloader/csv/CsvGrammar.hpp | 229 ++++++++++++++++++++++++++++-- src/openvic-dataloader/csv/Grammar.inc | 171 ---------------------- 2 files changed, 220 insertions(+), 180 deletions(-) delete mode 100644 src/openvic-dataloader/csv/Grammar.inc diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp index 8f8da82..4c1c1ab 100644 --- a/src/openvic-dataloader/csv/CsvGrammar.hpp +++ b/src/openvic-dataloader/csv/CsvGrammar.hpp @@ -14,19 +14,230 @@ #include "detail/LexyLitRange.hpp" // Grammar Definitions // +namespace ovdl::csv::grammar { + template + concept ParseChars = requires() { + { T::character }; + { T::control }; + }; + + template + struct ParseOptions { + /// @brief Seperator character + char SepChar; + /// @brief Determines whether StringValue is supported + bool SupportStrings; + /// @brief Paradox-style localization escape characters + /// @note Is ignored if SupportStrings is true + char EscapeChar; + + static constexpr auto parse_chars = T {}; + static constexpr auto character = parse_chars.character; + static constexpr auto control = parse_chars.control; + }; + + constexpr auto escaped_symbols = lexy::symbol_table // + .map<'"'>('"') + .map<'\''>('\'') + .map<'\\'>('\\') + .map<'/'>('/') + .map<'b'>('\b') + .map<'f'>('\f') + .map<'n'>('\n') + .map<'r'>('\r') + .map<'t'>('\t'); + + constexpr auto escaped_quote = lexy::symbol_table // + .map<'"'>('"'); + + template + struct StringValue { + static constexpr auto rule = [] { + // Arbitrary code points + auto c = Options.character - Options.control; + + auto back_escape = lexy::dsl::backslash_escape // + .symbol(); + + auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + .template symbol(); + + return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape); + }(); + + static constexpr auto value = lexy::as_string; + }; + + template + struct PlainValue { + static constexpr auto rule = [] { + if constexpr (Options.SupportStrings) { + return lexy::dsl::identifier(Options.character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline)); + } else { + auto escape_check_char = Options.character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline); + auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>; + auto id_segment = lexy::dsl::identifier(id_check_char); + auto escape_segement = lexy::dsl::token(escape_check_char); + auto escape_sym = lexy::dsl::symbol(escape_segement); + auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym; + return lexy::dsl::list(id_segment | escape_rule); + } + }(); + static constexpr auto value = lexy::as_string; + }; + + template + struct Value { + static constexpr auto rule = [] { + if constexpr (Options.SupportStrings) { + return lexy::dsl::p> | lexy::dsl::p>; + } else { + return lexy::dsl::p>; + } + }(); + static constexpr auto value = lexy::forward; + }; + + template + struct SepConst { + static constexpr auto rule = lexy::dsl::lit_b; + static constexpr auto value = lexy::constant(1); + }; + + template + struct Seperator { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>); + static constexpr auto value = lexy::count; + }; + + template + struct LineEnd { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::p>)); + static constexpr auto value = lexy::fold_inplace( + std::initializer_list {}, + [](ovdl::csv::LineObject& result, auto&& arg) { + if constexpr (std::is_same_v, std::size_t>) { + // Count seperators, adds to previous value, making it a position + using position_type = ovdl::csv::LineObject::position_type; + result.emplace_back(static_cast(arg + result.back().first), ""); + } else { + if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); + else { + auto& [pos, value] = result.back(); + value = arg; + } + } + }); + }; + + template + struct Line { + + static constexpr auto suffix_setter(ovdl::csv::LineObject& line) { + auto& [position, value] = line.back(); + if (value.empty()) { + line.set_suffix_end(position); + line.pop_back(); + } else { + line.set_suffix_end(position + 1); + } + }; + + static constexpr auto rule = lexy::dsl::p> | lexy::dsl::p> >> lexy::dsl::p>; + static constexpr auto value = + lexy::callback( + [](ovdl::csv::LineObject&& line) { + suffix_setter(line); + return LEXY_MOV(line); + }, + [](std::size_t prefix_count, ovdl::csv::LineObject&& line) { + line.set_prefix_end(prefix_count); + // position needs to be adjusted to prefix + for (auto& [position, value] : line) { + position += prefix_count; + } + suffix_setter(line); + return LEXY_MOV(line); + }); + }; + + template + struct File { + static constexpr auto rule = + lexy::dsl::whitespace(lexy::dsl::newline) + + lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::eol))); + + static constexpr auto value = lexy::as_list>; + }; + + template + using CommaFile = File { ',', false, '$' }>; + template + using ColonFile = File { ':', false, '$' }>; + template + using SemiColonFile = File { ';', false, '$' }>; + template + using TabFile = File { '\t', false, '$' }>; + template + using BarFile = File { '|', false, '$' }>; + + namespace strings { + template + using CommaFile = File { ',', true, '$' }>; + template + using ColonFile = File { ':', true, '$' }>; + template + using SemiColonFile = File { ';', true, '$' }>; + template + using TabFile = File { '\t', true, '$' }>; + template + using BarFile = File { '|', true, '$' }>; + } +} + namespace ovdl::csv::grammar::windows1252 { - constexpr auto character = detail::lexydsl::make_range<0x01, 0xFF>(); - constexpr auto control = - lexy::dsl::ascii::control / - lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> / - lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>; + struct windows1252_t { + static constexpr auto character = detail::lexydsl::make_range<0x01, 0xFF>(); + static constexpr auto control = + lexy::dsl::ascii::control / + lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> / + lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>; + }; + + using CommaFile = CommaFile; + using ColonFile = ColonFile; + using SemiColonFile = SemiColonFile; + using TabFile = TabFile; + using BarFile = BarFile; -#include "Grammar.inc" + namespace strings { + using CommaFile = grammar::strings::CommaFile; + using ColonFile = grammar::strings::ColonFile; + using SemiColonFile = grammar::strings::SemiColonFile; + using TabFile = grammar::strings::TabFile; + using BarFile = grammar::strings::BarFile; + + } } namespace ovdl::csv::grammar::utf8 { - constexpr auto character = lexy::dsl::unicode::character; - constexpr auto control = lexy::dsl::unicode::control; + struct unicode_t { + static constexpr auto character = lexy::dsl::unicode::character; + static constexpr auto control = lexy::dsl::unicode::control; + }; + + using CommaFile = CommaFile; + using ColonFile = ColonFile; + using SemiColonFile = SemiColonFile; + using TabFile = TabFile; + using BarFile = BarFile; + + namespace strings { + using CommaFile = grammar::strings::CommaFile; + using ColonFile = grammar::strings::ColonFile; + using SemiColonFile = grammar::strings::SemiColonFile; + using TabFile = grammar::strings::TabFile; + using BarFile = grammar::strings::BarFile; -#include "Grammar.inc" + } } \ No newline at end of file diff --git a/src/openvic-dataloader/csv/Grammar.inc b/src/openvic-dataloader/csv/Grammar.inc deleted file mode 100644 index ed7f455..0000000 --- a/src/openvic-dataloader/csv/Grammar.inc +++ /dev/null @@ -1,171 +0,0 @@ -// This is designed to be reused in CsvGrammar.hpp multiple times -// Please ignore undeclared variable use for character and control, this is intended -// Sorry was the cleanest way I could think of to prevent code duplication - -// Includes to keep file errors small -#include -#include -#include -#include -#include - -#include - -#include -#include - -#include "detail/LexyLitRange.hpp" - -struct ParseOptions { - /// @brief Seperator character - char SepChar; - /// @brief Determines whether StringValue is supported - bool SupportStrings; -}; - -struct StringValue { - static constexpr auto escaped_symbols = lexy::symbol_table // - .map<'"'>('"') - .map<'\''>('\'') - .map<'\\'>('\\') - .map<'/'>('/') - .map<'b'>('\b') - .map<'f'>('\f') - .map<'n'>('\n') - .map<'r'>('\r') - .map<'t'>('\t'); - /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" - static constexpr auto escaped_quote = lexy::symbol_table // - .map<'"'>('"'); - - static constexpr auto rule = [] { - // Arbitrary code points - auto c = character - control; - - auto back_escape = lexy::dsl::backslash_escape // - .symbol(); - - auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // - .symbol(); - - return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape); - }(); - - static constexpr auto value = lexy::as_string; -}; - -template -struct PlainValue { - static constexpr auto rule = [] { - if constexpr (Options.SupportStrings) { - return lexy::dsl::identifier(character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline)); - } else { - auto escape_check_char = character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline); - auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>; - auto id_segment = lexy::dsl::identifier(id_check_char); - auto escape_segement = lexy::dsl::token(escape_check_char); - auto escape_sym = lexy::dsl::symbol(escape_segement); - auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym; - return lexy::dsl::list(id_segment | escape_rule); - } - }(); - static constexpr auto value = lexy::as_string; -}; - -template -struct Value { - static constexpr auto rule = [] { - if constexpr (Options.SupportStrings) { - return lexy::dsl::p | lexy::dsl::p>; - } else { - return lexy::dsl::p>; - } - }(); - static constexpr auto value = lexy::forward; -}; - -template -struct SepConst { - static constexpr auto rule = lexy::dsl::lit_b; - static constexpr auto value = lexy::constant(1); -}; - -template -struct Seperator { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>); - static constexpr auto value = lexy::count; -}; - -template -struct LineEnd { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::p>)); - static constexpr auto value = lexy::fold_inplace( - std::initializer_list {}, - [](ovdl::csv::LineObject& result, auto&& arg) { - if constexpr (std::is_same_v, std::size_t>) { - // Count seperators, adds to previous value, making it a position - using position_type = ovdl::csv::LineObject::position_type; - result.emplace_back(static_cast(arg + result.back().first), ""); - } else { - if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); - else { - auto& [pos, value] = result.back(); - value = arg; - } - } - }); -}; - -template -struct Line { - - static constexpr auto suffix_setter(ovdl::csv::LineObject& line) { - auto& [position, value] = line.back(); - if (value.empty()) { - line.set_suffix_end(position); - line.pop_back(); - } else { - line.set_suffix_end(position + 1); - } - }; - - static constexpr auto rule = lexy::dsl::p> | lexy::dsl::p> >> lexy::dsl::p>; - static constexpr auto value = - lexy::callback( - [](ovdl::csv::LineObject&& line) { - suffix_setter(line); - return LEXY_MOV(line); - }, - [](std::size_t prefix_count, ovdl::csv::LineObject&& line) { - line.set_prefix_end(prefix_count); - // position needs to be adjusted to prefix - for (auto& [position, value] : line) { - position += prefix_count; - } - suffix_setter(line); - return LEXY_MOV(line); - }); -}; - -template -struct File { - static constexpr auto rule = - lexy::dsl::whitespace(lexy::dsl::newline) + - lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::eol))); - - static constexpr auto value = lexy::as_list>; -}; - -using CommaFile = File; -using ColonFile = File; -using SemiColonFile = File; -using TabFile = File; -using BarFile = File; - -namespace strings { - using CommaFile = File; - using ColonFile = File; - using SemiColonFile = File; - using TabFile = File; - using BarFile = File; -} \ No newline at end of file -- cgit v1.2.3-56-ga3b1