diff options
Diffstat (limited to 'src/openvic-dataloader/csv/CsvGrammar.hpp')
| -rw-r--r-- | src/openvic-dataloader/csv/CsvGrammar.hpp | 201 |
1 files changed, 99 insertions, 102 deletions
diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp index 712bddc..5451f26 100644 --- a/src/openvic-dataloader/csv/CsvGrammar.hpp +++ b/src/openvic-dataloader/csv/CsvGrammar.hpp @@ -7,14 +7,17 @@ #include <vector> #include <openvic-dataloader/csv/LineObject.hpp> +#include <openvic-dataloader/csv/Parser.hpp> #include <lexy/callback.hpp> #include <lexy/dsl.hpp> -#include "detail/LexyLitRange.hpp" +#include "detail/dsl.hpp" // Grammar Definitions // namespace ovdl::csv::grammar { + using EncodingType = ovdl::csv::EncodingType; + template<typename T> concept ParseChars = requires() { { T::character }; @@ -51,123 +54,117 @@ namespace ovdl::csv::grammar { .map<'"'>('"'); template<ParseOptions Options> - struct StringValue { - static constexpr auto rule = [] { - // Arbitrary code points - auto c = Options.character - Options.control; - - auto back_escape = lexy::dsl::backslash_escape // - .symbol<escaped_symbols>(); + struct CsvGrammar { + struct StringValue { + static constexpr auto rule = [] { + // Arbitrary code points + auto c = Options.character - Options.control; - auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // - .template symbol<escaped_quote>(); + auto back_escape = lexy::dsl::backslash_escape // + .symbol<escaped_symbols>(); - return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape); - }(); - - static constexpr auto value = lexy::as_string<std::string>; - }; + auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + .template symbol<escaped_quote>(); - template<ParseOptions Options> - struct PlainValue { - static constexpr auto rule = [] { - if constexpr (Options.SupportStrings) { - return lexy::dsl::identifier(Options.character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline)); - } else { - auto escape_check_char = Options.character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline); - auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>; - auto id_segment = lexy::dsl::identifier(id_check_char); - auto escape_segement = lexy::dsl::token(escape_check_char); - auto escape_sym = lexy::dsl::symbol<escaped_symbols>(escape_segement); - auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym; - return lexy::dsl::list(id_segment | escape_rule); - } - }(); - static constexpr auto value = lexy::as_string<std::string>; - }; + return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape); + }(); - template<ParseOptions Options> - struct Value { - static constexpr auto rule = [] { - if constexpr (Options.SupportStrings) { - return lexy::dsl::p<StringValue<Options>> | lexy::dsl::p<PlainValue<Options>>; - } else { - return lexy::dsl::p<PlainValue<Options>>; - } - }(); - static constexpr auto value = lexy::forward<std::string>; - }; - - template<ParseOptions Options> - struct SepConst { - static constexpr auto rule = lexy::dsl::lit_b<Options.SepChar>; - static constexpr auto value = lexy::constant(1); - }; + static constexpr auto value = lexy::as_string<std::string>; + }; - template<ParseOptions Options> - struct Seperator { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Options>>); - static constexpr auto value = lexy::count; - }; + struct PlainValue { + static constexpr auto rule = [] { + if constexpr (Options.SupportStrings) { + return lexy::dsl::identifier(Options.character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline)); + } else { + auto escape_check_char = Options.character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline); + auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>; + auto id_segment = lexy::dsl::identifier(id_check_char); + auto escape_segement = lexy::dsl::token(escape_check_char); + auto escape_sym = lexy::dsl::symbol<escaped_symbols>(escape_segement); + auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym; + return lexy::dsl::list(id_segment | escape_rule); + } + }(); + static constexpr auto value = lexy::as_string<std::string>; + }; - template<ParseOptions Options> - struct LineEnd { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Options>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Options>>)); - static constexpr auto value = lexy::fold_inplace<ovdl::csv::LineObject>( - std::initializer_list<ovdl::csv::LineObject::value_type> {}, - [](ovdl::csv::LineObject& result, auto&& arg) { - if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) { - // Count seperators, adds to previous value, making it a position - using position_type = ovdl::csv::LineObject::position_type; - result.emplace_back(static_cast<position_type>(arg + result.back().first), ""); + struct Value { + static constexpr auto rule = [] { + if constexpr (Options.SupportStrings) { + return lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue>; } else { - if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); - else { - auto& [pos, value] = result.back(); - value = arg; - } + return lexy::dsl::p<PlainValue>; } - }); - }; + }(); + static constexpr auto value = lexy::forward<std::string>; + }; - template<ParseOptions Options> - struct Line { - static constexpr auto suffix_setter(ovdl::csv::LineObject& line) { - auto& [position, value] = line.back(); - if (value.empty()) { - line.set_suffix_end(position); - line.pop_back(); - } else { - line.set_suffix_end(position + 1); - } + struct SepConst { + static constexpr auto rule = lexy::dsl::lit_b<Options.SepChar>; + static constexpr auto value = lexy::constant(1); }; - static constexpr auto rule = lexy::dsl::p<LineEnd<Options>> | lexy::dsl::p<Seperator<Options>> >> lexy::dsl::opt(lexy::dsl::p<LineEnd<Options>>); - static constexpr auto value = - lexy::callback<ovdl::csv::LineObject>( - [](ovdl::csv::LineObject&& line) { - suffix_setter(line); - return LEXY_MOV(line); - }, - [](std::size_t prefix_count, ovdl::csv::LineObject&& line) { - line.set_prefix_end(prefix_count); - // position needs to be adjusted to prefix - for (auto& [position, value] : line) { - position += prefix_count; + struct Seperator { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst>); + static constexpr auto value = lexy::count; + }; + + struct LineEnd { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator>)); + static constexpr auto value = lexy::fold_inplace<ovdl::csv::LineObject>( + std::initializer_list<ovdl::csv::LineObject::value_type> {}, + [](ovdl::csv::LineObject& result, auto&& arg) { + if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) { + // Count seperators, adds to previous value, making it a position + using position_type = ovdl::csv::LineObject::position_type; + result.emplace_back(static_cast<position_type>(arg + result.back().first), ""); + } else { + if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); + else { + auto& [pos, value] = result.back(); + value = arg; + } } - suffix_setter(line); - return LEXY_MOV(line); - }, - [](std::size_t suffix_count, lexy::nullopt = {}) { - return ovdl::csv::LineObject(0, {}, suffix_count + 1); }); + }; + + struct Line { + static constexpr auto suffix_setter(ovdl::csv::LineObject& line) { + auto& [position, value] = line.back(); + if (value.empty()) { + line.set_suffix_end(position); + line.pop_back(); + } else { + line.set_suffix_end(position + 1); + } + }; + + static constexpr auto rule = lexy::dsl::p<LineEnd> | lexy::dsl::p<Seperator> >> lexy::dsl::opt(lexy::dsl::p<LineEnd>); + static constexpr auto value = + lexy::callback<ovdl::csv::LineObject>( + [](ovdl::csv::LineObject&& line) { + suffix_setter(line); + return LEXY_MOV(line); + }, + [](std::size_t prefix_count, ovdl::csv::LineObject&& line) { + line.set_prefix_end(prefix_count); + // position needs to be adjusted to prefix + for (auto& [position, value] : line) { + position += prefix_count; + } + suffix_setter(line); + return LEXY_MOV(line); + }, + [](std::size_t suffix_count, lexy::nullopt = {}) { + return ovdl::csv::LineObject(0, {}, suffix_count + 1); + }); + }; }; template<ParseOptions Options> struct File { - static constexpr auto rule = lexy::dsl::terminator(lexy::dsl::eof).opt_list( - lexy::dsl::p<Line<Options>> | lexy::dsl::newline - ); + static constexpr auto rule = lexy::dsl::terminator(lexy::dsl::eof).opt_list(lexy::dsl::p<typename CsvGrammar<Options>::Line> | lexy::dsl::newline); static constexpr auto value = lexy::as_list<std::vector<ovdl::csv::LineObject>>; }; @@ -199,7 +196,7 @@ namespace ovdl::csv::grammar { namespace ovdl::csv::grammar::windows1252 { struct windows1252_t { - static constexpr auto character = detail::lexydsl::make_range<0x01, 0xFF>(); + static constexpr auto character = dsl::make_range<0x01, 0xFF>(); static constexpr auto control = lexy::dsl::ascii::control / lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> / |