aboutsummaryrefslogtreecommitdiff
path: root/src/openvic-dataloader/csv/CsvGrammar.hpp
diff options
context:
space:
mode:
author Spartan322 <Megacake1234@gmail.com>2023-11-28 11:09:26 +0100
committer Spartan322 <Megacake1234@gmail.com>2024-05-09 22:11:26 +0200
commit757114a3c5b748567b42f273c7b78ca039ae983c (patch)
treee07390b682052129c91f4b157068bcdd84ceecb4 /src/openvic-dataloader/csv/CsvGrammar.hpp
parent7211a228e68c8a6b1ad1c1c5ec68c8d720b6d2ba (diff)
Add `deps/dryad` -> https://github.com/Spartan322/dryadadd/dryad
Add `deps/fmt` -> https://github.com/fmtlib/fmt Add `deps/range-v3` -> https://github.com/ericniebler/range-v3 Improve parser error and warning support Update .clang-format Update `deps/SCsub`
Diffstat (limited to 'src/openvic-dataloader/csv/CsvGrammar.hpp')
-rw-r--r--src/openvic-dataloader/csv/CsvGrammar.hpp201
1 files changed, 99 insertions, 102 deletions
diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp
index 712bddc..5451f26 100644
--- a/src/openvic-dataloader/csv/CsvGrammar.hpp
+++ b/src/openvic-dataloader/csv/CsvGrammar.hpp
@@ -7,14 +7,17 @@
#include <vector>
#include <openvic-dataloader/csv/LineObject.hpp>
+#include <openvic-dataloader/csv/Parser.hpp>
#include <lexy/callback.hpp>
#include <lexy/dsl.hpp>
-#include "detail/LexyLitRange.hpp"
+#include "detail/dsl.hpp"
// Grammar Definitions //
namespace ovdl::csv::grammar {
+ using EncodingType = ovdl::csv::EncodingType;
+
template<typename T>
concept ParseChars = requires() {
{ T::character };
@@ -51,123 +54,117 @@ namespace ovdl::csv::grammar {
.map<'"'>('"');
template<ParseOptions Options>
- struct StringValue {
- static constexpr auto rule = [] {
- // Arbitrary code points
- auto c = Options.character - Options.control;
-
- auto back_escape = lexy::dsl::backslash_escape //
- .symbol<escaped_symbols>();
+ struct CsvGrammar {
+ struct StringValue {
+ static constexpr auto rule = [] {
+ // Arbitrary code points
+ auto c = Options.character - Options.control;
- auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
- .template symbol<escaped_quote>();
+ auto back_escape = lexy::dsl::backslash_escape //
+ .symbol<escaped_symbols>();
- return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape);
- }();
-
- static constexpr auto value = lexy::as_string<std::string>;
- };
+ auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
+ .template symbol<escaped_quote>();
- template<ParseOptions Options>
- struct PlainValue {
- static constexpr auto rule = [] {
- if constexpr (Options.SupportStrings) {
- return lexy::dsl::identifier(Options.character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline));
- } else {
- auto escape_check_char = Options.character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline);
- auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>;
- auto id_segment = lexy::dsl::identifier(id_check_char);
- auto escape_segement = lexy::dsl::token(escape_check_char);
- auto escape_sym = lexy::dsl::symbol<escaped_symbols>(escape_segement);
- auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym;
- return lexy::dsl::list(id_segment | escape_rule);
- }
- }();
- static constexpr auto value = lexy::as_string<std::string>;
- };
+ return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape);
+ }();
- template<ParseOptions Options>
- struct Value {
- static constexpr auto rule = [] {
- if constexpr (Options.SupportStrings) {
- return lexy::dsl::p<StringValue<Options>> | lexy::dsl::p<PlainValue<Options>>;
- } else {
- return lexy::dsl::p<PlainValue<Options>>;
- }
- }();
- static constexpr auto value = lexy::forward<std::string>;
- };
-
- template<ParseOptions Options>
- struct SepConst {
- static constexpr auto rule = lexy::dsl::lit_b<Options.SepChar>;
- static constexpr auto value = lexy::constant(1);
- };
+ static constexpr auto value = lexy::as_string<std::string>;
+ };
- template<ParseOptions Options>
- struct Seperator {
- static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Options>>);
- static constexpr auto value = lexy::count;
- };
+ struct PlainValue {
+ static constexpr auto rule = [] {
+ if constexpr (Options.SupportStrings) {
+ return lexy::dsl::identifier(Options.character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline));
+ } else {
+ auto escape_check_char = Options.character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline);
+ auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>;
+ auto id_segment = lexy::dsl::identifier(id_check_char);
+ auto escape_segement = lexy::dsl::token(escape_check_char);
+ auto escape_sym = lexy::dsl::symbol<escaped_symbols>(escape_segement);
+ auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym;
+ return lexy::dsl::list(id_segment | escape_rule);
+ }
+ }();
+ static constexpr auto value = lexy::as_string<std::string>;
+ };
- template<ParseOptions Options>
- struct LineEnd {
- static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Options>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Options>>));
- static constexpr auto value = lexy::fold_inplace<ovdl::csv::LineObject>(
- std::initializer_list<ovdl::csv::LineObject::value_type> {},
- [](ovdl::csv::LineObject& result, auto&& arg) {
- if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
- // Count seperators, adds to previous value, making it a position
- using position_type = ovdl::csv::LineObject::position_type;
- result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
+ struct Value {
+ static constexpr auto rule = [] {
+ if constexpr (Options.SupportStrings) {
+ return lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue>;
} else {
- if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
- else {
- auto& [pos, value] = result.back();
- value = arg;
- }
+ return lexy::dsl::p<PlainValue>;
}
- });
- };
+ }();
+ static constexpr auto value = lexy::forward<std::string>;
+ };
- template<ParseOptions Options>
- struct Line {
- static constexpr auto suffix_setter(ovdl::csv::LineObject& line) {
- auto& [position, value] = line.back();
- if (value.empty()) {
- line.set_suffix_end(position);
- line.pop_back();
- } else {
- line.set_suffix_end(position + 1);
- }
+ struct SepConst {
+ static constexpr auto rule = lexy::dsl::lit_b<Options.SepChar>;
+ static constexpr auto value = lexy::constant(1);
};
- static constexpr auto rule = lexy::dsl::p<LineEnd<Options>> | lexy::dsl::p<Seperator<Options>> >> lexy::dsl::opt(lexy::dsl::p<LineEnd<Options>>);
- static constexpr auto value =
- lexy::callback<ovdl::csv::LineObject>(
- [](ovdl::csv::LineObject&& line) {
- suffix_setter(line);
- return LEXY_MOV(line);
- },
- [](std::size_t prefix_count, ovdl::csv::LineObject&& line) {
- line.set_prefix_end(prefix_count);
- // position needs to be adjusted to prefix
- for (auto& [position, value] : line) {
- position += prefix_count;
+ struct Seperator {
+ static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst>);
+ static constexpr auto value = lexy::count;
+ };
+
+ struct LineEnd {
+ static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator>));
+ static constexpr auto value = lexy::fold_inplace<ovdl::csv::LineObject>(
+ std::initializer_list<ovdl::csv::LineObject::value_type> {},
+ [](ovdl::csv::LineObject& result, auto&& arg) {
+ if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
+ // Count seperators, adds to previous value, making it a position
+ using position_type = ovdl::csv::LineObject::position_type;
+ result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
+ } else {
+ if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
+ else {
+ auto& [pos, value] = result.back();
+ value = arg;
+ }
}
- suffix_setter(line);
- return LEXY_MOV(line);
- },
- [](std::size_t suffix_count, lexy::nullopt = {}) {
- return ovdl::csv::LineObject(0, {}, suffix_count + 1);
});
+ };
+
+ struct Line {
+ static constexpr auto suffix_setter(ovdl::csv::LineObject& line) {
+ auto& [position, value] = line.back();
+ if (value.empty()) {
+ line.set_suffix_end(position);
+ line.pop_back();
+ } else {
+ line.set_suffix_end(position + 1);
+ }
+ };
+
+ static constexpr auto rule = lexy::dsl::p<LineEnd> | lexy::dsl::p<Seperator> >> lexy::dsl::opt(lexy::dsl::p<LineEnd>);
+ static constexpr auto value =
+ lexy::callback<ovdl::csv::LineObject>(
+ [](ovdl::csv::LineObject&& line) {
+ suffix_setter(line);
+ return LEXY_MOV(line);
+ },
+ [](std::size_t prefix_count, ovdl::csv::LineObject&& line) {
+ line.set_prefix_end(prefix_count);
+ // position needs to be adjusted to prefix
+ for (auto& [position, value] : line) {
+ position += prefix_count;
+ }
+ suffix_setter(line);
+ return LEXY_MOV(line);
+ },
+ [](std::size_t suffix_count, lexy::nullopt = {}) {
+ return ovdl::csv::LineObject(0, {}, suffix_count + 1);
+ });
+ };
};
template<ParseOptions Options>
struct File {
- static constexpr auto rule = lexy::dsl::terminator(lexy::dsl::eof).opt_list(
- lexy::dsl::p<Line<Options>> | lexy::dsl::newline
- );
+ static constexpr auto rule = lexy::dsl::terminator(lexy::dsl::eof).opt_list(lexy::dsl::p<typename CsvGrammar<Options>::Line> | lexy::dsl::newline);
static constexpr auto value = lexy::as_list<std::vector<ovdl::csv::LineObject>>;
};
@@ -199,7 +196,7 @@ namespace ovdl::csv::grammar {
namespace ovdl::csv::grammar::windows1252 {
struct windows1252_t {
- static constexpr auto character = detail::lexydsl::make_range<0x01, 0xFF>();
+ static constexpr auto character = dsl::make_range<0x01, 0xFF>();
static constexpr auto control =
lexy::dsl::ascii::control /
lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> /