diff options
Diffstat (limited to 'src/openvic-dataloader/csv')
| -rw-r--r-- | src/openvic-dataloader/csv/CsvGrammar.hpp | 187 | ||||
| -rw-r--r-- | src/openvic-dataloader/csv/Parser.cpp | 86 |
2 files changed, 227 insertions, 46 deletions
diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp index edce97b..4765112 100644 --- a/src/openvic-dataloader/csv/CsvGrammar.hpp +++ b/src/openvic-dataloader/csv/CsvGrammar.hpp @@ -11,8 +11,141 @@ #include <lexy/callback.hpp> #include <lexy/dsl.hpp> +#include "detail/LexyLitRange.hpp" + // Grammar Definitions // -namespace ovdl::csv::grammar { +namespace ovdl::csv::grammar::windows1252 { + constexpr auto character = detail::lexydsl::make_range<0x01, 0xFF>(); + constexpr auto control = + lexy::dsl::ascii::control / + lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> / + lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>; + + struct StringValue { + static constexpr auto escaped_symbols = lexy::symbol_table<char> // + .map<'"'>('"') + .map<'\''>('\'') + .map<'\\'>('\\') + .map<'/'>('/') + .map<'b'>('\b') + .map<'f'>('\f') + .map<'n'>('\n') + .map<'r'>('\r') + .map<'t'>('\t'); + /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" + // static constexpr auto escaped_quote = lexy::symbol_table<char> // + // .map<'"'>('"'); + static constexpr auto rule = [] { + // Arbitrary code points + auto c = character - (control / lexy::dsl::lit_b<'"'>); + + auto back_escape = lexy::dsl::backslash_escape // + .symbol<escaped_symbols>(); + + // auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + // .symbol<escaped_quote>(); + + return lexy::dsl::quoted(c, back_escape); + }(); + + static constexpr auto value = lexy::as_string<std::string>; + }; + + template<char Sep> + struct PlainValue { + static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b<Sep> / lexy::dsl::ascii::newline)); + static constexpr auto value = lexy::as_string<std::string>; + }; + + template<char Sep> + struct Value { + static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>; + static constexpr auto value = lexy::forward<std::string>; + }; + + template<char Sep> + struct SepConst { + static constexpr auto rule = lexy::dsl::lit_b<Sep>; + static constexpr auto value = lexy::constant(1); + }; + + template<char Sep> + struct Seperator { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Sep>>); + static constexpr auto value = lexy::count; + }; + + template<char Sep> + struct LineEnd { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Sep>>)); + static constexpr auto value = lexy::fold_inplace<csv::LineObject>( + std::initializer_list<csv::LineObject::value_type> {}, + [](csv::LineObject& result, auto&& arg) { + if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) { + // Count seperators, adds to previous value, making it a position + using position_type = csv::LineObject::position_type; + result.emplace_back(static_cast<position_type>(arg + result.back().first), ""); + } else { + if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); + else { + auto& [pos, value] = result.back(); + value = arg; + } + } + }); + }; + + template<char Sep> + struct Line { + + static constexpr auto suffix_setter(csv::LineObject& line) { + auto& [position, value] = line.back(); + if (value.empty()) { + line.set_suffix_end(position); + line.pop_back(); + } else { + line.set_suffix_end(position + 1); + } + }; + + static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<Seperator<Sep>> >> lexy::dsl::p<LineEnd<Sep>>; + static constexpr auto value = + lexy::callback<csv::LineObject>( + [](csv::LineObject&& line) { + suffix_setter(line); + return LEXY_MOV(line); + }, + [](std::size_t prefix_count, csv::LineObject&& line) { + line.set_prefix_end(prefix_count); + // position needs to be adjusted to prefix + for (auto& [position, value] : line) { + position += prefix_count; + } + suffix_setter(line); + return LEXY_MOV(line); + }); + }; + + template<char Sep> + struct File { + static constexpr auto rule = + lexy::dsl::whitespace(lexy::dsl::newline) + + lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p<Line<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::eol))); + + static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>; + }; + + using CommaFile = File<','>; + using ColonFile = File<':'>; + using SemiColonFile = File<';'>; + using TabFile = File<'\t'>; + using BarFile = File<'|'>; +} + +namespace ovdl::csv::grammar::utf8 { + constexpr auto character = lexy::dsl::unicode::character; + constexpr auto control = lexy::dsl::unicode::control; + struct StringValue { static constexpr auto escaped_symbols = lexy::symbol_table<char> // .map<'"'>('"') @@ -25,52 +158,58 @@ namespace ovdl::csv::grammar { .map<'r'>('\r') .map<'t'>('\t'); /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" - static constexpr auto escaped_quote = lexy::symbol_table<char> // - .map<'"'>('"'); + // static constexpr auto escaped_quote = lexy::symbol_table<char> // + // .map<'"'>('"'); static constexpr auto rule = [] { // Arbitrary code points - auto c = -lexy::dsl::lit_c<'"'>; + auto c = character - (control / lexy::dsl::lit_b<'"'>); auto back_escape = lexy::dsl::backslash_escape // .symbol<escaped_symbols>(); - auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // - .symbol<escaped_quote>(); + // auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + // .symbol<escaped_quote>(); - return lexy::dsl::quoted(c, back_escape, quote_escape); + return lexy::dsl::quoted(c, back_escape); }(); static constexpr auto value = lexy::as_string<std::string>; }; - template<auto Sep> + template<char Sep> struct PlainValue { - static constexpr auto rule = lexy::dsl::identifier(-(Sep / lexy::dsl::lit_c<'\n'>)); + static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b<Sep> / lexy::dsl::ascii::newline)); static constexpr auto value = lexy::as_string<std::string>; }; - template<auto Sep> + template<char Sep> struct Value { static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>; static constexpr auto value = lexy::forward<std::string>; }; - template<auto Sep> - struct SeperatorCount { - static constexpr auto rule = lexy::dsl::list(Sep); + template<char Sep> + struct SepConst { + static constexpr auto rule = lexy::dsl::lit_b<Sep>; + static constexpr auto value = lexy::constant(1); + }; + + template<char Sep> + struct Seperator { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Sep>>); static constexpr auto value = lexy::count; }; - template<auto Sep> + template<char Sep> struct LineEnd { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<SeperatorCount<Sep>>)); + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Sep>>)); static constexpr auto value = lexy::fold_inplace<csv::LineObject>( std::initializer_list<csv::LineObject::value_type> {}, [](csv::LineObject& result, auto&& arg) { if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) { // Count seperators, adds to previous value, making it a position using position_type = csv::LineObject::position_type; - result.emplace_back(static_cast<position_type>(arg + std::get<0>(result.back())), ""); + result.emplace_back(static_cast<position_type>(arg + result.back().first), ""); } else { if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); else { @@ -81,7 +220,7 @@ namespace ovdl::csv::grammar { }); }; - template<auto Sep> + template<char Sep> struct Line { static constexpr auto suffix_setter(csv::LineObject& line) { @@ -94,7 +233,7 @@ namespace ovdl::csv::grammar { } }; - static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<SeperatorCount<Sep>> >> lexy::dsl::p<LineEnd<Sep>>; + static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<Seperator<Sep>> >> lexy::dsl::p<LineEnd<Sep>>; static constexpr auto value = lexy::callback<csv::LineObject>( [](csv::LineObject&& line) { @@ -112,7 +251,7 @@ namespace ovdl::csv::grammar { }); }; - template<auto Sep> + template<char Sep> struct File { static constexpr auto rule = lexy::dsl::whitespace(lexy::dsl::newline) + @@ -121,9 +260,9 @@ namespace ovdl::csv::grammar { static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>; }; - using CommaFile = File<lexy::dsl::lit_c<','>>; - using ColonFile = File<lexy::dsl::lit_c<':'>>; - using SemiColonFile = File<lexy::dsl::lit_c<';'>>; - using TabFile = File<lexy::dsl::lit_c<'\t'>>; - using BarFile = File<lexy::dsl::lit_c<'|'>>; + using CommaFile = File<','>; + using ColonFile = File<':'>; + using SemiColonFile = File<';'>; + using TabFile = File<'\t'>; + using BarFile = File<'|'>; }
\ No newline at end of file diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp index 45f3142..86ad02e 100644 --- a/src/openvic-dataloader/csv/Parser.cpp +++ b/src/openvic-dataloader/csv/Parser.cpp @@ -3,6 +3,7 @@ #include <openvic-dataloader/csv/LineObject.hpp> #include <openvic-dataloader/csv/Parser.hpp> +#include <openvic-dataloader/detail/ClassExport.hpp> #include <lexy/action/parse.hpp> #include <lexy/encoding.hpp> @@ -20,11 +21,26 @@ using namespace ovdl::csv; /// BufferHandler /// -class Parser::BufferHandler final : public detail::BasicBufferHandler<lexy::utf8_char_encoding> { +template<EncodingType> +struct LexyEncodingFrom { +}; + +template<> +struct LexyEncodingFrom<EncodingType::Windows1252> { + using encoding = lexy::default_encoding; +}; + +template<> +struct LexyEncodingFrom<EncodingType::Utf8> { + using encoding = lexy::utf8_char_encoding; +}; + +template<EncodingType Encoding> +class Parser<Encoding>::BufferHandler final : public detail::BasicBufferHandler<typename LexyEncodingFrom<Encoding>::encoding> { public: template<typename Node, typename ErrorCallback> std::optional<std::vector<ParseError>> parse(const ErrorCallback& callback) { - auto result = lexy::parse<Node>(_buffer, callback); + auto result = lexy::parse<Node>(this->_buffer, callback); if (!result) { return result.errors(); } @@ -42,36 +58,45 @@ private: /// BufferHandler /// -Parser::Parser() +template<EncodingType Encoding> +Parser<Encoding>::Parser() : _buffer_handler(std::make_unique<BufferHandler>()) { set_error_log_to_stderr(); } -Parser::Parser(Parser&&) = default; -Parser& Parser::operator=(Parser&&) = default; -Parser::~Parser() = default; +template<EncodingType Encoding> +Parser<Encoding>::Parser(Parser&&) = default; +template<EncodingType Encoding> +Parser<Encoding>& Parser<Encoding>::operator=(Parser&&) = default; +template<EncodingType Encoding> +Parser<Encoding>::~Parser() = default; -Parser Parser::from_buffer(const char* data, std::size_t size) { +template<EncodingType Encoding> +Parser<Encoding> Parser<Encoding>::from_buffer(const char* data, std::size_t size) { Parser result; return std::move(result.load_from_buffer(data, size)); } -Parser Parser::from_buffer(const char* start, const char* end) { +template<EncodingType Encoding> +Parser<Encoding> Parser<Encoding>::from_buffer(const char* start, const char* end) { Parser result; return std::move(result.load_from_buffer(start, end)); } -Parser Parser::from_string(const std::string_view string) { +template<EncodingType Encoding> +Parser<Encoding> Parser<Encoding>::from_string(const std::string_view string) { Parser result; return std::move(result.load_from_string(string)); } -Parser Parser::from_file(const char* path) { +template<EncodingType Encoding> +Parser<Encoding> Parser<Encoding>::from_file(const char* path) { Parser result; return std::move(result.load_from_file(path)); } -Parser Parser::from_file(const std::filesystem::path& path) { +template<EncodingType Encoding> +Parser<Encoding> Parser<Encoding>::from_file(const std::filesystem::path& path) { Parser result; return std::move(result.load_from_file(path)); } @@ -89,8 +114,9 @@ Parser Parser::from_file(const std::filesystem::path& path) { /// @param func /// @param args /// +template<EncodingType Encoding> template<typename... Args> -constexpr void Parser::_run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args) { +constexpr void Parser<Encoding>::_run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args) { _warnings.clear(); _errors.clear(); _has_fatal_error = false; @@ -101,43 +127,55 @@ constexpr void Parser::_run_load_func(detail::LoadCallback<BufferHandler, Args.. } } -constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size) { +template<EncodingType Encoding> +constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* data, std::size_t size) { // Type can't be deduced? _run_load_func(std::mem_fn(&BufferHandler::load_buffer_size), data, size); return *this; } -constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) { +template<EncodingType Encoding> +constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* start, const char* end) { // Type can't be deduced? _run_load_func(std::mem_fn(&BufferHandler::load_buffer), start, end); return *this; } -constexpr Parser& Parser::load_from_string(const std::string_view string) { +template<EncodingType Encoding> +constexpr Parser<Encoding>& Parser<Encoding>::load_from_string(const std::string_view string) { return load_from_buffer(string.data(), string.size()); } -constexpr Parser& Parser::load_from_file(const char* path) { +template<EncodingType Encoding> +constexpr Parser<Encoding>& Parser<Encoding>::load_from_file(const char* path) { _file_path = path; // Type can be deduced?? _run_load_func(std::mem_fn(&BufferHandler::load_file), path); return *this; } -Parser& Parser::load_from_file(const std::filesystem::path& path) { +template<EncodingType Encoding> +Parser<Encoding>& Parser<Encoding>::load_from_file(const std::filesystem::path& path) { return load_from_file(path.string().c_str()); } -constexpr Parser& Parser::load_from_file(const detail::Has_c_str auto& path) { +template<EncodingType Encoding> +constexpr Parser<Encoding>& Parser<Encoding>::load_from_file(const detail::Has_c_str auto& path) { return load_from_file(path.c_str()); } -bool Parser::parse_csv() { +template<EncodingType Encoding> +bool Parser<Encoding>::parse_csv() { if (!_buffer_handler->is_valid()) { return false; } - auto errors = _buffer_handler->parse<csv::grammar::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + std::optional<std::vector<ParseError>> errors; + if constexpr (Encoding == EncodingType::Windows1252) { + errors = _buffer_handler->template parse<csv::grammar::windows1252::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + } else { + errors = _buffer_handler->template parse<csv::grammar::utf8::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + } if (errors) { _errors.reserve(errors->size()); for (auto& err : errors.value()) { @@ -150,6 +188,10 @@ bool Parser::parse_csv() { return true; } -const std::vector<csv::LineObject>& Parser::get_lines() const { +template<EncodingType Encoding> +const std::vector<csv::LineObject>& Parser<Encoding>::get_lines() const { return _lines; -}
\ No newline at end of file +} + +template class ovdl::csv::Parser<EncodingType::Windows1252>; +template class ovdl::csv::Parser<EncodingType::Utf8>;
\ No newline at end of file |