From a705b0223664f915d8bb00d4f89e05838da3e4bc Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Sun, 3 Sep 2023 17:54:18 -0400 Subject: Add Windows-1252 CSV Parser Fix CsvGrammar counting behavior Add csv parse argument to headless/main.cpp Change LineObject contained value type to pair Add ostream print to LineObject Add vscode tasks.json Add vscode launch.json Move csv::Parser template class specialization to end of Parser.cpp Add detail/ClassExport.hpp to assist compiler export --- src/openvic-dataloader/csv/CsvGrammar.hpp | 187 ++++++++++++++++++++++++++---- src/openvic-dataloader/csv/Parser.cpp | 86 ++++++++++---- 2 files changed, 227 insertions(+), 46 deletions(-) (limited to 'src/openvic-dataloader/csv') diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp index edce97b..4765112 100644 --- a/src/openvic-dataloader/csv/CsvGrammar.hpp +++ b/src/openvic-dataloader/csv/CsvGrammar.hpp @@ -11,8 +11,141 @@ #include #include +#include "detail/LexyLitRange.hpp" + // Grammar Definitions // -namespace ovdl::csv::grammar { +namespace ovdl::csv::grammar::windows1252 { + constexpr auto character = detail::lexydsl::make_range<0x01, 0xFF>(); + constexpr auto control = + lexy::dsl::ascii::control / + lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> / + lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>; + + struct StringValue { + static constexpr auto escaped_symbols = lexy::symbol_table // + .map<'"'>('"') + .map<'\''>('\'') + .map<'\\'>('\\') + .map<'/'>('/') + .map<'b'>('\b') + .map<'f'>('\f') + .map<'n'>('\n') + .map<'r'>('\r') + .map<'t'>('\t'); + /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" + // static constexpr auto escaped_quote = lexy::symbol_table // + // .map<'"'>('"'); + static constexpr auto rule = [] { + // Arbitrary code points + auto c = character - (control / lexy::dsl::lit_b<'"'>); + + auto back_escape = lexy::dsl::backslash_escape // + .symbol(); + + // auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + // .symbol(); + + return lexy::dsl::quoted(c, back_escape); + }(); + + static constexpr auto value = lexy::as_string; + }; + + template + struct PlainValue { + static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline)); + static constexpr auto value = lexy::as_string; + }; + + template + struct Value { + static constexpr auto rule = lexy::dsl::p | lexy::dsl::p>; + static constexpr auto value = lexy::forward; + }; + + template + struct SepConst { + static constexpr auto rule = lexy::dsl::lit_b; + static constexpr auto value = lexy::constant(1); + }; + + template + struct Seperator { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>); + static constexpr auto value = lexy::count; + }; + + template + struct LineEnd { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::p>)); + static constexpr auto value = lexy::fold_inplace( + std::initializer_list {}, + [](csv::LineObject& result, auto&& arg) { + if constexpr (std::is_same_v, std::size_t>) { + // Count seperators, adds to previous value, making it a position + using position_type = csv::LineObject::position_type; + result.emplace_back(static_cast(arg + result.back().first), ""); + } else { + if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); + else { + auto& [pos, value] = result.back(); + value = arg; + } + } + }); + }; + + template + struct Line { + + static constexpr auto suffix_setter(csv::LineObject& line) { + auto& [position, value] = line.back(); + if (value.empty()) { + line.set_suffix_end(position); + line.pop_back(); + } else { + line.set_suffix_end(position + 1); + } + }; + + static constexpr auto rule = lexy::dsl::p> | lexy::dsl::p> >> lexy::dsl::p>; + static constexpr auto value = + lexy::callback( + [](csv::LineObject&& line) { + suffix_setter(line); + return LEXY_MOV(line); + }, + [](std::size_t prefix_count, csv::LineObject&& line) { + line.set_prefix_end(prefix_count); + // position needs to be adjusted to prefix + for (auto& [position, value] : line) { + position += prefix_count; + } + suffix_setter(line); + return LEXY_MOV(line); + }); + }; + + template + struct File { + static constexpr auto rule = + lexy::dsl::whitespace(lexy::dsl::newline) + + lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::eol))); + + static constexpr auto value = lexy::as_list>; + }; + + using CommaFile = File<','>; + using ColonFile = File<':'>; + using SemiColonFile = File<';'>; + using TabFile = File<'\t'>; + using BarFile = File<'|'>; +} + +namespace ovdl::csv::grammar::utf8 { + constexpr auto character = lexy::dsl::unicode::character; + constexpr auto control = lexy::dsl::unicode::control; + struct StringValue { static constexpr auto escaped_symbols = lexy::symbol_table // .map<'"'>('"') @@ -25,52 +158,58 @@ namespace ovdl::csv::grammar { .map<'r'>('\r') .map<'t'>('\t'); /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" - static constexpr auto escaped_quote = lexy::symbol_table // - .map<'"'>('"'); + // static constexpr auto escaped_quote = lexy::symbol_table // + // .map<'"'>('"'); static constexpr auto rule = [] { // Arbitrary code points - auto c = -lexy::dsl::lit_c<'"'>; + auto c = character - (control / lexy::dsl::lit_b<'"'>); auto back_escape = lexy::dsl::backslash_escape // .symbol(); - auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // - .symbol(); + // auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + // .symbol(); - return lexy::dsl::quoted(c, back_escape, quote_escape); + return lexy::dsl::quoted(c, back_escape); }(); static constexpr auto value = lexy::as_string; }; - template + template struct PlainValue { - static constexpr auto rule = lexy::dsl::identifier(-(Sep / lexy::dsl::lit_c<'\n'>)); + static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b / lexy::dsl::ascii::newline)); static constexpr auto value = lexy::as_string; }; - template + template struct Value { static constexpr auto rule = lexy::dsl::p | lexy::dsl::p>; static constexpr auto value = lexy::forward; }; - template - struct SeperatorCount { - static constexpr auto rule = lexy::dsl::list(Sep); + template + struct SepConst { + static constexpr auto rule = lexy::dsl::lit_b; + static constexpr auto value = lexy::constant(1); + }; + + template + struct Seperator { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>); static constexpr auto value = lexy::count; }; - template + template struct LineEnd { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::p>)); + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::p>)); static constexpr auto value = lexy::fold_inplace( std::initializer_list {}, [](csv::LineObject& result, auto&& arg) { if constexpr (std::is_same_v, std::size_t>) { // Count seperators, adds to previous value, making it a position using position_type = csv::LineObject::position_type; - result.emplace_back(static_cast(arg + std::get<0>(result.back())), ""); + result.emplace_back(static_cast(arg + result.back().first), ""); } else { if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); else { @@ -81,7 +220,7 @@ namespace ovdl::csv::grammar { }); }; - template + template struct Line { static constexpr auto suffix_setter(csv::LineObject& line) { @@ -94,7 +233,7 @@ namespace ovdl::csv::grammar { } }; - static constexpr auto rule = lexy::dsl::p> | lexy::dsl::p> >> lexy::dsl::p>; + static constexpr auto rule = lexy::dsl::p> | lexy::dsl::p> >> lexy::dsl::p>; static constexpr auto value = lexy::callback( [](csv::LineObject&& line) { @@ -112,7 +251,7 @@ namespace ovdl::csv::grammar { }); }; - template + template struct File { static constexpr auto rule = lexy::dsl::whitespace(lexy::dsl::newline) + @@ -121,9 +260,9 @@ namespace ovdl::csv::grammar { static constexpr auto value = lexy::as_list>; }; - using CommaFile = File>; - using ColonFile = File>; - using SemiColonFile = File>; - using TabFile = File>; - using BarFile = File>; + using CommaFile = File<','>; + using ColonFile = File<':'>; + using SemiColonFile = File<';'>; + using TabFile = File<'\t'>; + using BarFile = File<'|'>; } \ No newline at end of file diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp index 45f3142..86ad02e 100644 --- a/src/openvic-dataloader/csv/Parser.cpp +++ b/src/openvic-dataloader/csv/Parser.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -20,11 +21,26 @@ using namespace ovdl::csv; /// BufferHandler /// -class Parser::BufferHandler final : public detail::BasicBufferHandler { +template +struct LexyEncodingFrom { +}; + +template<> +struct LexyEncodingFrom { + using encoding = lexy::default_encoding; +}; + +template<> +struct LexyEncodingFrom { + using encoding = lexy::utf8_char_encoding; +}; + +template +class Parser::BufferHandler final : public detail::BasicBufferHandler::encoding> { public: template std::optional> parse(const ErrorCallback& callback) { - auto result = lexy::parse(_buffer, callback); + auto result = lexy::parse(this->_buffer, callback); if (!result) { return result.errors(); } @@ -42,36 +58,45 @@ private: /// BufferHandler /// -Parser::Parser() +template +Parser::Parser() : _buffer_handler(std::make_unique()) { set_error_log_to_stderr(); } -Parser::Parser(Parser&&) = default; -Parser& Parser::operator=(Parser&&) = default; -Parser::~Parser() = default; +template +Parser::Parser(Parser&&) = default; +template +Parser& Parser::operator=(Parser&&) = default; +template +Parser::~Parser() = default; -Parser Parser::from_buffer(const char* data, std::size_t size) { +template +Parser Parser::from_buffer(const char* data, std::size_t size) { Parser result; return std::move(result.load_from_buffer(data, size)); } -Parser Parser::from_buffer(const char* start, const char* end) { +template +Parser Parser::from_buffer(const char* start, const char* end) { Parser result; return std::move(result.load_from_buffer(start, end)); } -Parser Parser::from_string(const std::string_view string) { +template +Parser Parser::from_string(const std::string_view string) { Parser result; return std::move(result.load_from_string(string)); } -Parser Parser::from_file(const char* path) { +template +Parser Parser::from_file(const char* path) { Parser result; return std::move(result.load_from_file(path)); } -Parser Parser::from_file(const std::filesystem::path& path) { +template +Parser Parser::from_file(const std::filesystem::path& path) { Parser result; return std::move(result.load_from_file(path)); } @@ -89,8 +114,9 @@ Parser Parser::from_file(const std::filesystem::path& path) { /// @param func /// @param args /// +template template -constexpr void Parser::_run_load_func(detail::LoadCallback auto func, Args... args) { +constexpr void Parser::_run_load_func(detail::LoadCallback auto func, Args... args) { _warnings.clear(); _errors.clear(); _has_fatal_error = false; @@ -101,43 +127,55 @@ constexpr void Parser::_run_load_func(detail::LoadCallback +constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size) { // Type can't be deduced? _run_load_func(std::mem_fn(&BufferHandler::load_buffer_size), data, size); return *this; } -constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) { +template +constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) { // Type can't be deduced? _run_load_func(std::mem_fn(&BufferHandler::load_buffer), start, end); return *this; } -constexpr Parser& Parser::load_from_string(const std::string_view string) { +template +constexpr Parser& Parser::load_from_string(const std::string_view string) { return load_from_buffer(string.data(), string.size()); } -constexpr Parser& Parser::load_from_file(const char* path) { +template +constexpr Parser& Parser::load_from_file(const char* path) { _file_path = path; // Type can be deduced?? _run_load_func(std::mem_fn(&BufferHandler::load_file), path); return *this; } -Parser& Parser::load_from_file(const std::filesystem::path& path) { +template +Parser& Parser::load_from_file(const std::filesystem::path& path) { return load_from_file(path.string().c_str()); } -constexpr Parser& Parser::load_from_file(const detail::Has_c_str auto& path) { +template +constexpr Parser& Parser::load_from_file(const detail::Has_c_str auto& path) { return load_from_file(path.c_str()); } -bool Parser::parse_csv() { +template +bool Parser::parse_csv() { if (!_buffer_handler->is_valid()) { return false; } - auto errors = _buffer_handler->parse(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + std::optional> errors; + if constexpr (Encoding == EncodingType::Windows1252) { + errors = _buffer_handler->template parse(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + } else { + errors = _buffer_handler->template parse(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + } if (errors) { _errors.reserve(errors->size()); for (auto& err : errors.value()) { @@ -150,6 +188,10 @@ bool Parser::parse_csv() { return true; } -const std::vector& Parser::get_lines() const { +template +const std::vector& Parser::get_lines() const { return _lines; -} \ No newline at end of file +} + +template class ovdl::csv::Parser; +template class ovdl::csv::Parser; \ No newline at end of file -- cgit v1.2.3-56-ga3b1