diff options
Diffstat (limited to 'src/openvic-dataloader/csv')
| -rw-r--r-- | src/openvic-dataloader/csv/CsvGrammar.hpp | 129 | ||||
| -rw-r--r-- | src/openvic-dataloader/csv/Parser.cpp | 151 |
2 files changed, 280 insertions, 0 deletions
diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp new file mode 100644 index 0000000..edce97b --- /dev/null +++ b/src/openvic-dataloader/csv/CsvGrammar.hpp @@ -0,0 +1,129 @@ +#pragma once + +#include <initializer_list> +#include <string> +#include <tuple> +#include <type_traits> +#include <vector> + +#include <openvic-dataloader/csv/LineObject.hpp> + +#include <lexy/callback.hpp> +#include <lexy/dsl.hpp> + +// Grammar Definitions // +namespace ovdl::csv::grammar { + struct StringValue { + static constexpr auto escaped_symbols = lexy::symbol_table<char> // + .map<'"'>('"') + .map<'\''>('\'') + .map<'\\'>('\\') + .map<'/'>('/') + .map<'b'>('\b') + .map<'f'>('\f') + .map<'n'>('\n') + .map<'r'>('\r') + .map<'t'>('\t'); + /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" + static constexpr auto escaped_quote = lexy::symbol_table<char> // + .map<'"'>('"'); + static constexpr auto rule = [] { + // Arbitrary code points + auto c = -lexy::dsl::lit_c<'"'>; + + auto back_escape = lexy::dsl::backslash_escape // + .symbol<escaped_symbols>(); + + auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + .symbol<escaped_quote>(); + + return lexy::dsl::quoted(c, back_escape, quote_escape); + }(); + + static constexpr auto value = lexy::as_string<std::string>; + }; + + template<auto Sep> + struct PlainValue { + static constexpr auto rule = lexy::dsl::identifier(-(Sep / lexy::dsl::lit_c<'\n'>)); + static constexpr auto value = lexy::as_string<std::string>; + }; + + template<auto Sep> + struct Value { + static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>; + static constexpr auto value = lexy::forward<std::string>; + }; + + template<auto Sep> + struct SeperatorCount { + static constexpr auto rule = lexy::dsl::list(Sep); + static constexpr auto value = lexy::count; + }; + + template<auto Sep> + struct LineEnd { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<SeperatorCount<Sep>>)); + static constexpr auto value = lexy::fold_inplace<csv::LineObject>( + std::initializer_list<csv::LineObject::value_type> {}, + [](csv::LineObject& result, auto&& arg) { + if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) { + // Count seperators, adds to previous value, making it a position + using position_type = csv::LineObject::position_type; + result.emplace_back(static_cast<position_type>(arg + std::get<0>(result.back())), ""); + } else { + if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); + else { + auto& [pos, value] = result.back(); + value = arg; + } + } + }); + }; + + template<auto Sep> + struct Line { + + static constexpr auto suffix_setter(csv::LineObject& line) { + auto& [position, value] = line.back(); + if (value.empty()) { + line.set_suffix_end(position); + line.pop_back(); + } else { + line.set_suffix_end(position + 1); + } + }; + + static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<SeperatorCount<Sep>> >> lexy::dsl::p<LineEnd<Sep>>; + static constexpr auto value = + lexy::callback<csv::LineObject>( + [](csv::LineObject&& line) { + suffix_setter(line); + return LEXY_MOV(line); + }, + [](std::size_t prefix_count, csv::LineObject&& line) { + line.set_prefix_end(prefix_count); + // position needs to be adjusted to prefix + for (auto& [position, value] : line) { + position += prefix_count; + } + suffix_setter(line); + return LEXY_MOV(line); + }); + }; + + template<auto Sep> + struct File { + static constexpr auto rule = + lexy::dsl::whitespace(lexy::dsl::newline) + + lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p<Line<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::eol))); + + static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>; + }; + + using CommaFile = File<lexy::dsl::lit_c<','>>; + using ColonFile = File<lexy::dsl::lit_c<':'>>; + using SemiColonFile = File<lexy::dsl::lit_c<';'>>; + using TabFile = File<lexy::dsl::lit_c<'\t'>>; + using BarFile = File<lexy::dsl::lit_c<'|'>>; +}
\ No newline at end of file diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp new file mode 100644 index 0000000..8a99085 --- /dev/null +++ b/src/openvic-dataloader/csv/Parser.cpp @@ -0,0 +1,151 @@ +#include <memory> +#include <vector> + +#include <openvic-dataloader/csv/LineObject.hpp> +#include <openvic-dataloader/csv/Parser.hpp> + +#include <lexy/action/parse.hpp> +#include <lexy/encoding.hpp> +#include <lexy/input/buffer.hpp> +#include <lexy/input/file.hpp> + +#include "csv/CsvGrammar.hpp" +#include "detail/BasicBufferHandler.hpp" +#include "detail/Errors.hpp" +#include "detail/LexyReportError.hpp" +#include "detail/OStreamOutputIterator.hpp" + +using namespace ovdl; +using namespace ovdl::csv; + +/// BufferHandler /// + +class Parser::BufferHandler final : public detail::BasicBufferHandler<lexy::utf8_char_encoding> { +public: + template<typename Node, typename ErrorCallback> + std::optional<std::vector<ParseError>> parse(const ErrorCallback& callback) { + auto result = lexy::parse<Node>(_buffer, callback); + if (!result) { + return result.errors(); + } + _lines = std::move(result.value()); + return std::nullopt; + } + + std::vector<csv::LineObject>& get_lines() { + return _lines; + } + +private: + std::vector<csv::LineObject> _lines; +}; + +/// BufferHandler /// + +Parser::Parser() + : _buffer_handler(std::make_unique<BufferHandler>()) { + set_error_log_to_stderr(); +} + +Parser::Parser(Parser&&) = default; +Parser& Parser::operator=(Parser&&) = default; +Parser::~Parser() = default; + +Parser Parser::from_buffer(const char* data, std::size_t size) { + Parser result; + return std::move(result.load_from_buffer(data, size)); +} + +Parser Parser::from_buffer(const char* start, const char* end) { + Parser result; + return std::move(result.load_from_buffer(start, end)); +} + +Parser Parser::from_string(const std::string_view string) { + Parser result; + return std::move(result.load_from_string(string)); +} + +Parser Parser::from_file(const char* path) { + Parser result; + return std::move(result.load_from_file(path)); +} + +Parser Parser::from_file(const std::filesystem::path& path) { + Parser result; + return std::move(result.load_from_file(path)); +} + +/// +/// @brief Executes a function on _buffer_handler that is expected to load a buffer +/// +/// Expected Use: +/// @code {.cpp} +/// _run_load_func(&BufferHandler::<load_function>, <arguments>); +/// @endcode +/// +/// @tparam Type +/// @tparam Args +/// @param func +/// @param args +/// +template<typename... Args> +constexpr void Parser::_run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args) { + _warnings.clear(); + _errors.clear(); + _has_fatal_error = false; + if (auto error = func(_buffer_handler.get(), std::forward<Args>(args)...); error) { + _has_fatal_error = error.value().type == ParseError::Type::Fatal; + _errors.push_back(error.value()); + _error_stream.get() << "Error: " << _errors.back().message << '\n'; + } +} + +constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size) { + // Type can't be deduced? + _run_load_func(std::mem_fn(&BufferHandler::load_buffer_size), data, size); + return *this; +} + +constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) { + // Type can't be deduced? + _run_load_func(std::mem_fn(&BufferHandler::load_buffer), start, end); + return *this; +} + +constexpr Parser& Parser::load_from_string(const std::string_view string) { + return load_from_buffer(string.data(), string.size()); +} + +constexpr Parser& Parser::load_from_file(const char* path) { + _file_path = path; + // Type can be deduced?? + _run_load_func(std::mem_fn(&BufferHandler::load_file), path); + return *this; +} + +Parser& Parser::load_from_file(const std::filesystem::path& path) { + return load_from_file(path.string().c_str()); +} + +constexpr Parser& Parser::load_from_file(const detail::Has_c_str auto& path) { + return load_from_file(path.c_str()); +} + +bool Parser::parse_csv() { + if (!_buffer_handler->is_valid()) { + return false; + } + + auto errors = _buffer_handler->parse<csv::grammar::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + if (errors) { + _errors.reserve(errors->size()); + for (auto& err : errors.value()) { + _has_fatal_error |= err.type == ParseError::Type::Fatal; + _errors.push_back(err); + } + return false; + } + _lines = std::move(_buffer_handler->get_lines()); + return true; +}
\ No newline at end of file |