aboutsummaryrefslogtreecommitdiff
path: root/src/openvic-dataloader/csv
diff options
context:
space:
mode:
author George L. Albany <Megacake1234@gmail.com>2024-06-18 19:43:20 +0200
committer GitHub <noreply@github.com>2024-06-18 19:43:20 +0200
commit8b623bf4087aa360842ad31145d4ab6946cee9aa (patch)
treef15ebc47d6bf370031af28e4bb4814ae30ef46e1 /src/openvic-dataloader/csv
parent7b521d6023113372cf6b02e562828273c4040f0e (diff)
parentb0c3ba3f91926b0c95625bdbf4aab69269130b13 (diff)
Merge pull request #46 from OpenVicProject/fix/char-detection
Add runtime encoding detection and conversion
Diffstat (limited to 'src/openvic-dataloader/csv')
-rw-r--r--src/openvic-dataloader/csv/CsvGrammar.hpp244
-rw-r--r--src/openvic-dataloader/csv/CsvParseState.hpp26
-rw-r--r--src/openvic-dataloader/csv/Parser.cpp182
3 files changed, 222 insertions, 230 deletions
diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp
index 5451f26..19aee54 100644
--- a/src/openvic-dataloader/csv/CsvGrammar.hpp
+++ b/src/openvic-dataloader/csv/CsvGrammar.hpp
@@ -9,22 +9,20 @@
#include <openvic-dataloader/csv/LineObject.hpp>
#include <openvic-dataloader/csv/Parser.hpp>
+#include <lexy/_detail/config.hpp>
#include <lexy/callback.hpp>
+#include <lexy/callback/string.hpp>
#include <lexy/dsl.hpp>
+#include <lexy/dsl/ascii.hpp>
+#include <lexy/dsl/option.hpp>
+#include <lexy/encoding.hpp>
+#include "detail/Convert.hpp"
+#include "detail/InternalConcepts.hpp"
#include "detail/dsl.hpp"
// Grammar Definitions //
namespace ovdl::csv::grammar {
- using EncodingType = ovdl::csv::EncodingType;
-
- template<typename T>
- concept ParseChars = requires() {
- { T::character };
- { T::control };
- };
-
- template<ParseChars T>
struct ParseOptions {
/// @brief Seperator character
char SepChar;
@@ -33,12 +31,34 @@ namespace ovdl::csv::grammar {
/// @brief Paradox-style localization escape characters
/// @note Is ignored if SupportStrings is true
char EscapeChar;
+ };
- static constexpr auto parse_chars = T {};
- static constexpr auto character = parse_chars.character;
- static constexpr auto control = parse_chars.control;
+ struct ConvertErrorHandler {
+ static constexpr void on_invalid_character(detail::IsStateType auto& state, auto reader) {
+ state.logger().warning("invalid character value '{}' found", static_cast<int>(reader.peek())) //
+ .primary(BasicNodeLocation { reader.position() }, "here")
+ .finish();
+ }
};
+ constexpr bool IsUtf8(auto encoding) {
+ return std::same_as<std::decay_t<decltype(encoding)>, lexy::utf8_char_encoding>;
+ }
+
+ template<ParseOptions Options, typename String>
+ constexpr auto convert_as_string = convert::convert_as_string<
+ String,
+ ConvertErrorHandler>;
+
+ constexpr auto ansi_character = lexy::dsl::ascii::character / dsl::lit_b_range<0x80, 0xFF>;
+ constexpr auto ansi_control =
+ lexy::dsl::ascii::control /
+ lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> /
+ lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>;
+
+ constexpr auto utf_character = lexy::dsl::unicode::character;
+ constexpr auto utf_control = lexy::dsl::unicode::control;
+
constexpr auto escaped_symbols = lexy::symbol_table<char> //
.map<'"'>('"')
.map<'\''>('\'')
@@ -55,38 +75,95 @@ namespace ovdl::csv::grammar {
template<ParseOptions Options>
struct CsvGrammar {
- struct StringValue {
- static constexpr auto rule = [] {
- // Arbitrary code points
- auto c = Options.character - Options.control;
+ struct StringValue : lexy::scan_production<std::string>,
+ lexy::token_production {
+
+ template<typename Context, typename Reader>
+ static constexpr scan_result scan(lexy::rule_scanner<Context, Reader>& scanner, detail::IsFileParseState auto& state) {
+ using encoding = typename Reader::encoding;
+
+ constexpr auto rule = [] {
+ // Arbitrary code points
+ auto c = [] {
+ if constexpr (std::same_as<encoding, lexy::default_encoding> || std::same_as<encoding, lexy::byte_encoding>) {
+ return ansi_character - ansi_control;
+ } else {
+ return utf_character - utf_control;
+ }
+ }();
- auto back_escape = lexy::dsl::backslash_escape //
- .symbol<escaped_symbols>();
+ auto back_escape = lexy::dsl::backslash_escape //
+ .symbol<escaped_symbols>();
- auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
- .template symbol<escaped_quote>();
+ auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
+ .template symbol<escaped_quote>();
- return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape);
- }();
+ return lexy::dsl::delimited(lexy::dsl::lit_c<'"'>, lexy::dsl::not_followed_by(lexy::dsl::lit_c<'"'>, lexy::dsl::lit_c<'"'>))(c, back_escape, quote_escape);
+ }();
+
+ lexy::scan_result<std::string> str_result = scanner.template parse<std::string>(rule);
+ if (!scanner || !str_result)
+ return lexy::scan_failed;
+ return str_result.value();
+ }
- static constexpr auto value = lexy::as_string<std::string>;
+ static constexpr auto rule = lexy::dsl::peek(lexy::dsl::lit_c<'"'>) >> lexy::dsl::scan;
+
+ static constexpr auto value = convert_as_string<Options, std::string> >> lexy::forward<std::string>;
};
- struct PlainValue {
- static constexpr auto rule = [] {
+ struct PlainValue : lexy::scan_production<std::string>,
+ lexy::token_production {
+
+ template<auto character>
+ static constexpr auto _escape_check = character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline);
+
+ template<typename Context, typename Reader>
+ static constexpr scan_result scan(lexy::rule_scanner<Context, Reader>& scanner, detail::IsFileParseState auto& state) {
+ using encoding = typename Reader::encoding;
+
+ constexpr auto rule = [] {
+ constexpr auto character = [] {
+ if constexpr (std::same_as<encoding, lexy::default_encoding> || std::same_as<encoding, lexy::byte_encoding>) {
+ return ansi_character;
+ } else {
+ return utf_character;
+ }
+ }();
+
+ if constexpr (Options.SupportStrings) {
+ return lexy::dsl::identifier(character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline));
+ } else {
+ auto escape_check_char = _escape_check<character>;
+ auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>;
+ auto id_segment = lexy::dsl::identifier(id_check_char);
+ auto escape_segement = lexy::dsl::token(escape_check_char);
+ auto escape_sym = lexy::dsl::symbol<escaped_symbols>(escape_segement);
+ auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym;
+ return lexy::dsl::list(id_segment | escape_rule);
+ }
+ }();
+
if constexpr (Options.SupportStrings) {
- return lexy::dsl::identifier(Options.character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline));
+ auto lexeme_result = scanner.template parse<lexy::lexeme<Reader>>(rule);
+ if (!scanner || !lexeme_result)
+ return lexy::scan_failed;
+ return std::string { lexeme_result.value().begin(), lexeme_result.value().end() };
} else {
- auto escape_check_char = Options.character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline);
- auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>;
- auto id_segment = lexy::dsl::identifier(id_check_char);
- auto escape_segement = lexy::dsl::token(escape_check_char);
- auto escape_sym = lexy::dsl::symbol<escaped_symbols>(escape_segement);
- auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym;
- return lexy::dsl::list(id_segment | escape_rule);
+ lexy::scan_result<std::string> str_result = scanner.template parse<std::string>(rule);
+ if (!scanner || !str_result)
+ return lexy::scan_failed;
+ return str_result.value();
}
- }();
- static constexpr auto value = lexy::as_string<std::string>;
+ }
+
+ static constexpr auto rule =
+ dsl::peek(
+ _escape_check<ansi_character>,
+ _escape_check<utf_character>) >>
+ lexy::dsl::scan;
+
+ static constexpr auto value = convert_as_string<Options, std::string> >> lexy::forward<std::string>;
};
struct Value {
@@ -114,17 +191,17 @@ namespace ovdl::csv::grammar {
static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator>));
static constexpr auto value = lexy::fold_inplace<ovdl::csv::LineObject>(
std::initializer_list<ovdl::csv::LineObject::value_type> {},
- [](ovdl::csv::LineObject& result, auto&& arg) {
- if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
- // Count seperators, adds to previous value, making it a position
- using position_type = ovdl::csv::LineObject::position_type;
- result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
+ [](ovdl::csv::LineObject& result, std::size_t&& arg) {
+ // Count seperators, adds to previous value, making it a position
+ using position_type = ovdl::csv::LineObject::position_type;
+ result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
+ },
+ [](ovdl::csv::LineObject& result, std::string&& arg) {
+ if (result.empty()) {
+ result.emplace_back(0u, LEXY_MOV(arg));
} else {
- if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
- else {
- auto& [pos, value] = result.back();
- value = arg;
- }
+ auto& [pos, value] = result.back();
+ value = LEXY_MOV(arg);
}
});
};
@@ -169,74 +246,17 @@ namespace ovdl::csv::grammar {
static constexpr auto value = lexy::as_list<std::vector<ovdl::csv::LineObject>>;
};
- template<ParseChars T>
- using CommaFile = File<ParseOptions<T> { ',', false, '$' }>;
- template<ParseChars T>
- using ColonFile = File<ParseOptions<T> { ':', false, '$' }>;
- template<ParseChars T>
- using SemiColonFile = File<ParseOptions<T> { ';', false, '$' }>;
- template<ParseChars T>
- using TabFile = File<ParseOptions<T> { '\t', false, '$' }>;
- template<ParseChars T>
- using BarFile = File<ParseOptions<T> { '|', false, '$' }>;
-
- namespace strings {
- template<ParseChars T>
- using CommaFile = File<ParseOptions<T> { ',', true, '$' }>;
- template<ParseChars T>
- using ColonFile = File<ParseOptions<T> { ':', true, '$' }>;
- template<ParseChars T>
- using SemiColonFile = File<ParseOptions<T> { ';', true, '$' }>;
- template<ParseChars T>
- using TabFile = File<ParseOptions<T> { '\t', true, '$' }>;
- template<ParseChars T>
- using BarFile = File<ParseOptions<T> { '|', true, '$' }>;
- }
-}
-
-namespace ovdl::csv::grammar::windows1252 {
- struct windows1252_t {
- static constexpr auto character = dsl::make_range<0x01, 0xFF>();
- static constexpr auto control =
- lexy::dsl::ascii::control /
- lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> /
- lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>;
- };
-
- using CommaFile = CommaFile<windows1252_t>;
- using ColonFile = ColonFile<windows1252_t>;
- using SemiColonFile = SemiColonFile<windows1252_t>;
- using TabFile = TabFile<windows1252_t>;
- using BarFile = BarFile<windows1252_t>;
-
- namespace strings {
- using CommaFile = grammar::strings::CommaFile<windows1252_t>;
- using ColonFile = grammar::strings::ColonFile<windows1252_t>;
- using SemiColonFile = grammar::strings::SemiColonFile<windows1252_t>;
- using TabFile = grammar::strings::TabFile<windows1252_t>;
- using BarFile = grammar::strings::BarFile<windows1252_t>;
-
- }
-}
-
-namespace ovdl::csv::grammar::utf8 {
- struct unicode_t {
- static constexpr auto character = lexy::dsl::unicode::character;
- static constexpr auto control = lexy::dsl::unicode::control;
- };
-
- using CommaFile = CommaFile<unicode_t>;
- using ColonFile = ColonFile<unicode_t>;
- using SemiColonFile = SemiColonFile<unicode_t>;
- using TabFile = TabFile<unicode_t>;
- using BarFile = BarFile<unicode_t>;
+ using CommaFile = File<ParseOptions { ',', false, '$' }>;
+ using ColonFile = File<ParseOptions { ':', false, '$' }>;
+ using SemiColonFile = File<ParseOptions { ';', false, '$' }>;
+ using TabFile = File<ParseOptions { '\t', false, '$' }>;
+ using BarFile = File<ParseOptions { '|', false, '$' }>;
namespace strings {
- using CommaFile = grammar::strings::CommaFile<unicode_t>;
- using ColonFile = grammar::strings::ColonFile<unicode_t>;
- using SemiColonFile = grammar::strings::SemiColonFile<unicode_t>;
- using TabFile = grammar::strings::TabFile<unicode_t>;
- using BarFile = grammar::strings::BarFile<unicode_t>;
-
+ using CommaFile = File<ParseOptions { ',', true, '$' }>;
+ using ColonFile = File<ParseOptions { ':', true, '$' }>;
+ using SemiColonFile = File<ParseOptions { ';', true, '$' }>;
+ using TabFile = File<ParseOptions { '\t', true, '$' }>;
+ using BarFile = File<ParseOptions { '|', true, '$' }>;
}
} \ No newline at end of file
diff --git a/src/openvic-dataloader/csv/CsvParseState.hpp b/src/openvic-dataloader/csv/CsvParseState.hpp
index 2390453..ee60c34 100644
--- a/src/openvic-dataloader/csv/CsvParseState.hpp
+++ b/src/openvic-dataloader/csv/CsvParseState.hpp
@@ -1,28 +1,16 @@
#pragma once
-#include <openvic-dataloader/File.hpp>
-#include <openvic-dataloader/ParseState.hpp>
#include <openvic-dataloader/csv/LineObject.hpp>
#include <openvic-dataloader/csv/Parser.hpp>
#include <lexy/encoding.hpp>
-template<ovdl::csv::EncodingType>
-struct LexyEncodingFrom {
-};
+#include "File.hpp"
+#include "ParseState.hpp"
+#include "detail/InternalConcepts.hpp"
-template<>
-struct LexyEncodingFrom<ovdl::csv::EncodingType::Windows1252> {
- using encoding = lexy::default_encoding;
-};
+namespace ovdl::csv {
+ using CsvParseState = ovdl::FileParseState<ovdl::BasicFile<std::vector<ovdl::csv::LineObject>>>;
-template<>
-struct LexyEncodingFrom<ovdl::csv::EncodingType::Utf8> {
- using encoding = lexy::utf8_char_encoding;
-};
-
-template<ovdl::csv::EncodingType Encoding>
-using CsvFile = ovdl::BasicFile<typename LexyEncodingFrom<Encoding>::encoding, std::vector<ovdl::csv::LineObject>>;
-
-template<ovdl::csv::EncodingType Encoding>
-using CsvParseState = ovdl::FileParseState<CsvFile<Encoding>>; \ No newline at end of file
+ static_assert(detail::IsFileParseState<CsvParseState>, "CsvParseState failed IsFileParseState concept");
+} \ No newline at end of file
diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp
index 361f6ad..5dbee32 100644
--- a/src/openvic-dataloader/csv/Parser.cpp
+++ b/src/openvic-dataloader/csv/Parser.cpp
@@ -1,11 +1,14 @@
+#include <iostream>
+#include <optional>
+#include <type_traits>
#include <vector>
-#include <openvic-dataloader/File.hpp>
+#include <openvic-dataloader/NodeLocation.hpp>
#include <openvic-dataloader/csv/LineObject.hpp>
#include <openvic-dataloader/csv/Parser.hpp>
-#include <openvic-dataloader/detail/LexyReportError.hpp>
+#include <openvic-dataloader/detail/Encoding.hpp>
#include <openvic-dataloader/detail/OStreamOutputIterator.hpp>
-#include <openvic-dataloader/detail/utility/Utility.hpp>
+#include <openvic-dataloader/detail/Utility.hpp>
#include <lexy/action/parse.hpp>
#include <lexy/encoding.hpp>
@@ -22,15 +25,27 @@ using namespace ovdl::csv;
/// ParseHandler ///
-template<EncodingType Encoding>
-struct Parser<Encoding>::ParseHandler final : detail::BasicFileParseHandler<CsvParseState<Encoding>> {
+struct Parser::ParseHandler final : detail::BasicFileParseHandler<CsvParseState> {
template<typename Node>
std::optional<DiagnosticLogger::error_range> parse() {
- auto result = lexy::parse<Node>(this->buffer(), *this->_parse_state, this->_parse_state->logger().error_callback());
+ auto result = [&] {
+ switch (parse_state().encoding()) {
+ using enum detail::Encoding;
+ case Ascii:
+ case Utf8:
+ return lexy::parse<Node>(buffer<lexy::utf8_char_encoding>(), parse_state(), parse_state().logger().error_callback());
+ case Unknown:
+ case Windows1251:
+ case Windows1252:
+ return lexy::parse<Node>(buffer<lexy::default_encoding>(), parse_state(), parse_state().logger().error_callback());
+ default:
+ ovdl::detail::unreachable();
+ }
+ }();
if (!result) {
- return this->_parse_state->logger().get_errors();
+ return this->parse_state().logger().get_errors();
}
- _lines = std::move(result.value());
+ _lines = LEXY_MOV(result).value();
return std::nullopt;
}
@@ -42,55 +57,45 @@ private:
std::vector<csv::LineObject> _lines;
};
-/// BufferHandler ///
+/// ParserHandler ///
-template<EncodingType Encoding>
-Parser<Encoding>::Parser()
+Parser::Parser()
: _parse_handler(std::make_unique<ParseHandler>()) {
set_error_log_to_null();
}
-template<EncodingType Encoding>
-Parser<Encoding>::Parser(std::basic_ostream<char>& error_stream)
+Parser::Parser(std::basic_ostream<char>& error_stream)
: _parse_handler(std::make_unique<ParseHandler>()) {
set_error_log_to(error_stream);
}
-template<EncodingType Encoding>
-Parser<Encoding>::Parser(Parser&&) = default;
-template<EncodingType Encoding>
-Parser<Encoding>& Parser<Encoding>::operator=(Parser&&) = default;
-template<EncodingType Encoding>
-Parser<Encoding>::~Parser() = default;
+Parser::Parser(Parser&&) = default;
+Parser& Parser::operator=(Parser&&) = default;
+Parser::~Parser() = default;
-template<EncodingType Encoding>
-Parser<Encoding> Parser<Encoding>::from_buffer(const char* data, std::size_t size) {
+Parser Parser::from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback) {
Parser result;
- return std::move(result.load_from_buffer(data, size));
+ return std::move(result.load_from_buffer(data, size, encoding_fallback));
}
-template<EncodingType Encoding>
-Parser<Encoding> Parser<Encoding>::from_buffer(const char* start, const char* end) {
+Parser Parser::from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback) {
Parser result;
- return std::move(result.load_from_buffer(start, end));
+ return std::move(result.load_from_buffer(start, end, encoding_fallback));
}
-template<EncodingType Encoding>
-Parser<Encoding> Parser<Encoding>::from_string(const std::string_view string) {
+Parser Parser::from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback) {
Parser result;
- return std::move(result.load_from_string(string));
+ return std::move(result.load_from_string(string, encoding_fallback));
}
-template<EncodingType Encoding>
-Parser<Encoding> Parser<Encoding>::from_file(const char* path) {
+Parser Parser::from_file(const char* path, std::optional<detail::Encoding> encoding_fallback) {
Parser result;
- return std::move(result.load_from_file(path));
+ return std::move(result.load_from_file(path, encoding_fallback));
}
-template<EncodingType Encoding>
-Parser<Encoding> Parser<Encoding>::from_file(const std::filesystem::path& path) {
+Parser Parser::from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback) {
Parser result;
- return std::move(result.load_from_file(path));
+ return std::move(result.load_from_file(path, encoding_fallback));
}
///
@@ -106,9 +111,8 @@ Parser<Encoding> Parser<Encoding>::from_file(const std::filesystem::path& path)
/// @param func
/// @param args
///
-template<EncodingType Encoding>
template<typename... Args>
-constexpr void Parser<Encoding>::_run_load_func(detail::LoadCallback<ParseHandler, Args...> auto func, Args... args) {
+constexpr void Parser::_run_load_func(detail::LoadCallback<ParseHandler, Args...> auto func, Args... args) {
_has_fatal_error = false;
auto error = func(_parse_handler.get(), std::forward<Args>(args)...);
auto error_message = _parse_handler->make_error_from(error);
@@ -122,82 +126,66 @@ constexpr void Parser<Encoding>::_run_load_func(detail::LoadCallback<ParseHandle
}
}
-template<EncodingType Encoding>
-constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* data, std::size_t size) {
+constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback) {
// Type can't be deduced?
- _run_load_func(std::mem_fn(&ParseHandler::load_buffer_size), data, size);
+ _run_load_func(std::mem_fn(&ParseHandler::load_buffer_size), data, size, encoding_fallback);
return *this;
}
-template<EncodingType Encoding>
-constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* start, const char* end) {
+constexpr Parser& Parser::load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback) {
// Type can't be deduced?
- _run_load_func(std::mem_fn(&ParseHandler::load_buffer), start, end);
+ _run_load_func(std::mem_fn(&ParseHandler::load_buffer), start, end, encoding_fallback);
return *this;
}
-template<EncodingType Encoding>
-constexpr Parser<Encoding>& Parser<Encoding>::load_from_string(const std::string_view string) {
- return load_from_buffer(string.data(), string.size());
+constexpr Parser& Parser::load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback) {
+ return load_from_buffer(string.data(), string.size(), encoding_fallback);
}
-template<EncodingType Encoding>
-Parser<Encoding>& Parser<Encoding>::load_from_file(const char* path) {
+Parser& Parser::load_from_file(const char* path, std::optional<detail::Encoding> encoding_fallback) {
set_file_path(path);
// Type can be deduced??
- _run_load_func(std::mem_fn(&ParseHandler::load_file), path);
+ _run_load_func(std::mem_fn(&ParseHandler::load_file), get_file_path().data(), encoding_fallback);
return *this;
}
-template<EncodingType Encoding>
-Parser<Encoding>& Parser<Encoding>::load_from_file(const std::filesystem::path& path) {
- return load_from_file(path.string().c_str());
+Parser& Parser::load_from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback) {
+ return load_from_file(path.string().c_str(), encoding_fallback);
}
-template<EncodingType Encoding>
-bool Parser<Encoding>::parse_csv(bool handle_strings) {
+bool Parser::parse_csv(bool handle_strings) {
if (!_parse_handler->is_valid()) {
return false;
}
- std::optional<Parser<Encoding>::error_range> errors;
- // auto report_error = ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream });
- if constexpr (Encoding == EncodingType::Windows1252) {
+ std::optional<Parser::error_range> errors = [&] {
if (handle_strings)
- errors = _parse_handler->template parse<csv::grammar::windows1252::strings::SemiColonFile>();
+ return _parse_handler->template parse<csv::grammar::strings::SemiColonFile>();
else
- errors = _parse_handler->template parse<csv::grammar::windows1252::SemiColonFile>();
- } else {
- if (handle_strings)
- errors = _parse_handler->template parse<csv::grammar::utf8::strings::SemiColonFile>();
- else
- errors = _parse_handler->template parse<csv::grammar::utf8::SemiColonFile>();
- }
+ return _parse_handler->template parse<csv::grammar::SemiColonFile>();
+ }();
_has_error = _parse_handler->parse_state().logger().errored();
_has_warning = _parse_handler->parse_state().logger().warned();
if (!errors->empty()) {
+ _has_error = true;
_has_fatal_error = true;
if (&_error_stream.get() != &detail::cnull) {
print_errors_to(_error_stream);
}
return false;
}
- _lines = std::move(_parse_handler->get_lines());
return true;
}
-template<EncodingType Encoding>
-const std::vector<csv::LineObject>& Parser<Encoding>::get_lines() const {
- return _lines;
+const std::vector<csv::LineObject>& Parser::get_lines() const {
+ return _parse_handler->get_lines();
}
-template<EncodingType Encoding>
-typename Parser<Encoding>::error_range Parser<Encoding>::get_errors() const {
+typename Parser::error_range Parser::get_errors() const {
return _parse_handler->parse_state().logger().get_errors();
}
-template<EncodingType Encoding>
-const FilePosition Parser<Encoding>::get_error_position(const error::Error* error) const {
+const FilePosition Parser::get_error_position(const error::Error* error) const {
if (!error || !error->is_linked_in_tree()) {
return {};
}
@@ -206,18 +194,27 @@ const FilePosition Parser<Encoding>::get_error_position(const error::Error* erro
return {};
}
- auto loc_begin = lexy::get_input_location(_parse_handler->buffer(), err_location.begin());
- FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() };
- if (err_location.begin() < err_location.end()) {
- auto loc_end = lexy::get_input_location(_parse_handler->buffer(), err_location.end(), loc_begin.anchor());
- result.end_line = loc_end.line_nr();
- result.end_column = loc_end.column_nr();
- }
- return result;
+// TODO: Remove reinterpret_cast
+// WARNING: This almost certainly breaks on utf16 and utf32 encodings, luckily we don't parse in that format
+// This is purely to silence the node_location errors because char8_t is useless
+#define REINTERPRET_IT(IT) reinterpret_cast<const std::decay_t<decltype(buffer)>::encoding::char_type*>((IT))
+
+ return _parse_handler->parse_state().file().visit_buffer(
+ [&](auto&& buffer) -> FilePosition {
+ auto loc_begin = lexy::get_input_location(buffer, REINTERPRET_IT(err_location.begin()));
+ FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() };
+ if (err_location.begin() < err_location.end()) {
+ auto loc_end = lexy::get_input_location(buffer, REINTERPRET_IT(err_location.end()), loc_begin.anchor());
+ result.end_line = loc_end.line_nr();
+ result.end_column = loc_end.column_nr();
+ }
+ return result;
+ });
+
+#undef REINTERPRET_IT
}
-template<EncodingType Encoding>
-void Parser<Encoding>::print_errors_to(std::basic_ostream<char>& stream) const {
+void Parser::print_errors_to(std::basic_ostream<char>& stream) const {
auto errors = get_errors();
if (errors.empty()) return;
for (const auto error : errors) {
@@ -226,19 +223,9 @@ void Parser<Encoding>::print_errors_to(std::basic_ostream<char>& stream) const {
[&](const error::BufferError* buffer_error) {
stream << "buffer error: " << buffer_error->message() << '\n';
},
- [&](const error::ParseError* parse_error) {
- auto position = get_error_position(parse_error);
- std::string pos_str = fmt::format(":{}:{}: ", position.start_line, position.start_column);
- stream << _file_path << pos_str << "parse error for '" << parse_error->production_name() << "': " << parse_error->message() << '\n';
- },
- [&](dryad::child_visitor<error::ErrorKind> visitor, const error::Semantic* semantic) {
- auto position = get_error_position(semantic);
- std::string pos_str = ": ";
- if (!position.is_empty()) {
- pos_str = fmt::format(":{}:{}: ", position.start_line, position.start_column);
- }
- stream << _file_path << pos_str << semantic->message() << '\n';
- auto annotations = semantic->annotations();
+ [&](dryad::child_visitor<error::ErrorKind> visitor, const error::AnnotatedError* annotated_error) {
+ stream << annotated_error->message() << '\n';
+ auto annotations = annotated_error->annotations();
for (auto annotation : annotations) {
visitor(annotation);
}
@@ -250,7 +237,4 @@ void Parser<Encoding>::print_errors_to(std::basic_ostream<char>& stream) const {
stream << secondary->message() << '\n';
});
}
-}
-
-template class ovdl::csv::Parser<EncodingType::Windows1252>;
-template class ovdl::csv::Parser<EncodingType::Utf8>; \ No newline at end of file
+} \ No newline at end of file