aboutsummaryrefslogtreecommitdiff
path: root/src/openvic-dataloader/csv/Parser.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/openvic-dataloader/csv/Parser.cpp')
-rw-r--r--src/openvic-dataloader/csv/Parser.cpp182
1 files changed, 83 insertions, 99 deletions
diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp
index 361f6ad..5dbee32 100644
--- a/src/openvic-dataloader/csv/Parser.cpp
+++ b/src/openvic-dataloader/csv/Parser.cpp
@@ -1,11 +1,14 @@
+#include <iostream>
+#include <optional>
+#include <type_traits>
#include <vector>
-#include <openvic-dataloader/File.hpp>
+#include <openvic-dataloader/NodeLocation.hpp>
#include <openvic-dataloader/csv/LineObject.hpp>
#include <openvic-dataloader/csv/Parser.hpp>
-#include <openvic-dataloader/detail/LexyReportError.hpp>
+#include <openvic-dataloader/detail/Encoding.hpp>
#include <openvic-dataloader/detail/OStreamOutputIterator.hpp>
-#include <openvic-dataloader/detail/utility/Utility.hpp>
+#include <openvic-dataloader/detail/Utility.hpp>
#include <lexy/action/parse.hpp>
#include <lexy/encoding.hpp>
@@ -22,15 +25,27 @@ using namespace ovdl::csv;
/// ParseHandler ///
-template<EncodingType Encoding>
-struct Parser<Encoding>::ParseHandler final : detail::BasicFileParseHandler<CsvParseState<Encoding>> {
+struct Parser::ParseHandler final : detail::BasicFileParseHandler<CsvParseState> {
template<typename Node>
std::optional<DiagnosticLogger::error_range> parse() {
- auto result = lexy::parse<Node>(this->buffer(), *this->_parse_state, this->_parse_state->logger().error_callback());
+ auto result = [&] {
+ switch (parse_state().encoding()) {
+ using enum detail::Encoding;
+ case Ascii:
+ case Utf8:
+ return lexy::parse<Node>(buffer<lexy::utf8_char_encoding>(), parse_state(), parse_state().logger().error_callback());
+ case Unknown:
+ case Windows1251:
+ case Windows1252:
+ return lexy::parse<Node>(buffer<lexy::default_encoding>(), parse_state(), parse_state().logger().error_callback());
+ default:
+ ovdl::detail::unreachable();
+ }
+ }();
if (!result) {
- return this->_parse_state->logger().get_errors();
+ return this->parse_state().logger().get_errors();
}
- _lines = std::move(result.value());
+ _lines = LEXY_MOV(result).value();
return std::nullopt;
}
@@ -42,55 +57,45 @@ private:
std::vector<csv::LineObject> _lines;
};
-/// BufferHandler ///
+/// ParserHandler ///
-template<EncodingType Encoding>
-Parser<Encoding>::Parser()
+Parser::Parser()
: _parse_handler(std::make_unique<ParseHandler>()) {
set_error_log_to_null();
}
-template<EncodingType Encoding>
-Parser<Encoding>::Parser(std::basic_ostream<char>& error_stream)
+Parser::Parser(std::basic_ostream<char>& error_stream)
: _parse_handler(std::make_unique<ParseHandler>()) {
set_error_log_to(error_stream);
}
-template<EncodingType Encoding>
-Parser<Encoding>::Parser(Parser&&) = default;
-template<EncodingType Encoding>
-Parser<Encoding>& Parser<Encoding>::operator=(Parser&&) = default;
-template<EncodingType Encoding>
-Parser<Encoding>::~Parser() = default;
+Parser::Parser(Parser&&) = default;
+Parser& Parser::operator=(Parser&&) = default;
+Parser::~Parser() = default;
-template<EncodingType Encoding>
-Parser<Encoding> Parser<Encoding>::from_buffer(const char* data, std::size_t size) {
+Parser Parser::from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback) {
Parser result;
- return std::move(result.load_from_buffer(data, size));
+ return std::move(result.load_from_buffer(data, size, encoding_fallback));
}
-template<EncodingType Encoding>
-Parser<Encoding> Parser<Encoding>::from_buffer(const char* start, const char* end) {
+Parser Parser::from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback) {
Parser result;
- return std::move(result.load_from_buffer(start, end));
+ return std::move(result.load_from_buffer(start, end, encoding_fallback));
}
-template<EncodingType Encoding>
-Parser<Encoding> Parser<Encoding>::from_string(const std::string_view string) {
+Parser Parser::from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback) {
Parser result;
- return std::move(result.load_from_string(string));
+ return std::move(result.load_from_string(string, encoding_fallback));
}
-template<EncodingType Encoding>
-Parser<Encoding> Parser<Encoding>::from_file(const char* path) {
+Parser Parser::from_file(const char* path, std::optional<detail::Encoding> encoding_fallback) {
Parser result;
- return std::move(result.load_from_file(path));
+ return std::move(result.load_from_file(path, encoding_fallback));
}
-template<EncodingType Encoding>
-Parser<Encoding> Parser<Encoding>::from_file(const std::filesystem::path& path) {
+Parser Parser::from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback) {
Parser result;
- return std::move(result.load_from_file(path));
+ return std::move(result.load_from_file(path, encoding_fallback));
}
///
@@ -106,9 +111,8 @@ Parser<Encoding> Parser<Encoding>::from_file(const std::filesystem::path& path)
/// @param func
/// @param args
///
-template<EncodingType Encoding>
template<typename... Args>
-constexpr void Parser<Encoding>::_run_load_func(detail::LoadCallback<ParseHandler, Args...> auto func, Args... args) {
+constexpr void Parser::_run_load_func(detail::LoadCallback<ParseHandler, Args...> auto func, Args... args) {
_has_fatal_error = false;
auto error = func(_parse_handler.get(), std::forward<Args>(args)...);
auto error_message = _parse_handler->make_error_from(error);
@@ -122,82 +126,66 @@ constexpr void Parser<Encoding>::_run_load_func(detail::LoadCallback<ParseHandle
}
}
-template<EncodingType Encoding>
-constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* data, std::size_t size) {
+constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback) {
// Type can't be deduced?
- _run_load_func(std::mem_fn(&ParseHandler::load_buffer_size), data, size);
+ _run_load_func(std::mem_fn(&ParseHandler::load_buffer_size), data, size, encoding_fallback);
return *this;
}
-template<EncodingType Encoding>
-constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* start, const char* end) {
+constexpr Parser& Parser::load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback) {
// Type can't be deduced?
- _run_load_func(std::mem_fn(&ParseHandler::load_buffer), start, end);
+ _run_load_func(std::mem_fn(&ParseHandler::load_buffer), start, end, encoding_fallback);
return *this;
}
-template<EncodingType Encoding>
-constexpr Parser<Encoding>& Parser<Encoding>::load_from_string(const std::string_view string) {
- return load_from_buffer(string.data(), string.size());
+constexpr Parser& Parser::load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback) {
+ return load_from_buffer(string.data(), string.size(), encoding_fallback);
}
-template<EncodingType Encoding>
-Parser<Encoding>& Parser<Encoding>::load_from_file(const char* path) {
+Parser& Parser::load_from_file(const char* path, std::optional<detail::Encoding> encoding_fallback) {
set_file_path(path);
// Type can be deduced??
- _run_load_func(std::mem_fn(&ParseHandler::load_file), path);
+ _run_load_func(std::mem_fn(&ParseHandler::load_file), get_file_path().data(), encoding_fallback);
return *this;
}
-template<EncodingType Encoding>
-Parser<Encoding>& Parser<Encoding>::load_from_file(const std::filesystem::path& path) {
- return load_from_file(path.string().c_str());
+Parser& Parser::load_from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback) {
+ return load_from_file(path.string().c_str(), encoding_fallback);
}
-template<EncodingType Encoding>
-bool Parser<Encoding>::parse_csv(bool handle_strings) {
+bool Parser::parse_csv(bool handle_strings) {
if (!_parse_handler->is_valid()) {
return false;
}
- std::optional<Parser<Encoding>::error_range> errors;
- // auto report_error = ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream });
- if constexpr (Encoding == EncodingType::Windows1252) {
+ std::optional<Parser::error_range> errors = [&] {
if (handle_strings)
- errors = _parse_handler->template parse<csv::grammar::windows1252::strings::SemiColonFile>();
+ return _parse_handler->template parse<csv::grammar::strings::SemiColonFile>();
else
- errors = _parse_handler->template parse<csv::grammar::windows1252::SemiColonFile>();
- } else {
- if (handle_strings)
- errors = _parse_handler->template parse<csv::grammar::utf8::strings::SemiColonFile>();
- else
- errors = _parse_handler->template parse<csv::grammar::utf8::SemiColonFile>();
- }
+ return _parse_handler->template parse<csv::grammar::SemiColonFile>();
+ }();
_has_error = _parse_handler->parse_state().logger().errored();
_has_warning = _parse_handler->parse_state().logger().warned();
if (!errors->empty()) {
+ _has_error = true;
_has_fatal_error = true;
if (&_error_stream.get() != &detail::cnull) {
print_errors_to(_error_stream);
}
return false;
}
- _lines = std::move(_parse_handler->get_lines());
return true;
}
-template<EncodingType Encoding>
-const std::vector<csv::LineObject>& Parser<Encoding>::get_lines() const {
- return _lines;
+const std::vector<csv::LineObject>& Parser::get_lines() const {
+ return _parse_handler->get_lines();
}
-template<EncodingType Encoding>
-typename Parser<Encoding>::error_range Parser<Encoding>::get_errors() const {
+typename Parser::error_range Parser::get_errors() const {
return _parse_handler->parse_state().logger().get_errors();
}
-template<EncodingType Encoding>
-const FilePosition Parser<Encoding>::get_error_position(const error::Error* error) const {
+const FilePosition Parser::get_error_position(const error::Error* error) const {
if (!error || !error->is_linked_in_tree()) {
return {};
}
@@ -206,18 +194,27 @@ const FilePosition Parser<Encoding>::get_error_position(const error::Error* erro
return {};
}
- auto loc_begin = lexy::get_input_location(_parse_handler->buffer(), err_location.begin());
- FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() };
- if (err_location.begin() < err_location.end()) {
- auto loc_end = lexy::get_input_location(_parse_handler->buffer(), err_location.end(), loc_begin.anchor());
- result.end_line = loc_end.line_nr();
- result.end_column = loc_end.column_nr();
- }
- return result;
+// TODO: Remove reinterpret_cast
+// WARNING: This almost certainly breaks on utf16 and utf32 encodings, luckily we don't parse in that format
+// This is purely to silence the node_location errors because char8_t is useless
+#define REINTERPRET_IT(IT) reinterpret_cast<const std::decay_t<decltype(buffer)>::encoding::char_type*>((IT))
+
+ return _parse_handler->parse_state().file().visit_buffer(
+ [&](auto&& buffer) -> FilePosition {
+ auto loc_begin = lexy::get_input_location(buffer, REINTERPRET_IT(err_location.begin()));
+ FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() };
+ if (err_location.begin() < err_location.end()) {
+ auto loc_end = lexy::get_input_location(buffer, REINTERPRET_IT(err_location.end()), loc_begin.anchor());
+ result.end_line = loc_end.line_nr();
+ result.end_column = loc_end.column_nr();
+ }
+ return result;
+ });
+
+#undef REINTERPRET_IT
}
-template<EncodingType Encoding>
-void Parser<Encoding>::print_errors_to(std::basic_ostream<char>& stream) const {
+void Parser::print_errors_to(std::basic_ostream<char>& stream) const {
auto errors = get_errors();
if (errors.empty()) return;
for (const auto error : errors) {
@@ -226,19 +223,9 @@ void Parser<Encoding>::print_errors_to(std::basic_ostream<char>& stream) const {
[&](const error::BufferError* buffer_error) {
stream << "buffer error: " << buffer_error->message() << '\n';
},
- [&](const error::ParseError* parse_error) {
- auto position = get_error_position(parse_error);
- std::string pos_str = fmt::format(":{}:{}: ", position.start_line, position.start_column);
- stream << _file_path << pos_str << "parse error for '" << parse_error->production_name() << "': " << parse_error->message() << '\n';
- },
- [&](dryad::child_visitor<error::ErrorKind> visitor, const error::Semantic* semantic) {
- auto position = get_error_position(semantic);
- std::string pos_str = ": ";
- if (!position.is_empty()) {
- pos_str = fmt::format(":{}:{}: ", position.start_line, position.start_column);
- }
- stream << _file_path << pos_str << semantic->message() << '\n';
- auto annotations = semantic->annotations();
+ [&](dryad::child_visitor<error::ErrorKind> visitor, const error::AnnotatedError* annotated_error) {
+ stream << annotated_error->message() << '\n';
+ auto annotations = annotated_error->annotations();
for (auto annotation : annotations) {
visitor(annotation);
}
@@ -250,7 +237,4 @@ void Parser<Encoding>::print_errors_to(std::basic_ostream<char>& stream) const {
stream << secondary->message() << '\n';
});
}
-}
-
-template class ovdl::csv::Parser<EncodingType::Windows1252>;
-template class ovdl::csv::Parser<EncodingType::Utf8>; \ No newline at end of file
+} \ No newline at end of file