Merge pull request #7 from OpenVicProject/dataloading-event-and-functions

2 files changed, 280 insertions, 0 deletions

diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp
new file mode 100644
index 0000000..edce97b
--- /dev/null
+++ b/src/openvic-dataloader/csv/CsvGrammar.hpp
@@ -0,0 +1,129 @@
+#pragma once
+
+#include <initializer_list>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <vector>
+
+#include <openvic-dataloader/csv/LineObject.hpp>
+
+#include <lexy/callback.hpp>
+#include <lexy/dsl.hpp>
+
+// Grammar Definitions //
+namespace ovdl::csv::grammar {
+	struct StringValue {
+		static constexpr auto escaped_symbols = lexy::symbol_table<char> //
+													.map<'"'>('"')
+													.map<'\''>('\'')
+													.map<'\\'>('\\')
+													.map<'/'>('/')
+													.map<'b'>('\b')
+													.map<'f'>('\f')
+													.map<'n'>('\n')
+													.map<'r'>('\r')
+													.map<'t'>('\t');
+		/// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from ""
+		static constexpr auto escaped_quote = lexy::symbol_table<char> //
+												  .map<'"'>('"');
+		static constexpr auto rule = [] {
+			// Arbitrary code points
+			auto c = -lexy::dsl::lit_c<'"'>;
+
+			auto back_escape = lexy::dsl::backslash_escape //
+								   .symbol<escaped_symbols>();
+
+			auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
+									.symbol<escaped_quote>();
+
+			return lexy::dsl::quoted(c, back_escape, quote_escape);
+		}();
+
+		static constexpr auto value = lexy::as_string<std::string>;
+	};
+
+	template<auto Sep>
+	struct PlainValue {
+		static constexpr auto rule = lexy::dsl::identifier(-(Sep / lexy::dsl::lit_c<'\n'>));
+		static constexpr auto value = lexy::as_string<std::string>;
+	};
+
+	template<auto Sep>
+	struct Value {
+		static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>;
+		static constexpr auto value = lexy::forward<std::string>;
+	};
+
+	template<auto Sep>
+	struct SeperatorCount {
+		static constexpr auto rule = lexy::dsl::list(Sep);
+		static constexpr auto value = lexy::count;
+	};
+
+	template<auto Sep>
+	struct LineEnd {
+		static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<SeperatorCount<Sep>>));
+		static constexpr auto value = lexy::fold_inplace<csv::LineObject>(
+			std::initializer_list<csv::LineObject::value_type> {},
+			[](csv::LineObject& result, auto&& arg) {
+				if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
+					// Count seperators, adds to previous value, making it a position
+					using position_type = csv::LineObject::position_type;
+					result.emplace_back(static_cast<position_type>(arg + std::get<0>(result.back())), "");
+				} else {
+					if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
+					else {
+						auto& [pos, value] = result.back();
+						value = arg;
+					}
+				}
+			});
+	};
+
+	template<auto Sep>
+	struct Line {
+
+		static constexpr auto suffix_setter(csv::LineObject& line) {
+			auto& [position, value] = line.back();
+			if (value.empty()) {
+				line.set_suffix_end(position);
+				line.pop_back();
+			} else {
+				line.set_suffix_end(position + 1);
+			}
+		};
+
+		static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<SeperatorCount<Sep>> >> lexy::dsl::p<LineEnd<Sep>>;
+		static constexpr auto value =
+			lexy::callback<csv::LineObject>(
+				[](csv::LineObject&& line) {
+					suffix_setter(line);
+					return LEXY_MOV(line);
+				},
+				[](std::size_t prefix_count, csv::LineObject&& line) {
+					line.set_prefix_end(prefix_count);
+					// position needs to be adjusted to prefix
+					for (auto& [position, value] : line) {
+						position += prefix_count;
+					}
+					suffix_setter(line);
+					return LEXY_MOV(line);
+				});
+	};
+
+	template<auto Sep>
+	struct File {
+		static constexpr auto rule =
+			lexy::dsl::whitespace(lexy::dsl::newline) +
+			lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p<Line<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::eol)));
+
+		static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>;
+	};
+
+	using CommaFile = File<lexy::dsl::lit_c<','>>;
+	using ColonFile = File<lexy::dsl::lit_c<':'>>;
+	using SemiColonFile = File<lexy::dsl::lit_c<';'>>;
+	using TabFile = File<lexy::dsl::lit_c<'\t'>>;
+	using BarFile = File<lexy::dsl::lit_c<'|'>>;
+}
\ No newline at end of file
diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp
new file mode 100644
index 0000000..8a99085
--- /dev/null
+++ b/src/openvic-dataloader/csv/Parser.cpp
@@ -0,0 +1,151 @@
+#include <memory>
+#include <vector>
+
+#include <openvic-dataloader/csv/LineObject.hpp>
+#include <openvic-dataloader/csv/Parser.hpp>
+
+#include <lexy/action/parse.hpp>
+#include <lexy/encoding.hpp>
+#include <lexy/input/buffer.hpp>
+#include <lexy/input/file.hpp>
+
+#include "csv/CsvGrammar.hpp"
+#include "detail/BasicBufferHandler.hpp"
+#include "detail/Errors.hpp"
+#include "detail/LexyReportError.hpp"
+#include "detail/OStreamOutputIterator.hpp"
+
+using namespace ovdl;
+using namespace ovdl::csv;
+
+///	BufferHandler ///
+
+class Parser::BufferHandler final : public detail::BasicBufferHandler<lexy::utf8_char_encoding> {
+public:
+	template<typename Node, typename ErrorCallback>
+	std::optional<std::vector<ParseError>> parse(const ErrorCallback& callback) {
+		auto result = lexy::parse<Node>(_buffer, callback);
+		if (!result) {
+			return result.errors();
+		}
+		_lines = std::move(result.value());
+		return std::nullopt;
+	}
+
+	std::vector<csv::LineObject>& get_lines() {
+		return _lines;
+	}
+
+private:
+	std::vector<csv::LineObject> _lines;
+};
+
+/// BufferHandler ///
+
+Parser::Parser()
+	: _buffer_handler(std::make_unique<BufferHandler>()) {
+	set_error_log_to_stderr();
+}
+
+Parser::Parser(Parser&&) = default;
+Parser& Parser::operator=(Parser&&) = default;
+Parser::~Parser() = default;
+
+Parser Parser::from_buffer(const char* data, std::size_t size) {
+	Parser result;
+	return std::move(result.load_from_buffer(data, size));
+}
+
+Parser Parser::from_buffer(const char* start, const char* end) {
+	Parser result;
+	return std::move(result.load_from_buffer(start, end));
+}
+
+Parser Parser::from_string(const std::string_view string) {
+	Parser result;
+	return std::move(result.load_from_string(string));
+}
+
+Parser Parser::from_file(const char* path) {
+	Parser result;
+	return std::move(result.load_from_file(path));
+}
+
+Parser Parser::from_file(const std::filesystem::path& path) {
+	Parser result;
+	return std::move(result.load_from_file(path));
+}
+
+///
+/// @brief Executes a function on _buffer_handler that is expected to load a buffer
+///
+/// Expected Use:
+/// @code {.cpp}
+///	_run_load_func(&BufferHandler::<load_function>, <arguments>);
+/// @endcode
+///
+/// @tparam Type
+/// @tparam Args
+/// @param func
+/// @param args
+///
+template<typename... Args>
+constexpr void Parser::_run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args) {
+	_warnings.clear();
+	_errors.clear();
+	_has_fatal_error = false;
+	if (auto error = func(_buffer_handler.get(), std::forward<Args>(args)...); error) {
+		_has_fatal_error = error.value().type == ParseError::Type::Fatal;
+		_errors.push_back(error.value());
+		_error_stream.get() << "Error: " << _errors.back().message << '\n';
+	}
+}
+
+constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size) {
+	// Type can't be deduced?
+	_run_load_func(std::mem_fn(&BufferHandler::load_buffer_size), data, size);
+	return *this;
+}
+
+constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) {
+	// Type can't be deduced?
+	_run_load_func(std::mem_fn(&BufferHandler::load_buffer), start, end);
+	return *this;
+}
+
+constexpr Parser& Parser::load_from_string(const std::string_view string) {
+	return load_from_buffer(string.data(), string.size());
+}
+
+constexpr Parser& Parser::load_from_file(const char* path) {
+	_file_path = path;
+	// Type can be deduced??
+	_run_load_func(std::mem_fn(&BufferHandler::load_file), path);
+	return *this;
+}
+
+Parser& Parser::load_from_file(const std::filesystem::path& path) {
+	return load_from_file(path.string().c_str());
+}
+
+constexpr Parser& Parser::load_from_file(const detail::Has_c_str auto& path) {
+	return load_from_file(path.c_str());
+}
+
+bool Parser::parse_csv() {
+	if (!_buffer_handler->is_valid()) {
+		return false;
+	}
+
+	auto errors = _buffer_handler->parse<csv::grammar::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }));
+	if (errors) {
+		_errors.reserve(errors->size());
+		for (auto& err : errors.value()) {
+			_has_fatal_error |= err.type == ParseError::Type::Fatal;
+			_errors.push_back(err);
+		}
+		return false;
+	}
+	_lines = std::move(_buffer_handler->get_lines());
+	return true;
+}
\ No newline at end of file