aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/headless/main.cpp84
-rw-r--r--src/openvic-dataloader/csv/CsvGrammar.hpp187
-rw-r--r--src/openvic-dataloader/csv/Parser.cpp86
-rw-r--r--src/openvic-dataloader/detail/BasicBufferHandler.hpp2
4 files changed, 309 insertions, 50 deletions
diff --git a/src/headless/main.cpp b/src/headless/main.cpp
index 94d6b8d..c3ca8c2 100644
--- a/src/headless/main.cpp
+++ b/src/headless/main.cpp
@@ -1,17 +1,71 @@
+#include <algorithm>
+#include <cctype>
#include <cstdio>
+#include <cstdlib>
#include <iostream>
+#include <iterator>
#include <string>
+#include <string_view>
#include <vector>
+#include <openvic-dataloader/ParseWarning.hpp>
+#include <openvic-dataloader/csv/LineObject.hpp>
+#include <openvic-dataloader/csv/Parser.hpp>
+#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp>
#include <openvic-dataloader/v2script/Parser.hpp>
-int main(int argc, char** argv) {
- if (argc < 2) {
- std::fprintf(stderr, "usage: %s <filename>", argv[0]);
+std::string_view trim(std::string_view str) {
+ std::string_view::iterator begin = str.begin();
+ std::string_view::iterator end = str.end();
+ for (;; begin++) {
+ if (begin == end) return std::string_view();
+ if (!std::isspace(*begin)) break;
+ }
+ end--;
+ for (;; end--) {
+ if (end == begin) return std::string_view();
+ if (!std::isspace(*end)) break;
+ }
+ return std::string_view(&*begin, std::distance(begin, end));
+}
+
+bool insenitive_trim_eq(std::string_view lhs, std::string_view rhs) {
+ lhs = trim(lhs);
+ rhs = trim(rhs);
+ return std::equal(
+ lhs.begin(), lhs.end(),
+ rhs.begin(), rhs.end(),
+ [](char a, char b) { return std::tolower(a) == std::tolower(b); });
+}
+
+template<ovdl::csv::EncodingType Encoding>
+int print_csv(const std::string_view path) {
+ auto parser = ovdl::csv::Parser<Encoding>::from_file(path);
+ if (parser.has_error()) {
return 1;
}
- auto parser = ovdl::v2script::Parser::from_file(argv[1]);
+ parser.parse_csv();
+ if (parser.has_error()) {
+ return 2;
+ }
+
+ if (parser.has_warning()) {
+ for (auto& warning : parser.get_warnings()) {
+ std::cerr << "Warning: " << warning.message << std::endl;
+ }
+ }
+
+ std::cout << "lines:\t\t" << parser.get_lines().size() << std::endl;
+ for (const auto& line : parser.get_lines()) {
+ std::cout << "line size:\t" << line.value_count() << std::endl;
+ std::cout << "values:\t\t" << line << std::endl;
+ }
+ return EXIT_SUCCESS;
+}
+
+int print_v2script_simple(const std::string_view path) {
+ auto parser = ovdl::v2script::Parser::from_file(path);
if (parser.has_error()) {
return 1;
}
@@ -28,6 +82,28 @@ int main(int argc, char** argv) {
}
std::cout << parser.get_file_node() << std::endl;
+ return EXIT_SUCCESS;
+}
+
+int main(int argc, char** argv) {
+ switch (argc) {
+ case 2:
+ return print_v2script_simple(argv[1]);
+ case 4:
+ if (insenitive_trim_eq(argv[1], "csv") && insenitive_trim_eq(argv[2], "utf"))
+ return print_csv<ovdl::csv::EncodingType::Utf8>(argv[3]);
+ goto default_jump;
+ case 3:
+ if (insenitive_trim_eq(argv[1], "csv"))
+ return print_csv<ovdl::csv::EncodingType::Windows1252>(argv[2]);
+ [[fallthrough]];
+ default:
+ default_jump:
+ std::fprintf(stderr, "usage: %s <filename>\n", argv[0]);
+ std::fprintf(stderr, "usage: %s csv <filename>\n", argv[0]);
+ std::fprintf(stderr, "usage: %s csv utf <filename>", argv[0]);
+ return EXIT_FAILURE;
+ }
return 0;
} \ No newline at end of file
diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp
index edce97b..4765112 100644
--- a/src/openvic-dataloader/csv/CsvGrammar.hpp
+++ b/src/openvic-dataloader/csv/CsvGrammar.hpp
@@ -11,8 +11,141 @@
#include <lexy/callback.hpp>
#include <lexy/dsl.hpp>
+#include "detail/LexyLitRange.hpp"
+
// Grammar Definitions //
-namespace ovdl::csv::grammar {
+namespace ovdl::csv::grammar::windows1252 {
+ constexpr auto character = detail::lexydsl::make_range<0x01, 0xFF>();
+ constexpr auto control =
+ lexy::dsl::ascii::control /
+ lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> /
+ lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>;
+
+ struct StringValue {
+ static constexpr auto escaped_symbols = lexy::symbol_table<char> //
+ .map<'"'>('"')
+ .map<'\''>('\'')
+ .map<'\\'>('\\')
+ .map<'/'>('/')
+ .map<'b'>('\b')
+ .map<'f'>('\f')
+ .map<'n'>('\n')
+ .map<'r'>('\r')
+ .map<'t'>('\t');
+ /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from ""
+ // static constexpr auto escaped_quote = lexy::symbol_table<char> //
+ // .map<'"'>('"');
+ static constexpr auto rule = [] {
+ // Arbitrary code points
+ auto c = character - (control / lexy::dsl::lit_b<'"'>);
+
+ auto back_escape = lexy::dsl::backslash_escape //
+ .symbol<escaped_symbols>();
+
+ // auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
+ // .symbol<escaped_quote>();
+
+ return lexy::dsl::quoted(c, back_escape);
+ }();
+
+ static constexpr auto value = lexy::as_string<std::string>;
+ };
+
+ template<char Sep>
+ struct PlainValue {
+ static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b<Sep> / lexy::dsl::ascii::newline));
+ static constexpr auto value = lexy::as_string<std::string>;
+ };
+
+ template<char Sep>
+ struct Value {
+ static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>;
+ static constexpr auto value = lexy::forward<std::string>;
+ };
+
+ template<char Sep>
+ struct SepConst {
+ static constexpr auto rule = lexy::dsl::lit_b<Sep>;
+ static constexpr auto value = lexy::constant(1);
+ };
+
+ template<char Sep>
+ struct Seperator {
+ static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Sep>>);
+ static constexpr auto value = lexy::count;
+ };
+
+ template<char Sep>
+ struct LineEnd {
+ static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Sep>>));
+ static constexpr auto value = lexy::fold_inplace<csv::LineObject>(
+ std::initializer_list<csv::LineObject::value_type> {},
+ [](csv::LineObject& result, auto&& arg) {
+ if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
+ // Count seperators, adds to previous value, making it a position
+ using position_type = csv::LineObject::position_type;
+ result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
+ } else {
+ if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
+ else {
+ auto& [pos, value] = result.back();
+ value = arg;
+ }
+ }
+ });
+ };
+
+ template<char Sep>
+ struct Line {
+
+ static constexpr auto suffix_setter(csv::LineObject& line) {
+ auto& [position, value] = line.back();
+ if (value.empty()) {
+ line.set_suffix_end(position);
+ line.pop_back();
+ } else {
+ line.set_suffix_end(position + 1);
+ }
+ };
+
+ static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<Seperator<Sep>> >> lexy::dsl::p<LineEnd<Sep>>;
+ static constexpr auto value =
+ lexy::callback<csv::LineObject>(
+ [](csv::LineObject&& line) {
+ suffix_setter(line);
+ return LEXY_MOV(line);
+ },
+ [](std::size_t prefix_count, csv::LineObject&& line) {
+ line.set_prefix_end(prefix_count);
+ // position needs to be adjusted to prefix
+ for (auto& [position, value] : line) {
+ position += prefix_count;
+ }
+ suffix_setter(line);
+ return LEXY_MOV(line);
+ });
+ };
+
+ template<char Sep>
+ struct File {
+ static constexpr auto rule =
+ lexy::dsl::whitespace(lexy::dsl::newline) +
+ lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p<Line<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::eol)));
+
+ static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>;
+ };
+
+ using CommaFile = File<','>;
+ using ColonFile = File<':'>;
+ using SemiColonFile = File<';'>;
+ using TabFile = File<'\t'>;
+ using BarFile = File<'|'>;
+}
+
+namespace ovdl::csv::grammar::utf8 {
+ constexpr auto character = lexy::dsl::unicode::character;
+ constexpr auto control = lexy::dsl::unicode::control;
+
struct StringValue {
static constexpr auto escaped_symbols = lexy::symbol_table<char> //
.map<'"'>('"')
@@ -25,52 +158,58 @@ namespace ovdl::csv::grammar {
.map<'r'>('\r')
.map<'t'>('\t');
/// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from ""
- static constexpr auto escaped_quote = lexy::symbol_table<char> //
- .map<'"'>('"');
+ // static constexpr auto escaped_quote = lexy::symbol_table<char> //
+ // .map<'"'>('"');
static constexpr auto rule = [] {
// Arbitrary code points
- auto c = -lexy::dsl::lit_c<'"'>;
+ auto c = character - (control / lexy::dsl::lit_b<'"'>);
auto back_escape = lexy::dsl::backslash_escape //
.symbol<escaped_symbols>();
- auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
- .symbol<escaped_quote>();
+ // auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) //
+ // .symbol<escaped_quote>();
- return lexy::dsl::quoted(c, back_escape, quote_escape);
+ return lexy::dsl::quoted(c, back_escape);
}();
static constexpr auto value = lexy::as_string<std::string>;
};
- template<auto Sep>
+ template<char Sep>
struct PlainValue {
- static constexpr auto rule = lexy::dsl::identifier(-(Sep / lexy::dsl::lit_c<'\n'>));
+ static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b<Sep> / lexy::dsl::ascii::newline));
static constexpr auto value = lexy::as_string<std::string>;
};
- template<auto Sep>
+ template<char Sep>
struct Value {
static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>;
static constexpr auto value = lexy::forward<std::string>;
};
- template<auto Sep>
- struct SeperatorCount {
- static constexpr auto rule = lexy::dsl::list(Sep);
+ template<char Sep>
+ struct SepConst {
+ static constexpr auto rule = lexy::dsl::lit_b<Sep>;
+ static constexpr auto value = lexy::constant(1);
+ };
+
+ template<char Sep>
+ struct Seperator {
+ static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Sep>>);
static constexpr auto value = lexy::count;
};
- template<auto Sep>
+ template<char Sep>
struct LineEnd {
- static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<SeperatorCount<Sep>>));
+ static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Sep>>));
static constexpr auto value = lexy::fold_inplace<csv::LineObject>(
std::initializer_list<csv::LineObject::value_type> {},
[](csv::LineObject& result, auto&& arg) {
if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) {
// Count seperators, adds to previous value, making it a position
using position_type = csv::LineObject::position_type;
- result.emplace_back(static_cast<position_type>(arg + std::get<0>(result.back())), "");
+ result.emplace_back(static_cast<position_type>(arg + result.back().first), "");
} else {
if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg));
else {
@@ -81,7 +220,7 @@ namespace ovdl::csv::grammar {
});
};
- template<auto Sep>
+ template<char Sep>
struct Line {
static constexpr auto suffix_setter(csv::LineObject& line) {
@@ -94,7 +233,7 @@ namespace ovdl::csv::grammar {
}
};
- static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<SeperatorCount<Sep>> >> lexy::dsl::p<LineEnd<Sep>>;
+ static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<Seperator<Sep>> >> lexy::dsl::p<LineEnd<Sep>>;
static constexpr auto value =
lexy::callback<csv::LineObject>(
[](csv::LineObject&& line) {
@@ -112,7 +251,7 @@ namespace ovdl::csv::grammar {
});
};
- template<auto Sep>
+ template<char Sep>
struct File {
static constexpr auto rule =
lexy::dsl::whitespace(lexy::dsl::newline) +
@@ -121,9 +260,9 @@ namespace ovdl::csv::grammar {
static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>;
};
- using CommaFile = File<lexy::dsl::lit_c<','>>;
- using ColonFile = File<lexy::dsl::lit_c<':'>>;
- using SemiColonFile = File<lexy::dsl::lit_c<';'>>;
- using TabFile = File<lexy::dsl::lit_c<'\t'>>;
- using BarFile = File<lexy::dsl::lit_c<'|'>>;
+ using CommaFile = File<','>;
+ using ColonFile = File<':'>;
+ using SemiColonFile = File<';'>;
+ using TabFile = File<'\t'>;
+ using BarFile = File<'|'>;
} \ No newline at end of file
diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp
index 45f3142..86ad02e 100644
--- a/src/openvic-dataloader/csv/Parser.cpp
+++ b/src/openvic-dataloader/csv/Parser.cpp
@@ -3,6 +3,7 @@
#include <openvic-dataloader/csv/LineObject.hpp>
#include <openvic-dataloader/csv/Parser.hpp>
+#include <openvic-dataloader/detail/ClassExport.hpp>
#include <lexy/action/parse.hpp>
#include <lexy/encoding.hpp>
@@ -20,11 +21,26 @@ using namespace ovdl::csv;
/// BufferHandler ///
-class Parser::BufferHandler final : public detail::BasicBufferHandler<lexy::utf8_char_encoding> {
+template<EncodingType>
+struct LexyEncodingFrom {
+};
+
+template<>
+struct LexyEncodingFrom<EncodingType::Windows1252> {
+ using encoding = lexy::default_encoding;
+};
+
+template<>
+struct LexyEncodingFrom<EncodingType::Utf8> {
+ using encoding = lexy::utf8_char_encoding;
+};
+
+template<EncodingType Encoding>
+class Parser<Encoding>::BufferHandler final : public detail::BasicBufferHandler<typename LexyEncodingFrom<Encoding>::encoding> {
public:
template<typename Node, typename ErrorCallback>
std::optional<std::vector<ParseError>> parse(const ErrorCallback& callback) {
- auto result = lexy::parse<Node>(_buffer, callback);
+ auto result = lexy::parse<Node>(this->_buffer, callback);
if (!result) {
return result.errors();
}
@@ -42,36 +58,45 @@ private:
/// BufferHandler ///
-Parser::Parser()
+template<EncodingType Encoding>
+Parser<Encoding>::Parser()
: _buffer_handler(std::make_unique<BufferHandler>()) {
set_error_log_to_stderr();
}
-Parser::Parser(Parser&&) = default;
-Parser& Parser::operator=(Parser&&) = default;
-Parser::~Parser() = default;
+template<EncodingType Encoding>
+Parser<Encoding>::Parser(Parser&&) = default;
+template<EncodingType Encoding>
+Parser<Encoding>& Parser<Encoding>::operator=(Parser&&) = default;
+template<EncodingType Encoding>
+Parser<Encoding>::~Parser() = default;
-Parser Parser::from_buffer(const char* data, std::size_t size) {
+template<EncodingType Encoding>
+Parser<Encoding> Parser<Encoding>::from_buffer(const char* data, std::size_t size) {
Parser result;
return std::move(result.load_from_buffer(data, size));
}
-Parser Parser::from_buffer(const char* start, const char* end) {
+template<EncodingType Encoding>
+Parser<Encoding> Parser<Encoding>::from_buffer(const char* start, const char* end) {
Parser result;
return std::move(result.load_from_buffer(start, end));
}
-Parser Parser::from_string(const std::string_view string) {
+template<EncodingType Encoding>
+Parser<Encoding> Parser<Encoding>::from_string(const std::string_view string) {
Parser result;
return std::move(result.load_from_string(string));
}
-Parser Parser::from_file(const char* path) {
+template<EncodingType Encoding>
+Parser<Encoding> Parser<Encoding>::from_file(const char* path) {
Parser result;
return std::move(result.load_from_file(path));
}
-Parser Parser::from_file(const std::filesystem::path& path) {
+template<EncodingType Encoding>
+Parser<Encoding> Parser<Encoding>::from_file(const std::filesystem::path& path) {
Parser result;
return std::move(result.load_from_file(path));
}
@@ -89,8 +114,9 @@ Parser Parser::from_file(const std::filesystem::path& path) {
/// @param func
/// @param args
///
+template<EncodingType Encoding>
template<typename... Args>
-constexpr void Parser::_run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args) {
+constexpr void Parser<Encoding>::_run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args) {
_warnings.clear();
_errors.clear();
_has_fatal_error = false;
@@ -101,43 +127,55 @@ constexpr void Parser::_run_load_func(detail::LoadCallback<BufferHandler, Args..
}
}
-constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size) {
+template<EncodingType Encoding>
+constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* data, std::size_t size) {
// Type can't be deduced?
_run_load_func(std::mem_fn(&BufferHandler::load_buffer_size), data, size);
return *this;
}
-constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) {
+template<EncodingType Encoding>
+constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* start, const char* end) {
// Type can't be deduced?
_run_load_func(std::mem_fn(&BufferHandler::load_buffer), start, end);
return *this;
}
-constexpr Parser& Parser::load_from_string(const std::string_view string) {
+template<EncodingType Encoding>
+constexpr Parser<Encoding>& Parser<Encoding>::load_from_string(const std::string_view string) {
return load_from_buffer(string.data(), string.size());
}
-constexpr Parser& Parser::load_from_file(const char* path) {
+template<EncodingType Encoding>
+constexpr Parser<Encoding>& Parser<Encoding>::load_from_file(const char* path) {
_file_path = path;
// Type can be deduced??
_run_load_func(std::mem_fn(&BufferHandler::load_file), path);
return *this;
}
-Parser& Parser::load_from_file(const std::filesystem::path& path) {
+template<EncodingType Encoding>
+Parser<Encoding>& Parser<Encoding>::load_from_file(const std::filesystem::path& path) {
return load_from_file(path.string().c_str());
}
-constexpr Parser& Parser::load_from_file(const detail::Has_c_str auto& path) {
+template<EncodingType Encoding>
+constexpr Parser<Encoding>& Parser<Encoding>::load_from_file(const detail::Has_c_str auto& path) {
return load_from_file(path.c_str());
}
-bool Parser::parse_csv() {
+template<EncodingType Encoding>
+bool Parser<Encoding>::parse_csv() {
if (!_buffer_handler->is_valid()) {
return false;
}
- auto errors = _buffer_handler->parse<csv::grammar::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }));
+ std::optional<std::vector<ParseError>> errors;
+ if constexpr (Encoding == EncodingType::Windows1252) {
+ errors = _buffer_handler->template parse<csv::grammar::windows1252::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }));
+ } else {
+ errors = _buffer_handler->template parse<csv::grammar::utf8::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream }));
+ }
if (errors) {
_errors.reserve(errors->size());
for (auto& err : errors.value()) {
@@ -150,6 +188,10 @@ bool Parser::parse_csv() {
return true;
}
-const std::vector<csv::LineObject>& Parser::get_lines() const {
+template<EncodingType Encoding>
+const std::vector<csv::LineObject>& Parser<Encoding>::get_lines() const {
return _lines;
-} \ No newline at end of file
+}
+
+template class ovdl::csv::Parser<EncodingType::Windows1252>;
+template class ovdl::csv::Parser<EncodingType::Utf8>; \ No newline at end of file
diff --git a/src/openvic-dataloader/detail/BasicBufferHandler.hpp b/src/openvic-dataloader/detail/BasicBufferHandler.hpp
index 75ea8ee..5f4f82d 100644
--- a/src/openvic-dataloader/detail/BasicBufferHandler.hpp
+++ b/src/openvic-dataloader/detail/BasicBufferHandler.hpp
@@ -15,6 +15,8 @@ namespace ovdl::detail {
template<typename Encoding = lexy::default_encoding, typename MemoryResource = void>
class BasicBufferHandler {
public:
+ using encoding_type = Encoding;
+
OVDL_OPTIONAL_CONSTEXPR bool is_valid() const {
return _buffer.size() != 0;
}
generated by our cgit fork (git 2.39.0) at 2024-01-23 02:24:14 +0100