diff options
| -rw-r--r-- | .vscode/launch.json | 35 | ||||
| -rw-r--r-- | .vscode/tasks.json | 53 | ||||
| -rw-r--r-- | include/openvic-dataloader/csv/LineObject.hpp | 38 | ||||
| -rw-r--r-- | include/openvic-dataloader/csv/Parser.hpp | 9 | ||||
| -rw-r--r-- | include/openvic-dataloader/detail/ClassExport.hpp | 9 | ||||
| -rw-r--r-- | src/headless/main.cpp | 84 | ||||
| -rw-r--r-- | src/openvic-dataloader/csv/CsvGrammar.hpp | 187 | ||||
| -rw-r--r-- | src/openvic-dataloader/csv/Parser.cpp | 86 | ||||
| -rw-r--r-- | src/openvic-dataloader/detail/BasicBufferHandler.hpp | 2 |
9 files changed, 448 insertions, 55 deletions
diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..6ec9117 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,35 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "cppdbg", + "request": "launch", + "name": "Debug", + "program": "${workspaceFolder}/bin/openvic-dataloader.headless.linux.template_debug.dev.x86_64", + "args": ["csv", "./output.csv"], + "cwd": "${workspaceFolder}/bin", + "preLaunchTask": "dev_build", + "windows": { + "type": "cppvsdb", + "request": "launch", + "name": "Debug", + "program": "${workspaceFolder}\\bin\\openvic-dataloader.headless.windows.template_debug.dev.x86_64", + "args": ["csv", "./output.csv"], + "cwd": "${workspaceFolder}\\bin", + "preLaunchTask": "dev_build", + }, + "osx": { + "type": "cppvsdb", + "request": "launch", + "name": "Debug", + "program": "", // TODO: Mac executable? + "args": [], + "cwd": "${workspaceFolder}\\game", + "preLaunchTask": "dev_build", + } + } + ] +}
\ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..65e62ec --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,53 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "build", + "group": "build", + "type": "shell", + "command": "scons", + "args": [ + // enable for debugging with breakpoints + //"dev_build=yes", + ], + "problemMatcher": "$msCompile" + }, + { + "label": "dev_build", + "group": "build", + "type": "shell", + "command": "scons", + "args": [ + // enable for debugging with breakpoints + "dev_build=yes", + "debug_symbols=yes" + ], + "dependsOn": [ + "build" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "clean", + "group": "build", + "type": "shell", + "command": "scons", + "args": [ + "--clean" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "rebuild", + "group": "build", + "type": "shell", + "dependsOn": [ + "clean", + "build" + ], + "problemMatcher": "$msCompile" + } + ] +}
\ No newline at end of file diff --git a/include/openvic-dataloader/csv/LineObject.hpp b/include/openvic-dataloader/csv/LineObject.hpp index 8a9c2ec..c6d8e2a 100644 --- a/include/openvic-dataloader/csv/LineObject.hpp +++ b/include/openvic-dataloader/csv/LineObject.hpp @@ -1,13 +1,17 @@ #pragma once +#include <algorithm> +#include <cctype> #include <cstddef> #include <cstdint> #include <functional> #include <initializer_list> #include <optional> +#include <ostream> +#include <set> #include <string> #include <string_view> -#include <tuple> +#include <utility> #include <vector> #include <openvic-dataloader/detail/VectorConstexpr.hpp> @@ -24,13 +28,13 @@ namespace ovdl::csv { /// ;a;b;c -> 0,4+ == "" /// /// If this is incorrect, please report an issue. - class LineObject final : public std::vector<std::tuple<std::uint32_t, std::string>> { + class LineObject final : public std::vector<std::pair<std::uint32_t, std::string>> { public: // Stored position of value using position_type = std::uint32_t; // Value using inner_value_type = std::string; - using container_type = std::vector<std::tuple<position_type, inner_value_type>>; + using container_type = std::vector<std::pair<position_type, inner_value_type>>; OVDL_VECTOR_CONSTEXPR LineObject() = default; OVDL_VECTOR_CONSTEXPR LineObject(LineObject&) = default; @@ -54,7 +58,7 @@ namespace ovdl::csv { /// Special Functionality /// Retrieves value, produces "" for empty values constexpr std::string_view get_value_for(std::size_t position) const { - if (position <= _prefix_end || position >= _suffix_end) return ""; + if (position < _prefix_end || position >= _suffix_end) return ""; for (const auto& [pos, val] : *this) { if (pos == position) return val; } @@ -62,7 +66,7 @@ namespace ovdl::csv { } /// Tries to retrieve reference, produces nullopt for empty values constexpr std::optional<const std::reference_wrapper<const std::string>> try_get_string_at(std::size_t position) const { - if (position <= _prefix_end || position > _suffix_end) return std::nullopt; + if (position < _prefix_end || position >= _suffix_end) return std::nullopt; for (const auto& [pos, val] : *this) { if (pos == position) return std::cref(val); } @@ -83,4 +87,28 @@ namespace ovdl::csv { // Should be position after last value or position after last seperator position_type _suffix_end = 0; }; + + inline std::ostream& operator<<(std::ostream& stream, const LineObject& line) { + static const char SEP = ';'; + LineObject::position_type sep_index = 0; + for (const auto& [pos, val] : line) { + while (sep_index < pos) { + stream << SEP; + sep_index++; + } + if (std::any_of(val.begin(), val.end(), [](char c) { return c == SEP || std::isspace(c); })) { + stream << '"' << val << '"'; + } else { + stream << val; + } + } + return stream; + } + + inline std::ostream& operator<<(std::ostream& stream, const std::vector<LineObject>& lines) { + for (const LineObject& line : lines) { + stream << line << '\n'; + } + return stream; + } }
\ No newline at end of file diff --git a/include/openvic-dataloader/csv/Parser.hpp b/include/openvic-dataloader/csv/Parser.hpp index 0bd0670..fadaf3a 100644 --- a/include/openvic-dataloader/csv/Parser.hpp +++ b/include/openvic-dataloader/csv/Parser.hpp @@ -4,6 +4,12 @@ #include <openvic-dataloader/detail/BasicParser.hpp> namespace ovdl::csv { + enum class EncodingType { + Windows1252, + Utf8 + }; + + template<EncodingType Encoding = EncodingType::Windows1252> class Parser final : public detail::BasicParser { public: Parser(); @@ -39,4 +45,7 @@ namespace ovdl::csv { template<typename... Args> constexpr void _run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args); }; + + using Windows1252Parser = Parser<EncodingType::Windows1252>; + using Utf8Parser = Parser<EncodingType::Utf8>; }
\ No newline at end of file diff --git a/include/openvic-dataloader/detail/ClassExport.hpp b/include/openvic-dataloader/detail/ClassExport.hpp new file mode 100644 index 0000000..27098ed --- /dev/null +++ b/include/openvic-dataloader/detail/ClassExport.hpp @@ -0,0 +1,9 @@ +#pragma once + +#ifdef _MSC_VER +#define OVDL_EXPORT __declspec(dllexport) +#elif defined(__GNUC__) +#define OVDL_EXPORT __attribute__((visibility("default"))) +#else +#define OVDL_EXPORT +#endif
\ No newline at end of file diff --git a/src/headless/main.cpp b/src/headless/main.cpp index 94d6b8d..c3ca8c2 100644 --- a/src/headless/main.cpp +++ b/src/headless/main.cpp @@ -1,17 +1,71 @@ +#include <algorithm> +#include <cctype> #include <cstdio> +#include <cstdlib> #include <iostream> +#include <iterator> #include <string> +#include <string_view> #include <vector> +#include <openvic-dataloader/ParseWarning.hpp> +#include <openvic-dataloader/csv/LineObject.hpp> +#include <openvic-dataloader/csv/Parser.hpp> +#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp> #include <openvic-dataloader/v2script/Parser.hpp> -int main(int argc, char** argv) { - if (argc < 2) { - std::fprintf(stderr, "usage: %s <filename>", argv[0]); +std::string_view trim(std::string_view str) { + std::string_view::iterator begin = str.begin(); + std::string_view::iterator end = str.end(); + for (;; begin++) { + if (begin == end) return std::string_view(); + if (!std::isspace(*begin)) break; + } + end--; + for (;; end--) { + if (end == begin) return std::string_view(); + if (!std::isspace(*end)) break; + } + return std::string_view(&*begin, std::distance(begin, end)); +} + +bool insenitive_trim_eq(std::string_view lhs, std::string_view rhs) { + lhs = trim(lhs); + rhs = trim(rhs); + return std::equal( + lhs.begin(), lhs.end(), + rhs.begin(), rhs.end(), + [](char a, char b) { return std::tolower(a) == std::tolower(b); }); +} + +template<ovdl::csv::EncodingType Encoding> +int print_csv(const std::string_view path) { + auto parser = ovdl::csv::Parser<Encoding>::from_file(path); + if (parser.has_error()) { return 1; } - auto parser = ovdl::v2script::Parser::from_file(argv[1]); + parser.parse_csv(); + if (parser.has_error()) { + return 2; + } + + if (parser.has_warning()) { + for (auto& warning : parser.get_warnings()) { + std::cerr << "Warning: " << warning.message << std::endl; + } + } + + std::cout << "lines:\t\t" << parser.get_lines().size() << std::endl; + for (const auto& line : parser.get_lines()) { + std::cout << "line size:\t" << line.value_count() << std::endl; + std::cout << "values:\t\t" << line << std::endl; + } + return EXIT_SUCCESS; +} + +int print_v2script_simple(const std::string_view path) { + auto parser = ovdl::v2script::Parser::from_file(path); if (parser.has_error()) { return 1; } @@ -28,6 +82,28 @@ int main(int argc, char** argv) { } std::cout << parser.get_file_node() << std::endl; + return EXIT_SUCCESS; +} + +int main(int argc, char** argv) { + switch (argc) { + case 2: + return print_v2script_simple(argv[1]); + case 4: + if (insenitive_trim_eq(argv[1], "csv") && insenitive_trim_eq(argv[2], "utf")) + return print_csv<ovdl::csv::EncodingType::Utf8>(argv[3]); + goto default_jump; + case 3: + if (insenitive_trim_eq(argv[1], "csv")) + return print_csv<ovdl::csv::EncodingType::Windows1252>(argv[2]); + [[fallthrough]]; + default: + default_jump: + std::fprintf(stderr, "usage: %s <filename>\n", argv[0]); + std::fprintf(stderr, "usage: %s csv <filename>\n", argv[0]); + std::fprintf(stderr, "usage: %s csv utf <filename>", argv[0]); + return EXIT_FAILURE; + } return 0; }
\ No newline at end of file diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp index edce97b..4765112 100644 --- a/src/openvic-dataloader/csv/CsvGrammar.hpp +++ b/src/openvic-dataloader/csv/CsvGrammar.hpp @@ -11,8 +11,141 @@ #include <lexy/callback.hpp> #include <lexy/dsl.hpp> +#include "detail/LexyLitRange.hpp" + // Grammar Definitions // -namespace ovdl::csv::grammar { +namespace ovdl::csv::grammar::windows1252 { + constexpr auto character = detail::lexydsl::make_range<0x01, 0xFF>(); + constexpr auto control = + lexy::dsl::ascii::control / + lexy::dsl::lit_b<0x81> / lexy::dsl::lit_b<0x8D> / lexy::dsl::lit_b<0x8F> / + lexy::dsl::lit_b<0x90> / lexy::dsl::lit_b<0x9D>; + + struct StringValue { + static constexpr auto escaped_symbols = lexy::symbol_table<char> // + .map<'"'>('"') + .map<'\''>('\'') + .map<'\\'>('\\') + .map<'/'>('/') + .map<'b'>('\b') + .map<'f'>('\f') + .map<'n'>('\n') + .map<'r'>('\r') + .map<'t'>('\t'); + /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" + // static constexpr auto escaped_quote = lexy::symbol_table<char> // + // .map<'"'>('"'); + static constexpr auto rule = [] { + // Arbitrary code points + auto c = character - (control / lexy::dsl::lit_b<'"'>); + + auto back_escape = lexy::dsl::backslash_escape // + .symbol<escaped_symbols>(); + + // auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + // .symbol<escaped_quote>(); + + return lexy::dsl::quoted(c, back_escape); + }(); + + static constexpr auto value = lexy::as_string<std::string>; + }; + + template<char Sep> + struct PlainValue { + static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b<Sep> / lexy::dsl::ascii::newline)); + static constexpr auto value = lexy::as_string<std::string>; + }; + + template<char Sep> + struct Value { + static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>; + static constexpr auto value = lexy::forward<std::string>; + }; + + template<char Sep> + struct SepConst { + static constexpr auto rule = lexy::dsl::lit_b<Sep>; + static constexpr auto value = lexy::constant(1); + }; + + template<char Sep> + struct Seperator { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Sep>>); + static constexpr auto value = lexy::count; + }; + + template<char Sep> + struct LineEnd { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Sep>>)); + static constexpr auto value = lexy::fold_inplace<csv::LineObject>( + std::initializer_list<csv::LineObject::value_type> {}, + [](csv::LineObject& result, auto&& arg) { + if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) { + // Count seperators, adds to previous value, making it a position + using position_type = csv::LineObject::position_type; + result.emplace_back(static_cast<position_type>(arg + result.back().first), ""); + } else { + if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); + else { + auto& [pos, value] = result.back(); + value = arg; + } + } + }); + }; + + template<char Sep> + struct Line { + + static constexpr auto suffix_setter(csv::LineObject& line) { + auto& [position, value] = line.back(); + if (value.empty()) { + line.set_suffix_end(position); + line.pop_back(); + } else { + line.set_suffix_end(position + 1); + } + }; + + static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<Seperator<Sep>> >> lexy::dsl::p<LineEnd<Sep>>; + static constexpr auto value = + lexy::callback<csv::LineObject>( + [](csv::LineObject&& line) { + suffix_setter(line); + return LEXY_MOV(line); + }, + [](std::size_t prefix_count, csv::LineObject&& line) { + line.set_prefix_end(prefix_count); + // position needs to be adjusted to prefix + for (auto& [position, value] : line) { + position += prefix_count; + } + suffix_setter(line); + return LEXY_MOV(line); + }); + }; + + template<char Sep> + struct File { + static constexpr auto rule = + lexy::dsl::whitespace(lexy::dsl::newline) + + lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p<Line<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::eol))); + + static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>; + }; + + using CommaFile = File<','>; + using ColonFile = File<':'>; + using SemiColonFile = File<';'>; + using TabFile = File<'\t'>; + using BarFile = File<'|'>; +} + +namespace ovdl::csv::grammar::utf8 { + constexpr auto character = lexy::dsl::unicode::character; + constexpr auto control = lexy::dsl::unicode::control; + struct StringValue { static constexpr auto escaped_symbols = lexy::symbol_table<char> // .map<'"'>('"') @@ -25,52 +158,58 @@ namespace ovdl::csv::grammar { .map<'r'>('\r') .map<'t'>('\t'); /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" - static constexpr auto escaped_quote = lexy::symbol_table<char> // - .map<'"'>('"'); + // static constexpr auto escaped_quote = lexy::symbol_table<char> // + // .map<'"'>('"'); static constexpr auto rule = [] { // Arbitrary code points - auto c = -lexy::dsl::lit_c<'"'>; + auto c = character - (control / lexy::dsl::lit_b<'"'>); auto back_escape = lexy::dsl::backslash_escape // .symbol<escaped_symbols>(); - auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // - .symbol<escaped_quote>(); + // auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + // .symbol<escaped_quote>(); - return lexy::dsl::quoted(c, back_escape, quote_escape); + return lexy::dsl::quoted(c, back_escape); }(); static constexpr auto value = lexy::as_string<std::string>; }; - template<auto Sep> + template<char Sep> struct PlainValue { - static constexpr auto rule = lexy::dsl::identifier(-(Sep / lexy::dsl::lit_c<'\n'>)); + static constexpr auto rule = lexy::dsl::identifier(character - (lexy::dsl::lit_b<Sep> / lexy::dsl::ascii::newline)); static constexpr auto value = lexy::as_string<std::string>; }; - template<auto Sep> + template<char Sep> struct Value { static constexpr auto rule = lexy::dsl::p<StringValue> | lexy::dsl::p<PlainValue<Sep>>; static constexpr auto value = lexy::forward<std::string>; }; - template<auto Sep> - struct SeperatorCount { - static constexpr auto rule = lexy::dsl::list(Sep); + template<char Sep> + struct SepConst { + static constexpr auto rule = lexy::dsl::lit_b<Sep>; + static constexpr auto value = lexy::constant(1); + }; + + template<char Sep> + struct Seperator { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<SepConst<Sep>>); static constexpr auto value = lexy::count; }; - template<auto Sep> + template<char Sep> struct LineEnd { - static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<SeperatorCount<Sep>>)); + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p<Value<Sep>>, lexy::dsl::trailing_sep(lexy::dsl::p<Seperator<Sep>>)); static constexpr auto value = lexy::fold_inplace<csv::LineObject>( std::initializer_list<csv::LineObject::value_type> {}, [](csv::LineObject& result, auto&& arg) { if constexpr (std::is_same_v<std::decay_t<decltype(arg)>, std::size_t>) { // Count seperators, adds to previous value, making it a position using position_type = csv::LineObject::position_type; - result.emplace_back(static_cast<position_type>(arg + std::get<0>(result.back())), ""); + result.emplace_back(static_cast<position_type>(arg + result.back().first), ""); } else { if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); else { @@ -81,7 +220,7 @@ namespace ovdl::csv::grammar { }); }; - template<auto Sep> + template<char Sep> struct Line { static constexpr auto suffix_setter(csv::LineObject& line) { @@ -94,7 +233,7 @@ namespace ovdl::csv::grammar { } }; - static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<SeperatorCount<Sep>> >> lexy::dsl::p<LineEnd<Sep>>; + static constexpr auto rule = lexy::dsl::p<LineEnd<Sep>> | lexy::dsl::p<Seperator<Sep>> >> lexy::dsl::p<LineEnd<Sep>>; static constexpr auto value = lexy::callback<csv::LineObject>( [](csv::LineObject&& line) { @@ -112,7 +251,7 @@ namespace ovdl::csv::grammar { }); }; - template<auto Sep> + template<char Sep> struct File { static constexpr auto rule = lexy::dsl::whitespace(lexy::dsl::newline) + @@ -121,9 +260,9 @@ namespace ovdl::csv::grammar { static constexpr auto value = lexy::as_list<std::vector<csv::LineObject>>; }; - using CommaFile = File<lexy::dsl::lit_c<','>>; - using ColonFile = File<lexy::dsl::lit_c<':'>>; - using SemiColonFile = File<lexy::dsl::lit_c<';'>>; - using TabFile = File<lexy::dsl::lit_c<'\t'>>; - using BarFile = File<lexy::dsl::lit_c<'|'>>; + using CommaFile = File<','>; + using ColonFile = File<':'>; + using SemiColonFile = File<';'>; + using TabFile = File<'\t'>; + using BarFile = File<'|'>; }
\ No newline at end of file diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp index 45f3142..86ad02e 100644 --- a/src/openvic-dataloader/csv/Parser.cpp +++ b/src/openvic-dataloader/csv/Parser.cpp @@ -3,6 +3,7 @@ #include <openvic-dataloader/csv/LineObject.hpp> #include <openvic-dataloader/csv/Parser.hpp> +#include <openvic-dataloader/detail/ClassExport.hpp> #include <lexy/action/parse.hpp> #include <lexy/encoding.hpp> @@ -20,11 +21,26 @@ using namespace ovdl::csv; /// BufferHandler /// -class Parser::BufferHandler final : public detail::BasicBufferHandler<lexy::utf8_char_encoding> { +template<EncodingType> +struct LexyEncodingFrom { +}; + +template<> +struct LexyEncodingFrom<EncodingType::Windows1252> { + using encoding = lexy::default_encoding; +}; + +template<> +struct LexyEncodingFrom<EncodingType::Utf8> { + using encoding = lexy::utf8_char_encoding; +}; + +template<EncodingType Encoding> +class Parser<Encoding>::BufferHandler final : public detail::BasicBufferHandler<typename LexyEncodingFrom<Encoding>::encoding> { public: template<typename Node, typename ErrorCallback> std::optional<std::vector<ParseError>> parse(const ErrorCallback& callback) { - auto result = lexy::parse<Node>(_buffer, callback); + auto result = lexy::parse<Node>(this->_buffer, callback); if (!result) { return result.errors(); } @@ -42,36 +58,45 @@ private: /// BufferHandler /// -Parser::Parser() +template<EncodingType Encoding> +Parser<Encoding>::Parser() : _buffer_handler(std::make_unique<BufferHandler>()) { set_error_log_to_stderr(); } -Parser::Parser(Parser&&) = default; -Parser& Parser::operator=(Parser&&) = default; -Parser::~Parser() = default; +template<EncodingType Encoding> +Parser<Encoding>::Parser(Parser&&) = default; +template<EncodingType Encoding> +Parser<Encoding>& Parser<Encoding>::operator=(Parser&&) = default; +template<EncodingType Encoding> +Parser<Encoding>::~Parser() = default; -Parser Parser::from_buffer(const char* data, std::size_t size) { +template<EncodingType Encoding> +Parser<Encoding> Parser<Encoding>::from_buffer(const char* data, std::size_t size) { Parser result; return std::move(result.load_from_buffer(data, size)); } -Parser Parser::from_buffer(const char* start, const char* end) { +template<EncodingType Encoding> +Parser<Encoding> Parser<Encoding>::from_buffer(const char* start, const char* end) { Parser result; return std::move(result.load_from_buffer(start, end)); } -Parser Parser::from_string(const std::string_view string) { +template<EncodingType Encoding> +Parser<Encoding> Parser<Encoding>::from_string(const std::string_view string) { Parser result; return std::move(result.load_from_string(string)); } -Parser Parser::from_file(const char* path) { +template<EncodingType Encoding> +Parser<Encoding> Parser<Encoding>::from_file(const char* path) { Parser result; return std::move(result.load_from_file(path)); } -Parser Parser::from_file(const std::filesystem::path& path) { +template<EncodingType Encoding> +Parser<Encoding> Parser<Encoding>::from_file(const std::filesystem::path& path) { Parser result; return std::move(result.load_from_file(path)); } @@ -89,8 +114,9 @@ Parser Parser::from_file(const std::filesystem::path& path) { /// @param func /// @param args /// +template<EncodingType Encoding> template<typename... Args> -constexpr void Parser::_run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args) { +constexpr void Parser<Encoding>::_run_load_func(detail::LoadCallback<BufferHandler, Args...> auto func, Args... args) { _warnings.clear(); _errors.clear(); _has_fatal_error = false; @@ -101,43 +127,55 @@ constexpr void Parser::_run_load_func(detail::LoadCallback<BufferHandler, Args.. } } -constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size) { +template<EncodingType Encoding> +constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* data, std::size_t size) { // Type can't be deduced? _run_load_func(std::mem_fn(&BufferHandler::load_buffer_size), data, size); return *this; } -constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) { +template<EncodingType Encoding> +constexpr Parser<Encoding>& Parser<Encoding>::load_from_buffer(const char* start, const char* end) { // Type can't be deduced? _run_load_func(std::mem_fn(&BufferHandler::load_buffer), start, end); return *this; } -constexpr Parser& Parser::load_from_string(const std::string_view string) { +template<EncodingType Encoding> +constexpr Parser<Encoding>& Parser<Encoding>::load_from_string(const std::string_view string) { return load_from_buffer(string.data(), string.size()); } -constexpr Parser& Parser::load_from_file(const char* path) { +template<EncodingType Encoding> +constexpr Parser<Encoding>& Parser<Encoding>::load_from_file(const char* path) { _file_path = path; // Type can be deduced?? _run_load_func(std::mem_fn(&BufferHandler::load_file), path); return *this; } -Parser& Parser::load_from_file(const std::filesystem::path& path) { +template<EncodingType Encoding> +Parser<Encoding>& Parser<Encoding>::load_from_file(const std::filesystem::path& path) { return load_from_file(path.string().c_str()); } -constexpr Parser& Parser::load_from_file(const detail::Has_c_str auto& path) { +template<EncodingType Encoding> +constexpr Parser<Encoding>& Parser<Encoding>::load_from_file(const detail::Has_c_str auto& path) { return load_from_file(path.c_str()); } -bool Parser::parse_csv() { +template<EncodingType Encoding> +bool Parser<Encoding>::parse_csv() { if (!_buffer_handler->is_valid()) { return false; } - auto errors = _buffer_handler->parse<csv::grammar::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + std::optional<std::vector<ParseError>> errors; + if constexpr (Encoding == EncodingType::Windows1252) { + errors = _buffer_handler->template parse<csv::grammar::windows1252::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + } else { + errors = _buffer_handler->template parse<csv::grammar::utf8::SemiColonFile>(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + } if (errors) { _errors.reserve(errors->size()); for (auto& err : errors.value()) { @@ -150,6 +188,10 @@ bool Parser::parse_csv() { return true; } -const std::vector<csv::LineObject>& Parser::get_lines() const { +template<EncodingType Encoding> +const std::vector<csv::LineObject>& Parser<Encoding>::get_lines() const { return _lines; -}
\ No newline at end of file +} + +template class ovdl::csv::Parser<EncodingType::Windows1252>; +template class ovdl::csv::Parser<EncodingType::Utf8>;
\ No newline at end of file diff --git a/src/openvic-dataloader/detail/BasicBufferHandler.hpp b/src/openvic-dataloader/detail/BasicBufferHandler.hpp index 75ea8ee..5f4f82d 100644 --- a/src/openvic-dataloader/detail/BasicBufferHandler.hpp +++ b/src/openvic-dataloader/detail/BasicBufferHandler.hpp @@ -15,6 +15,8 @@ namespace ovdl::detail { template<typename Encoding = lexy::default_encoding, typename MemoryResource = void> class BasicBufferHandler { public: + using encoding_type = Encoding; + OVDL_OPTIONAL_CONSTEXPR bool is_valid() const { return _buffer.size() != 0; } |