From 7440a5d1433eec4bf87e3723022db187e7f61b1a Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Fri, 28 Jul 2023 00:52:00 -0400 Subject: Rework Grammar and Parser Add proper headless binary construction: Includes basic validation Add Error and Warning structs to Parser Add FileNode pointer getter to Parser Change all `char8_t*` and `const char8_t` to `const char*` in Parser Add Parser move operators and Parser deconstructor Add BufferHandler PIMPL object to Parser Add UTF-8 file Warning to v2script Add proper Grammar value retrieval Add AbstractSyntaxTree for v2script data parser: Has compile-time embedded type information accessible at compile-time and runtime Has Tab-based print functionality Fix wrong environment reference for headless construction in SConstruct Add error retrieval Add BasicCallbackOStreamBuffer for callback streaming Add CallbackStreamBuffer for char Add CallbackWStreamBuffer for wchar_t Add BasicCallbackStream Add CallbackStream for char Add CallbackWStream for wchar_t Add grammar for events and decisions Add event_parse to Parser Add decision_parse to Parser Add .clang-format Ignore dirty lexy module Add CSV parser and grammar: Creates std::vector for a list of lines Add BasicParser and BasicBufferHandler to reduce code reduplication --- .clang-format | 61 +++++ .gitmodules | 1 + SConstruct | 35 +-- deps/SCsub | 2 +- include/openvic-dataloader/ParseData.hpp | 11 + include/openvic-dataloader/ParseError.hpp | 20 ++ include/openvic-dataloader/ParseWarning.hpp | 10 + include/openvic-dataloader/csv/LineObject.hpp | 84 +++++++ include/openvic-dataloader/csv/Parser.hpp | 42 ++++ include/openvic-dataloader/detail/BasicParser.hpp | 37 +++ .../openvic-dataloader/detail/CallbackOStream.hpp | 100 +++++++++ include/openvic-dataloader/detail/Concepts.hpp | 22 ++ include/openvic-dataloader/detail/SelfType.hpp | 28 +++ include/openvic-dataloader/detail/TypeName.hpp | 52 +++++ .../v2script/AbstractSyntaxTree.hpp | 224 +++++++++++++++++++ include/openvic-dataloader/v2script/Parser.hpp | 58 +++-- src/headless/main.cpp | 32 ++- src/openvic-dataloader/csv/CsvGrammar.hpp | 129 +++++++++++ src/openvic-dataloader/csv/Parser.cpp | 151 +++++++++++++ .../detail/BasicBufferHandler.hpp | 44 ++++ src/openvic-dataloader/detail/BasicParser.cpp | 47 ++++ src/openvic-dataloader/detail/DetectUtf8.hpp | 53 +++++ src/openvic-dataloader/detail/Errors.hpp | 23 ++ src/openvic-dataloader/detail/LexyLitRange.hpp | 16 ++ src/openvic-dataloader/detail/LexyReportError.hpp | 102 +++++++++ src/openvic-dataloader/detail/NullBuff.hpp | 30 +++ .../detail/OStreamOutputIterator.hpp | 21 ++ src/openvic-dataloader/detail/Warnings.hpp | 21 ++ .../v2script/AbstractSyntaxTree.cpp | 247 +++++++++++++++++++++ .../v2script/AiBehaviorGrammar.hpp | 34 +++ .../v2script/DecisionGrammar.hpp | 128 +++++++++++ src/openvic-dataloader/v2script/EffectGrammar.hpp | 42 ++++ src/openvic-dataloader/v2script/EventGrammar.hpp | 183 +++++++++++++++ src/openvic-dataloader/v2script/Grammar.cpp | 74 ------ .../v2script/ModifierGrammar.hpp | 33 +++ src/openvic-dataloader/v2script/Parser.cpp | 224 +++++++++++++++++++ src/openvic-dataloader/v2script/SimpleGrammar.hpp | 120 ++++++++++ src/openvic-dataloader/v2script/TriggerGrammar.hpp | 42 ++++ 38 files changed, 2477 insertions(+), 106 deletions(-) create mode 100644 .clang-format create mode 100644 include/openvic-dataloader/ParseData.hpp create mode 100644 include/openvic-dataloader/ParseError.hpp create mode 100644 include/openvic-dataloader/ParseWarning.hpp create mode 100644 include/openvic-dataloader/csv/LineObject.hpp create mode 100644 include/openvic-dataloader/csv/Parser.hpp create mode 100644 include/openvic-dataloader/detail/BasicParser.hpp create mode 100644 include/openvic-dataloader/detail/CallbackOStream.hpp create mode 100644 include/openvic-dataloader/detail/Concepts.hpp create mode 100644 include/openvic-dataloader/detail/SelfType.hpp create mode 100644 include/openvic-dataloader/detail/TypeName.hpp create mode 100644 include/openvic-dataloader/v2script/AbstractSyntaxTree.hpp create mode 100644 src/openvic-dataloader/csv/CsvGrammar.hpp create mode 100644 src/openvic-dataloader/csv/Parser.cpp create mode 100644 src/openvic-dataloader/detail/BasicBufferHandler.hpp create mode 100644 src/openvic-dataloader/detail/BasicParser.cpp create mode 100644 src/openvic-dataloader/detail/DetectUtf8.hpp create mode 100644 src/openvic-dataloader/detail/Errors.hpp create mode 100644 src/openvic-dataloader/detail/LexyLitRange.hpp create mode 100644 src/openvic-dataloader/detail/LexyReportError.hpp create mode 100644 src/openvic-dataloader/detail/NullBuff.hpp create mode 100644 src/openvic-dataloader/detail/OStreamOutputIterator.hpp create mode 100644 src/openvic-dataloader/detail/Warnings.hpp create mode 100644 src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp create mode 100644 src/openvic-dataloader/v2script/AiBehaviorGrammar.hpp create mode 100644 src/openvic-dataloader/v2script/DecisionGrammar.hpp create mode 100644 src/openvic-dataloader/v2script/EffectGrammar.hpp create mode 100644 src/openvic-dataloader/v2script/EventGrammar.hpp delete mode 100644 src/openvic-dataloader/v2script/Grammar.cpp create mode 100644 src/openvic-dataloader/v2script/ModifierGrammar.hpp create mode 100644 src/openvic-dataloader/v2script/Parser.cpp create mode 100644 src/openvic-dataloader/v2script/SimpleGrammar.hpp create mode 100644 src/openvic-dataloader/v2script/TriggerGrammar.hpp diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..86fc638 --- /dev/null +++ b/.clang-format @@ -0,0 +1,61 @@ +--- +Language: Cpp +UseCRLF: false +Standard: c++20 +UseTab: Always +TabWidth: 4 +IndentWidth: 4 +ColumnLimit: 0 +SpacesInSquareBrackets: false +SpacesInParentheses: false +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: false +SpacesInConditionalStatement: false +SpacesInAngles: false +SpaceInEmptyParentheses: false +SpaceInEmptyBlock: false +SpaceBeforeSquareBrackets: false +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeInheritanceColon: true +SpaceBeforeCtorInitializerColon: true +SpaceBeforeCpp11BracedList: true +SpaceBeforeAssignmentOperators: true +SpaceAfterTemplateKeyword: false +SpaceAfterLogicalNot: false +PointerAlignment: Left +PackConstructorInitializers: CurrentLine +NamespaceIndentation: All +LambdaBodyIndentation: Signature +IndentExternBlock: Indent +IndentCaseLabels: true +IndentAccessModifiers: false +IncludeBlocks: Regroup +FixNamespaceComments: false +EmptyLineBeforeAccessModifier: LogicalBlock +Cpp11BracedListStyle: false +CompactNamespaces: false +BreakConstructorInitializers: BeforeColon +BreakBeforeBraces: Attach +AlwaysBreakTemplateDeclarations: Yes +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: AllIfsAndElse +AllowShortEnumsOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: true +AlignTrailingComments: true +AlignEscapedNewlines: Left +AlignAfterOpenBracket: DontAlign +AccessModifierOffset: -4 +IncludeCategories: + - Regex: <[[:alnum:]_]+> + Priority: 1 + - Regex: <[[:alnum:]_]+[.]h> + Priority: 2 + - Regex: ^ + +namespace ovdl { + struct ParseData { + const std::string production_name; + const unsigned int context_start_line; + const unsigned int context_start_column; + }; +} \ No newline at end of file diff --git a/include/openvic-dataloader/ParseError.hpp b/include/openvic-dataloader/ParseError.hpp new file mode 100644 index 0000000..9e4541e --- /dev/null +++ b/include/openvic-dataloader/ParseError.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include + +namespace ovdl { + struct ParseError { + const enum class Type : unsigned char { + Recoverable, + Fatal + } type; + const std::string message; + const int error_value; + const ParseData parse_data; + const unsigned int start_line; + const unsigned int start_column; + }; + +} \ No newline at end of file diff --git a/include/openvic-dataloader/ParseWarning.hpp b/include/openvic-dataloader/ParseWarning.hpp new file mode 100644 index 0000000..307599f --- /dev/null +++ b/include/openvic-dataloader/ParseWarning.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace ovdl { + struct ParseWarning { + const std::string message; + const int warning_value; + }; +} \ No newline at end of file diff --git a/include/openvic-dataloader/csv/LineObject.hpp b/include/openvic-dataloader/csv/LineObject.hpp new file mode 100644 index 0000000..0494ffb --- /dev/null +++ b/include/openvic-dataloader/csv/LineObject.hpp @@ -0,0 +1,84 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ovdl::csv { + /// LineObject should be able to recognize the differences between: + /// Input -> Indexes == "" + /// ;;a;b;c;; -> 0,1,5,6+ == "" + /// a;b;c -> 3+ == "" + /// a;;b;c;; -> 1,4,5+ == "" + /// a;b;;c; -> 2,4+ == "" + /// a;b;c;; -> 3,4+ == "" + /// a;b;c; -> 3+ == "" + /// ;a;b;c -> 0,4+ == "" + /// + /// If this is incorrect, please report an issue. + class LineObject final : public std::vector> { + public: + // Stored position of value + using position_type = std::uint32_t; + // Value + using inner_value_type = std::string; + using container_type = std::vector>; + + constexpr LineObject() = default; + constexpr LineObject(LineObject&) = default; + constexpr LineObject(LineObject&&) = default; + constexpr LineObject(const LineObject&) = default; + + constexpr LineObject& operator=(const LineObject& other) = default; + constexpr LineObject& operator=(LineObject&& other) = default; + + constexpr ~LineObject() = default; + + constexpr LineObject(std::initializer_list pos_and_val) : container_type(pos_and_val) { + } + + constexpr LineObject(position_type prefix_end, std::initializer_list pos_and_val, position_type suffix_end = 0) + : container_type(pos_and_val), + _prefix_end(prefix_end), + _suffix_end(suffix_end) { + } + + /// Special Functionality + /// Retrieves value, produces "" for empty values + constexpr std::string_view get_value_for(std::size_t position) const { + if (position <= _prefix_end || position >= _suffix_end) return ""; + for (const auto& [pos, val] : *this) { + if (pos == position) return val; + } + return ""; + } + /// Tries to retrieve reference, produces nullopt for empty values + constexpr std::optional> try_get_string_at(std::size_t position) const { + if (position <= _prefix_end || position > _suffix_end) return std::nullopt; + for (const auto& [pos, val] : *this) { + if (pos == position) return std::cref(val); + } + return std::nullopt; + } + + constexpr position_type prefix_end() const { return _prefix_end; } + constexpr void set_prefix_end(position_type value) { _prefix_end = value; } + + constexpr position_type suffix_end() const { return _suffix_end; } + constexpr void set_suffix_end(position_type value) { _suffix_end = value; } + + constexpr std::size_t value_count() const { return _suffix_end; } + + private: + // Should be position of first valid value on line + position_type _prefix_end; + // Should be position after last value or position after last seperator + position_type _suffix_end; + }; +} \ No newline at end of file diff --git a/include/openvic-dataloader/csv/Parser.hpp b/include/openvic-dataloader/csv/Parser.hpp new file mode 100644 index 0000000..3497864 --- /dev/null +++ b/include/openvic-dataloader/csv/Parser.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include + +namespace ovdl::csv { + class Parser final : public detail::BasicParser { + public: + Parser(); + + static Parser from_buffer(const char* data, std::size_t size); + static Parser from_buffer(const char* start, const char* end); + static Parser from_string(const std::string_view string); + static Parser from_file(const char* path); + static Parser from_file(const std::filesystem::path& path); + + constexpr Parser& load_from_buffer(const char* data, std::size_t size); + constexpr Parser& load_from_buffer(const char* start, const char* end); + constexpr Parser& load_from_string(const std::string_view string); + constexpr Parser& load_from_file(const char* path); + Parser& load_from_file(const std::filesystem::path& path); + + constexpr Parser& load_from_file(const detail::Has_c_str auto& path); + + bool parse_csv(); + + const std::vector get_lines() const; + + Parser(Parser&&); + Parser& operator=(Parser&&); + + ~Parser(); + + private: + class BufferHandler; + std::unique_ptr _buffer_handler; + std::vector _lines; + + template + constexpr void _run_load_func(detail::LoadCallback auto func, Args... args); + }; +} \ No newline at end of file diff --git a/include/openvic-dataloader/detail/BasicParser.hpp b/include/openvic-dataloader/detail/BasicParser.hpp new file mode 100644 index 0000000..5493804 --- /dev/null +++ b/include/openvic-dataloader/detail/BasicParser.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace ovdl::detail { + class BasicParser { + public: + BasicParser(); + + void set_error_log_to_null(); + void set_error_log_to_stderr(); + void set_error_log_to_stdout(); + void set_error_log_to(std::basic_ostream& stream); + + bool has_error() const; + bool has_fatal_error() const; + bool has_warning() const; + + const std::vector& get_errors() const; + const std::vector& get_warnings() const; + + protected: + std::vector _errors; + std::vector _warnings; + + std::reference_wrapper _error_stream; + const char* _file_path; + bool _has_fatal_error = false; + }; +} \ No newline at end of file diff --git a/include/openvic-dataloader/detail/CallbackOStream.hpp b/include/openvic-dataloader/detail/CallbackOStream.hpp new file mode 100644 index 0000000..641d53f --- /dev/null +++ b/include/openvic-dataloader/detail/CallbackOStream.hpp @@ -0,0 +1,100 @@ +#pragma once + +#include +#include +#include +#include + +namespace ovdl::detail { + template> + class BasicCallbackStreamBuffer : public std::basic_streambuf { + public: + using base_type = std::basic_streambuf; + using callback_type = Callback; + using char_type = typename base_type::char_type; + using int_type = typename base_type::int_type; + + BasicCallbackStreamBuffer(Callback cb, void* user_data = nullptr) + : _callback(cb), + _user_data(user_data) {} + + protected: + std::streamsize xsputn(const char_type* s, std::streamsize n) override { + if constexpr (std::is_same_v) { + _callback(s, n, _user_data); + return n; + } else { + return _callback(s, n, _user_data); // returns the number of characters successfully written. + } + }; + + int_type overflow(int_type ch) override { + if constexpr (std::is_same_v) { + _callback(&ch, 1, _user_data); + return 1; + } else { + return _callback(&ch, 1, _user_data); // returns the number of characters successfully written. + } + } + + private: + Callback _callback; + void* _user_data; + }; + + template + class CallbackStreamBuffer : public BasicCallbackStreamBuffer { + public: + using base_type = BasicCallbackStreamBuffer; + using callback_type = Callback; + using char_type = typename base_type::char_type; + using int_type = typename base_type::int_type; + + CallbackStreamBuffer(Callback cb, void* user_data = nullptr) : base_type(cb, user_data) {} + }; + + template + class CallbackWStreamBuffer : public BasicCallbackStreamBuffer { + public: + using base_type = BasicCallbackStreamBuffer; + using callback_type = Callback; + using char_type = typename base_type::char_type; + using int_type = typename base_type::int_type; + + CallbackWStreamBuffer(Callback cb, void* user_data = nullptr) : base_type(cb, user_data) {} + }; + + template> + class BasicCallbackStream : public std::basic_ostream { + public: + using base_type = std::basic_ostream; + + BasicCallbackStream(Callback cb, void* user_data = nullptr) + : m_sbuf(cb, user_data), + std::basic_ios(&m_sbuf), + std::basic_ostream(&m_sbuf) { + std::basic_ios::init(&m_sbuf); + } + + private: + BasicCallbackStreamBuffer m_sbuf; + }; + + template + class CallbackStream : public BasicCallbackStream { + public: + using base_type = BasicCallbackStream; + + CallbackStream(Callback cb, void* user_data = nullptr) : base_type(cb, user_data) { + } + }; + + template + class CallbackWStream : public BasicCallbackStream { + public: + using base_type = BasicCallbackStream; + + CallbackWStream(Callback cb, void* user_data = nullptr) : base_type(cb, user_data) { + } + }; +} \ No newline at end of file diff --git a/include/openvic-dataloader/detail/Concepts.hpp b/include/openvic-dataloader/detail/Concepts.hpp new file mode 100644 index 0000000..3e1c785 --- /dev/null +++ b/include/openvic-dataloader/detail/Concepts.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace ovdl::detail { + template + concept LoadCallback = + requires(T t, Self* self, Args... args) { + { t(self, std::forward(args)...) } -> std::same_as>; + }; + + template + concept Has_c_str = + requires(T t) { + { t.c_str() } -> std::same_as; + }; +} \ No newline at end of file diff --git a/include/openvic-dataloader/detail/SelfType.hpp b/include/openvic-dataloader/detail/SelfType.hpp new file mode 100644 index 0000000..5209700 --- /dev/null +++ b/include/openvic-dataloader/detail/SelfType.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +namespace ovdl::detail { +#if !defined(_MSC_VER) +#pragma GCC diagnostic push +#pragma clang diagnostic ignored "-Wunknown-warning-option" +#pragma GCC diagnostic ignored "-Wnon-template-friend" +#endif + template + struct Reader { + friend auto adl_GetSelfType(Reader); + }; + + template + struct Writer { + friend auto adl_GetSelfType(Reader) { return U {}; } + }; +#if !defined(_MSC_VER) +#pragma GCC diagnostic pop +#endif + + inline void adl_GetSelfType() {} + + template + using Read = std::remove_pointer_t {}))>; +} diff --git a/include/openvic-dataloader/detail/TypeName.hpp b/include/openvic-dataloader/detail/TypeName.hpp new file mode 100644 index 0000000..1a34a0f --- /dev/null +++ b/include/openvic-dataloader/detail/TypeName.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include + +namespace ovdl::detail { + + template + constexpr auto substring_as_array(std::string_view str, std::index_sequence) { + return std::array { str[Idxs]... }; + } + + template + constexpr auto type_name_array() { +#if defined(__clang__) + constexpr auto prefix = std::string_view { "[T = " }; + constexpr auto suffix = std::string_view { "]" }; + constexpr auto function = std::string_view { __PRETTY_FUNCTION__ }; +#elif defined(__GNUC__) + constexpr auto prefix = std::string_view { "with T = " }; + constexpr auto suffix = std::string_view { "]" }; + constexpr auto function = std::string_view { __PRETTY_FUNCTION__ }; +#elif defined(_MSC_VER) + constexpr auto prefix = std::string_view { "type_name_array<" }; + constexpr auto suffix = std::string_view { ">(void)" }; + constexpr auto function = std::string_view { __FUNCSIG__ }; +#else +#error Unsupported compiler +#endif + + constexpr auto start = function.find(prefix) + prefix.size(); + constexpr auto end = function.rfind(suffix); + + static_assert(start < end); + + constexpr auto name = function.substr(start, (end - start)); + return substring_as_array(name, std::make_index_sequence {}); + } + + template + struct type_name_holder { + static inline constexpr auto value = type_name_array(); + }; + + template + constexpr auto type_name() -> std::string_view { + constexpr auto& value = type_name_holder::value; + return std::string_view { value.data(), value.size() }; + } +} \ No newline at end of file diff --git a/include/openvic-dataloader/v2script/AbstractSyntaxTree.hpp b/include/openvic-dataloader/v2script/AbstractSyntaxTree.hpp new file mode 100644 index 0000000..7b382fd --- /dev/null +++ b/include/openvic-dataloader/v2script/AbstractSyntaxTree.hpp @@ -0,0 +1,224 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define OVDL_PRINT_FUNC_DEF std::ostream& print(std::ostream& stream, size_t indent) const override + +// defines get_type_static and get_type for string type naming +#define OVDL_RT_TYPE_DEF \ + static constexpr std::string_view get_type_static() { return ::ovdl::detail::type_name(); } \ + constexpr std::string_view get_type() const override { return ::ovdl::detail::type_name>(); } + +// defines type for self-class referencing +#define OVDL_TYPE_DEFINE_SELF \ + struct _self_type_tag {}; \ + constexpr auto _self_type_helper()->decltype(::ovdl::detail::Writer<_self_type_tag, decltype(this)> {}); \ + using type = ::ovdl::detail::Read<_self_type_tag>; + +namespace ovdl::v2script::ast { + + struct Node; + using NodePtr = Node*; + using NodeCPtr = const Node*; + using NodeUPtr = std::unique_ptr; + + struct Node { + Node(const Node&) = delete; + Node& operator=(const Node&) = delete; + Node() = default; + Node(Node&&) = default; + Node& operator=(Node&&) = default; + virtual ~Node() = default; + + virtual std::ostream& print(std::ostream& stream, size_t indent) const = 0; + static std::ostream& print_ptr(std::ostream& stream, NodeCPtr node, size_t indent); + explicit operator std::string() const; + + static constexpr std::string_view get_type_static() { return detail::type_name(); } + constexpr virtual std::string_view get_type() const = 0; + + template + constexpr bool is_type() const { + return get_type().compare(detail::type_name()) == 0; + } + + template + constexpr std::optional cast_to() { + if (is_type()) return static_cast(*this); + return std::nullopt; + } + }; + + inline std::ostream& operator<<(std::ostream& stream, Node const& node) { + return node.print(stream, 0); + } + inline std::ostream& operator<<(std::ostream& stream, NodeCPtr node) { + return Node::print_ptr(stream, node, 0); + } + + template + NodePtr make_node_ptr(Args&&... args) { + if constexpr (std::is_pointer_v) { + return new T(std::forward(args)...); + } else { + return NodePtr(new T(std::forward(args)...)); + } + } + + template + To& cast_node_ptr(const From& from) { + if constexpr (std::is_pointer_v) { + return *static_cast(from); + } else { + return *static_cast(from.get()); + } + } + + template + const To& cast_node_cptr(const From& from) { + if constexpr (std::is_pointer_v) { + return *static_cast(from); + } else { + return *static_cast(from.get()); + } + } + + void copy_into_node_ptr_vector(const std::vector& source, std::vector& dest); + + struct AbstractStringNode : public Node { + std::string _name; + explicit AbstractStringNode(std::string&& name); + OVDL_TYPE_DEFINE_SELF; + OVDL_RT_TYPE_DEF; + OVDL_PRINT_FUNC_DEF; + }; + +#define OVDL_AST_STRING_NODE(NAME) \ + struct NAME final : public AbstractStringNode { \ + explicit NAME(std::string&& name); \ + OVDL_TYPE_DEFINE_SELF; \ + OVDL_RT_TYPE_DEF; \ + OVDL_PRINT_FUNC_DEF; \ + } + + // Value Expression Nodes + OVDL_AST_STRING_NODE(IdentifierNode); + OVDL_AST_STRING_NODE(StringNode); + + // Assignment Nodes + OVDL_AST_STRING_NODE(FactorNode); + OVDL_AST_STRING_NODE(MonthNode); + OVDL_AST_STRING_NODE(NameNode); + OVDL_AST_STRING_NODE(FireOnlyNode); + OVDL_AST_STRING_NODE(IdNode); + OVDL_AST_STRING_NODE(TitleNode); + OVDL_AST_STRING_NODE(DescNode); + OVDL_AST_STRING_NODE(PictureNode); + OVDL_AST_STRING_NODE(IsTriggeredNode); + +#undef OVDL_AST_STRING_NODE + + struct AssignNode final : public Node { + std::string _name; + NodeUPtr _initializer; + explicit AssignNode(NodeCPtr name, NodePtr init); + OVDL_TYPE_DEFINE_SELF; + OVDL_RT_TYPE_DEF; + OVDL_PRINT_FUNC_DEF; + }; + + struct AbstractListNode : public Node { + std::vector _statements; + AbstractListNode(const std::vector& statements = std::vector {}); + OVDL_TYPE_DEFINE_SELF; + OVDL_RT_TYPE_DEF; + OVDL_PRINT_FUNC_DEF; + }; + +#define OVDL_AST_LIST_NODE(NAME) \ + struct NAME final : public AbstractListNode { \ + explicit NAME(const std::vector& statements = std::vector {}); \ + OVDL_TYPE_DEFINE_SELF; \ + OVDL_RT_TYPE_DEF; \ + OVDL_PRINT_FUNC_DEF; \ + } + + OVDL_AST_LIST_NODE(FileNode); + OVDL_AST_LIST_NODE(ListNode); + + OVDL_AST_LIST_NODE(ModifierNode); + OVDL_AST_LIST_NODE(MtthNode); + OVDL_AST_LIST_NODE(EventOptionNode); + OVDL_AST_LIST_NODE(BehaviorListNode); + OVDL_AST_LIST_NODE(DecisionListNode); + +#undef OVDL_AST_LIST_NODE + + struct EventNode final : public Node { + enum class Type { + Country, + Province + } _type; + std::vector _statements; + explicit EventNode(Type type, const std::vector& statements = {}); + OVDL_TYPE_DEFINE_SELF; + OVDL_RT_TYPE_DEF; + OVDL_PRINT_FUNC_DEF; + }; + + struct DecisionNode final : public Node { + NodeUPtr _name; + std::vector _statements; + explicit DecisionNode(NodePtr name, const std::vector& statements = {}); + OVDL_TYPE_DEFINE_SELF; + OVDL_RT_TYPE_DEF; + OVDL_PRINT_FUNC_DEF; + }; + + struct EventMtthModifierNode final : public Node { + NodeUPtr _factor_value; + std::vector _statements; + EventMtthModifierNode() {} + OVDL_TYPE_DEFINE_SELF; + OVDL_RT_TYPE_DEF; + OVDL_PRINT_FUNC_DEF; + }; + + // Packed single case + struct ExecutionNode final : public Node { + enum class Type { + Effect, + Trigger + } _type; + NodeUPtr _name; + NodeUPtr _initializer; + explicit ExecutionNode(Type type, NodePtr name, NodePtr init); + OVDL_TYPE_DEFINE_SELF; + OVDL_RT_TYPE_DEF; + OVDL_PRINT_FUNC_DEF; + }; + + struct ExecutionListNode final : public Node { + ExecutionNode::Type _type; + std::vector _statements; + explicit ExecutionListNode(ExecutionNode::Type type, const std::vector& statements); + OVDL_TYPE_DEFINE_SELF; + OVDL_RT_TYPE_DEF; + OVDL_PRINT_FUNC_DEF; + }; + +} + +#undef OVDL_PRINT_FUNC_DECL +#undef OVDL_PRINT_FUNC_DEF +#undef OVDL_TYPE_DEFINE_SELF \ No newline at end of file diff --git a/include/openvic-dataloader/v2script/Parser.hpp b/include/openvic-dataloader/v2script/Parser.hpp index 53aab90..1c524b2 100644 --- a/include/openvic-dataloader/v2script/Parser.hpp +++ b/include/openvic-dataloader/v2script/Parser.hpp @@ -1,27 +1,59 @@ #pragma once #include -#include +#include +#include +#include +#include #include +#include +#include + +#include +#include +#include +#include +#include namespace ovdl::v2script { - class Parser { + + using FileNode = ast::FileNode; + + class Parser final : public detail::BasicParser { public: - static Parser from_buffer(char8_t* data, std::size_t size); - static Parser from_buffer(char8_t* start, char8_t* end); - static Parser from_file(const char8_t* path); + Parser(); + + static Parser from_buffer(const char* data, std::size_t size); + static Parser from_buffer(const char* start, const char* end); + static Parser from_string(const std::string_view string); + static Parser from_file(const char* path); + static Parser from_file(const std::filesystem::path& path); - void set_error_log_to_stderr(); - void set_error_log_path(const char8_t* path); - void set_error_log_to(std::basic_ostream stream); - void set_error_log_to(std::FILE* file); + constexpr Parser& load_from_buffer(const char* data, std::size_t size); + constexpr Parser& load_from_buffer(const char* start, const char* end); + constexpr Parser& load_from_string(const std::string_view string); + constexpr Parser& load_from_file(const char* path); + Parser& load_from_file(const std::filesystem::path& path); - bool parse(); + constexpr Parser& load_from_file(const detail::Has_c_str auto& path); - bool has_error(); - bool has_warning(); + bool simple_parse(); + bool event_parse(); + bool decision_parse(); + + const FileNode* get_file_node() const; + + Parser(Parser&&); + Parser& operator=(Parser&&); + + ~Parser(); private: - Parser(); + class BufferHandler; + std::unique_ptr _buffer_handler; + std::unique_ptr _file_node; + + template + constexpr void _run_load_func(detail::LoadCallback auto func, Args... args); }; } \ No newline at end of file diff --git a/src/headless/main.cpp b/src/headless/main.cpp index ffc6dab..94d6b8d 100644 --- a/src/headless/main.cpp +++ b/src/headless/main.cpp @@ -1,3 +1,33 @@ -int main() { +#include +#include +#include +#include + +#include + +int main(int argc, char** argv) { + if (argc < 2) { + std::fprintf(stderr, "usage: %s ", argv[0]); + return 1; + } + + auto parser = ovdl::v2script::Parser::from_file(argv[1]); + if (parser.has_error()) { + return 1; + } + + parser.simple_parse(); + if (parser.has_error()) { + return 2; + } + + if (parser.has_warning()) { + for (auto& warning : parser.get_warnings()) { + std::cerr << "Warning: " << warning.message << std::endl; + } + } + + std::cout << parser.get_file_node() << std::endl; + return 0; } \ No newline at end of file diff --git a/src/openvic-dataloader/csv/CsvGrammar.hpp b/src/openvic-dataloader/csv/CsvGrammar.hpp new file mode 100644 index 0000000..edce97b --- /dev/null +++ b/src/openvic-dataloader/csv/CsvGrammar.hpp @@ -0,0 +1,129 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include +#include + +// Grammar Definitions // +namespace ovdl::csv::grammar { + struct StringValue { + static constexpr auto escaped_symbols = lexy::symbol_table // + .map<'"'>('"') + .map<'\''>('\'') + .map<'\\'>('\\') + .map<'/'>('/') + .map<'b'>('\b') + .map<'f'>('\f') + .map<'n'>('\n') + .map<'r'>('\r') + .map<'t'>('\t'); + /// This doesn't actually do anything, so this might to be manually parsed if vic2's CSV parser creates a " from "" + static constexpr auto escaped_quote = lexy::symbol_table // + .map<'"'>('"'); + static constexpr auto rule = [] { + // Arbitrary code points + auto c = -lexy::dsl::lit_c<'"'>; + + auto back_escape = lexy::dsl::backslash_escape // + .symbol(); + + auto quote_escape = lexy::dsl::escape(lexy::dsl::lit_c<'"'>) // + .symbol(); + + return lexy::dsl::quoted(c, back_escape, quote_escape); + }(); + + static constexpr auto value = lexy::as_string; + }; + + template + struct PlainValue { + static constexpr auto rule = lexy::dsl::identifier(-(Sep / lexy::dsl::lit_c<'\n'>)); + static constexpr auto value = lexy::as_string; + }; + + template + struct Value { + static constexpr auto rule = lexy::dsl::p | lexy::dsl::p>; + static constexpr auto value = lexy::forward; + }; + + template + struct SeperatorCount { + static constexpr auto rule = lexy::dsl::list(Sep); + static constexpr auto value = lexy::count; + }; + + template + struct LineEnd { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::p>)); + static constexpr auto value = lexy::fold_inplace( + std::initializer_list {}, + [](csv::LineObject& result, auto&& arg) { + if constexpr (std::is_same_v, std::size_t>) { + // Count seperators, adds to previous value, making it a position + using position_type = csv::LineObject::position_type; + result.emplace_back(static_cast(arg + std::get<0>(result.back())), ""); + } else { + if (result.empty()) result.emplace_back(0u, LEXY_MOV(arg)); + else { + auto& [pos, value] = result.back(); + value = arg; + } + } + }); + }; + + template + struct Line { + + static constexpr auto suffix_setter(csv::LineObject& line) { + auto& [position, value] = line.back(); + if (value.empty()) { + line.set_suffix_end(position); + line.pop_back(); + } else { + line.set_suffix_end(position + 1); + } + }; + + static constexpr auto rule = lexy::dsl::p> | lexy::dsl::p> >> lexy::dsl::p>; + static constexpr auto value = + lexy::callback( + [](csv::LineObject&& line) { + suffix_setter(line); + return LEXY_MOV(line); + }, + [](std::size_t prefix_count, csv::LineObject&& line) { + line.set_prefix_end(prefix_count); + // position needs to be adjusted to prefix + for (auto& [position, value] : line) { + position += prefix_count; + } + suffix_setter(line); + return LEXY_MOV(line); + }); + }; + + template + struct File { + static constexpr auto rule = + lexy::dsl::whitespace(lexy::dsl::newline) + + lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p>, lexy::dsl::trailing_sep(lexy::dsl::eol))); + + static constexpr auto value = lexy::as_list>; + }; + + using CommaFile = File>; + using ColonFile = File>; + using SemiColonFile = File>; + using TabFile = File>; + using BarFile = File>; +} \ No newline at end of file diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp new file mode 100644 index 0000000..8a99085 --- /dev/null +++ b/src/openvic-dataloader/csv/Parser.cpp @@ -0,0 +1,151 @@ +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include "csv/CsvGrammar.hpp" +#include "detail/BasicBufferHandler.hpp" +#include "detail/Errors.hpp" +#include "detail/LexyReportError.hpp" +#include "detail/OStreamOutputIterator.hpp" + +using namespace ovdl; +using namespace ovdl::csv; + +/// BufferHandler /// + +class Parser::BufferHandler final : public detail::BasicBufferHandler { +public: + template + std::optional> parse(const ErrorCallback& callback) { + auto result = lexy::parse(_buffer, callback); + if (!result) { + return result.errors(); + } + _lines = std::move(result.value()); + return std::nullopt; + } + + std::vector& get_lines() { + return _lines; + } + +private: + std::vector _lines; +}; + +/// BufferHandler /// + +Parser::Parser() + : _buffer_handler(std::make_unique()) { + set_error_log_to_stderr(); +} + +Parser::Parser(Parser&&) = default; +Parser& Parser::operator=(Parser&&) = default; +Parser::~Parser() = default; + +Parser Parser::from_buffer(const char* data, std::size_t size) { + Parser result; + return std::move(result.load_from_buffer(data, size)); +} + +Parser Parser::from_buffer(const char* start, const char* end) { + Parser result; + return std::move(result.load_from_buffer(start, end)); +} + +Parser Parser::from_string(const std::string_view string) { + Parser result; + return std::move(result.load_from_string(string)); +} + +Parser Parser::from_file(const char* path) { + Parser result; + return std::move(result.load_from_file(path)); +} + +Parser Parser::from_file(const std::filesystem::path& path) { + Parser result; + return std::move(result.load_from_file(path)); +} + +/// +/// @brief Executes a function on _buffer_handler that is expected to load a buffer +/// +/// Expected Use: +/// @code {.cpp} +/// _run_load_func(&BufferHandler::, ); +/// @endcode +/// +/// @tparam Type +/// @tparam Args +/// @param func +/// @param args +/// +template +constexpr void Parser::_run_load_func(detail::LoadCallback auto func, Args... args) { + _warnings.clear(); + _errors.clear(); + _has_fatal_error = false; + if (auto error = func(_buffer_handler.get(), std::forward(args)...); error) { + _has_fatal_error = error.value().type == ParseError::Type::Fatal; + _errors.push_back(error.value()); + _error_stream.get() << "Error: " << _errors.back().message << '\n'; + } +} + +constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size) { + // Type can't be deduced? + _run_load_func(std::mem_fn(&BufferHandler::load_buffer_size), data, size); + return *this; +} + +constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) { + // Type can't be deduced? + _run_load_func(std::mem_fn(&BufferHandler::load_buffer), start, end); + return *this; +} + +constexpr Parser& Parser::load_from_string(const std::string_view string) { + return load_from_buffer(string.data(), string.size()); +} + +constexpr Parser& Parser::load_from_file(const char* path) { + _file_path = path; + // Type can be deduced?? + _run_load_func(std::mem_fn(&BufferHandler::load_file), path); + return *this; +} + +Parser& Parser::load_from_file(const std::filesystem::path& path) { + return load_from_file(path.string().c_str()); +} + +constexpr Parser& Parser::load_from_file(const detail::Has_c_str auto& path) { + return load_from_file(path.c_str()); +} + +bool Parser::parse_csv() { + if (!_buffer_handler->is_valid()) { + return false; + } + + auto errors = _buffer_handler->parse(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + if (errors) { + _errors.reserve(errors->size()); + for (auto& err : errors.value()) { + _has_fatal_error |= err.type == ParseError::Type::Fatal; + _errors.push_back(err); + } + return false; + } + _lines = std::move(_buffer_handler->get_lines()); + return true; +} \ No newline at end of file diff --git a/src/openvic-dataloader/detail/BasicBufferHandler.hpp b/src/openvic-dataloader/detail/BasicBufferHandler.hpp new file mode 100644 index 0000000..ba2cef9 --- /dev/null +++ b/src/openvic-dataloader/detail/BasicBufferHandler.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include + +#include + +#include +#include +#include + +#include "detail/Errors.hpp" + +namespace ovdl::detail { + template + class BasicBufferHandler { + public: + constexpr bool is_valid() const { + return _buffer.size() != 0; + } + + constexpr std::optional load_buffer_size(const char* data, std::size_t size) { + _buffer = lexy::buffer(data, size); + return std::nullopt; + } + + constexpr std::optional load_buffer(const char* start, const char* end) { + _buffer = lexy::buffer(start, end); + return std::nullopt; + } + + std::optional load_file(const char* path) { + auto file = lexy::read_file(path); + if (!file) { + return ovdl::errors::make_no_file_error(path); + } + + _buffer = file.buffer(); + return std::nullopt; + } + + protected: + lexy::buffer _buffer; + }; +} \ No newline at end of file diff --git a/src/openvic-dataloader/detail/BasicParser.cpp b/src/openvic-dataloader/detail/BasicParser.cpp new file mode 100644 index 0000000..ee1b516 --- /dev/null +++ b/src/openvic-dataloader/detail/BasicParser.cpp @@ -0,0 +1,47 @@ +#include +#include + +#include + +#include "detail/NullBuff.hpp" + +using namespace ovdl; +using namespace ovdl::detail; + +BasicParser::BasicParser() : _error_stream(detail::cnull) {} + +void BasicParser::set_error_log_to_null() { + set_error_log_to(detail::cnull); +} + +void BasicParser::set_error_log_to_stderr() { + set_error_log_to(std::cerr); +} + +void BasicParser::set_error_log_to_stdout() { + set_error_log_to(std::cout); +} + +void BasicParser::set_error_log_to(std::basic_ostream& stream) { + _error_stream = stream; +} + +bool BasicParser::has_error() const { + return !_errors.empty(); +} + +bool BasicParser::has_fatal_error() const { + return _has_fatal_error; +} + +bool BasicParser::has_warning() const { + return !_warnings.empty(); +} + +const std::vector& BasicParser::get_errors() const { + return _errors; +} + +const std::vector& BasicParser::get_warnings() const { + return _warnings; +} \ No newline at end of file diff --git a/src/openvic-dataloader/detail/DetectUtf8.hpp b/src/openvic-dataloader/detail/DetectUtf8.hpp new file mode 100644 index 0000000..2045b3c --- /dev/null +++ b/src/openvic-dataloader/detail/DetectUtf8.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include + +#include "detail/LexyLitRange.hpp" + +namespace ovdl::detail { + namespace detect_utf8 { + + template + struct DetectUtf8 { + struct not_utf8 { + static constexpr auto name = "not utf8"; + }; + + static constexpr auto rule = [] { + constexpr auto is_not_ascii_flag = lexy::dsl::context_flag; + + // & 0b10000000 == 0b00000000 + constexpr auto ascii_values = lexydsl::make_range<0b00000000, 0b01111111>(); + // & 0b11100000 == 0b11000000 + constexpr auto two_byte = lexydsl::make_range<0b11000000, 0b11011111>(); + // & 0b11110000 == 0b11100000 + constexpr auto three_byte = lexydsl::make_range<0b11100000, 0b11101111>(); + // & 0b11111000 == 0b11110000 + constexpr auto four_byte = lexydsl::make_range<0b11110000, 0b11110111>(); + // & 0b11000000 == 0b10000000 + constexpr auto check_bytes = lexydsl::make_range<0b10000000, 0b10111111>(); + + constexpr auto utf8_check = + ((four_byte >> lexy::dsl::times<3>(check_bytes)) | + (three_byte >> lexy::dsl::times<2>(check_bytes)) | + (two_byte >> lexy::dsl::times<1>(check_bytes))) >> + is_not_ascii_flag.set(); + + return is_not_ascii_flag.template create() + + lexy::dsl::while_(utf8_check | ascii_values) + + lexy::dsl::must(is_not_ascii_flag.is_set()).template error; + }(); + }; + } + + template + constexpr bool is_utf8_no_ascii(const Input& input) { + return lexy::match>(input); + } + + template + constexpr bool is_utf8(const Input& input) { + return lexy::match>(input); + } +} \ No newline at end of file diff --git a/src/openvic-dataloader/detail/Errors.hpp b/src/openvic-dataloader/detail/Errors.hpp new file mode 100644 index 0000000..f53bedc --- /dev/null +++ b/src/openvic-dataloader/detail/Errors.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include "openvic-dataloader/v2script/Parser.hpp" + +namespace ovdl::errors { + inline const ParseError make_no_file_error(const char* file_path) { + std::string message; + if (!file_path) { + message = "File path not specified."; + } else { + message = "File '" + std::string(file_path) + "' was not found."; + } + + return ParseError { ParseError::Type::Fatal, message, 1 }; + } +} + +namespace ovdl::v2script::errors { + +} + +namespace ovdl::ovscript::errors { +} \ No newline at end of file diff --git a/src/openvic-dataloader/detail/LexyLitRange.hpp b/src/openvic-dataloader/detail/LexyLitRange.hpp new file mode 100644 index 0000000..a6761a8 --- /dev/null +++ b/src/openvic-dataloader/detail/LexyLitRange.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace ovdl::detail::lexydsl { + template + consteval auto make_range() { + if constexpr (LOW == HIGH) { + return lexy::dsl::lit_c; + } else if constexpr (LOW == (HIGH - 1)) { + return lexy::dsl::lit_c / lexy::dsl::lit_c; + } else { + return lexy::dsl::lit_c / make_range(); + } + } +} \ No newline at end of file diff --git a/src/openvic-dataloader/detail/LexyReportError.hpp b/src/openvic-dataloader/detail/LexyReportError.hpp new file mode 100644 index 0000000..684b5db --- /dev/null +++ b/src/openvic-dataloader/detail/LexyReportError.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include + +namespace ovdl::detail { + template + struct _ReportError { + OutputIterator _iter; + lexy::visualization_options _opts; + const char* _path; + + struct _sink { + OutputIterator _iter; + lexy::visualization_options _opts; + const char* _path; + std::size_t _count; + std::vector _errors; + + using return_type = std::vector; + + template + void operator()(const lexy::error_context& context, const lexy::error& error) { + _iter = lexy_ext::_detail::write_error(_iter, context, error, _opts, _path); + ++_count; + + // Convert the context location and error location into line/column information. + auto context_location = lexy::get_input_location(context.input(), context.position()); + auto location = lexy::get_input_location(context.input(), error.position(), context_location.anchor()); + + std::basic_stringstream message; + + // Write the main annotation. + if constexpr (std::is_same_v) { + auto string = lexy::_detail::make_literal_lexeme(error.string(), error.length()); + + message << "expected '" << string.data() << '\''; + } else if constexpr (std::is_same_v) { + auto string = lexy::_detail::make_literal_lexeme(error.string(), error.length()); + + message << "expected keyword '" << string.data() << '\''; + } else if constexpr (std::is_same_v) { + message << "expected " << error.name(); + } else { + message << error.message(); + } + + _errors.push_back( + ParseError { + ParseError::Type::Fatal, // TODO: distinguish recoverable errors from fatal errors + std::move(message.str()), + 0, // TODO: implement proper error codes + ParseData { + context.production(), + context_location.line_nr(), + context_location.column_nr(), + }, + location.line_nr(), + location.column_nr(), + }); + } + + return_type finish() && { + if (_count != 0) + *_iter++ = '\n'; + return _errors; + } + }; + constexpr auto sink() const { + return _sink { _iter, _opts, _path, 0 }; + } + + /// Specifies a path that will be printed alongside the diagnostic. + constexpr _ReportError path(const char* path) const { + return { _iter, _opts, path }; + } + + /// Specifies an output iterator where the errors are written to. + template + constexpr _ReportError to(OI out) const { + return { out, _opts, _path }; + } + + /// Overrides visualization options. + constexpr _ReportError opts(lexy::visualization_options opts) const { + return { _iter, opts, _path }; + } + }; + + constexpr auto ReporError = _ReportError {}; +} \ No newline at end of file diff --git a/src/openvic-dataloader/detail/NullBuff.hpp b/src/openvic-dataloader/detail/NullBuff.hpp new file mode 100644 index 0000000..baf9e1b --- /dev/null +++ b/src/openvic-dataloader/detail/NullBuff.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +namespace ovdl::detail { + template> + class basic_nullbuf : public std::basic_streambuf { + typename traits::int_type overflow(typename traits::int_type c) { + return traits::not_eof(c); // indicate success + } + }; + + template> + class basic_onullstream : public std::basic_ostream { + public: + basic_onullstream() : std::basic_ios(&m_sbuf), + std::basic_ostream(&m_sbuf) { + std::basic_ios::init(&m_sbuf); + } + + private: + basic_nullbuf m_sbuf; + }; + + typedef basic_onullstream onullstream; + typedef basic_onullstream wonullstream; + + inline onullstream cnull; + inline onullstream wcnull; +} \ No newline at end of file diff --git a/src/openvic-dataloader/detail/OStreamOutputIterator.hpp b/src/openvic-dataloader/detail/OStreamOutputIterator.hpp new file mode 100644 index 0000000..81f6c89 --- /dev/null +++ b/src/openvic-dataloader/detail/OStreamOutputIterator.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include + +namespace ovdl::detail { + struct OStreamOutputIterator { + std::reference_wrapper _stream; + + auto operator*() const noexcept { + return *this; + } + auto operator++(int) const noexcept { + return *this; + } + + OStreamOutputIterator& operator=(char c) { + _stream.get().put(c); + return *this; + } + }; +} \ No newline at end of file diff --git a/src/openvic-dataloader/detail/Warnings.hpp b/src/openvic-dataloader/detail/Warnings.hpp new file mode 100644 index 0000000..fc0fbed --- /dev/null +++ b/src/openvic-dataloader/detail/Warnings.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include "openvic-dataloader/v2script/Parser.hpp" + +namespace ovdl::v2script::warnings { + inline const ParseWarning make_utf8_warning(const char* file_path) { + constexpr std::string_view message_suffix = "This may cause problems. Prefer Windows-1252 encoding."; + + std::string message; + if (!file_path) { + message = "Buffer is a UTF-8 encoded string. " + std::string(message_suffix); + } else { + message = "File '" + std::string(file_path) + "' is a UTF-8 encoded file. " + std::string(message_suffix); + } + + return ParseWarning { message, 1 }; + } +} + +namespace ovdl::ovscript::warnings { +} \ No newline at end of file diff --git a/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp b/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp new file mode 100644 index 0000000..f9fb716 --- /dev/null +++ b/src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp @@ -0,0 +1,247 @@ +#include +#include +#include +#include +#include + +#include + +#include + +using namespace ovdl::v2script::ast; + +void ovdl::v2script::ast::copy_into_node_ptr_vector(const std::vector& source, std::vector& dest) { + dest.clear(); + dest.reserve(source.size()); + for (auto&& p : source) { + dest.push_back(NodeUPtr { p }); + } +} + +AbstractStringNode::AbstractStringNode(std::string&& name) : _name(std::move(name)) {} +std::ostream& AbstractStringNode::print(std::ostream& stream, size_t indent) const { + return stream << _name; +} + +#define OVDL_AST_STRING_NODE_DEF(NAME, ...) \ + NAME::NAME(std::string&& name) : AbstractStringNode(std::move(name)) {} \ + std::ostream& NAME::print(std::ostream& stream, size_t indent) const __VA_ARGS__ + +OVDL_AST_STRING_NODE_DEF(IdentifierNode, { + return stream << _name; +}); + +OVDL_AST_STRING_NODE_DEF(StringNode, { + return stream << '"' << _name << '"'; +}); + +OVDL_AST_STRING_NODE_DEF(FactorNode, { + return stream << "factor = " << _name; +}); + +OVDL_AST_STRING_NODE_DEF(MonthNode, { + return stream << "months = " << _name; +}); + +OVDL_AST_STRING_NODE_DEF(NameNode, { + return stream << "name = " << _name; +}); + +OVDL_AST_STRING_NODE_DEF(FireOnlyNode, { + return stream << "fire_only_once = " << _name; +}); + +OVDL_AST_STRING_NODE_DEF(IdNode, { + return stream << "id = " << _name; +}); + +OVDL_AST_STRING_NODE_DEF(TitleNode, { + return stream << "title = " << _name; +}); + +OVDL_AST_STRING_NODE_DEF(DescNode, { + return stream << "desc = " << _name; +}); + +OVDL_AST_STRING_NODE_DEF(PictureNode, { + return stream << "picture = " << _name; +}); + +OVDL_AST_STRING_NODE_DEF(IsTriggeredNode, { + return stream << "is_triggered_only = " << _name; +}); + +#undef OVDL_AST_STRING_NODE_DEF + +AssignNode::AssignNode(NodeCPtr name, NodePtr init) + : _initializer(std::move(init)) { + if (name->is_type()) { + _name = cast_node_cptr(name)._name; + } +} + +std::ostream& Node::print_ptr(std::ostream& stream, NodeCPtr node, size_t indent) { + return node != nullptr ? node->print(stream, indent) : stream << ""; +} + +static std::ostream& print_newline_indent(std::ostream& stream, size_t indent) { + return stream << "\n" + << std::setw(indent) << std::setfill('\t') << ""; +} + +/* Starts with a newline and ends at the end of a line, and so + * should be followed by a call to print_newline_indent. + */ +static std::ostream& print_nodeuptr_vector(const std::vector& nodes, + std::ostream& stream, size_t indent) { + for (NodeUPtr const& node : nodes) { + print_newline_indent(stream, indent); + Node::print_ptr(stream, node.get(), indent); + } + return stream; +} + +AbstractListNode::AbstractListNode(const std::vector& statements) { + copy_into_node_ptr_vector(statements, _statements); +} +std::ostream& AbstractListNode::print(std::ostream& stream, size_t indent) const { + stream << '{'; + if (!_statements.empty()) { + print_nodeuptr_vector(_statements, stream, indent + 1); + print_newline_indent(stream, indent); + } + return stream << "}"; +} + +#define OVDL_AST_LIST_NODE_DEF(NAME, ...) \ + NAME::NAME(const std::vector& statements) : AbstractListNode(statements) {} \ + std::ostream& NAME::print(std::ostream& stream, size_t indent) const __VA_ARGS__ + +OVDL_AST_LIST_NODE_DEF(FileNode, { + print_nodeuptr_vector(_statements, stream, indent); + return print_newline_indent(stream, indent); +}); + +OVDL_AST_LIST_NODE_DEF(ListNode, { + stream << '{'; + if (!_statements.empty()) { + print_nodeuptr_vector(_statements, stream, indent + 1); + print_newline_indent(stream, indent); + } + return stream << "}"; +}); + +OVDL_AST_LIST_NODE_DEF(ModifierNode, { + stream << "modifier = {"; + if (!_statements.empty()) { + print_nodeuptr_vector(_statements, stream, indent + 1); + print_newline_indent(stream, indent); + } + return stream << '}'; +}); + +OVDL_AST_LIST_NODE_DEF(MtthNode, { + stream << "mean_time_to_happen = {"; + if (!_statements.empty()) { + print_nodeuptr_vector(_statements, stream, indent + 1); + print_newline_indent(stream, indent); + } + return stream << '}'; +}); + +OVDL_AST_LIST_NODE_DEF(EventOptionNode, { + stream << "option = {"; + if (!_statements.empty()) { + print_nodeuptr_vector(_statements, stream, indent + 1); + print_newline_indent(stream, indent); + } + return stream << '}'; +}); + +OVDL_AST_LIST_NODE_DEF(BehaviorListNode, { + stream << "ai_chance = {"; // may be ai_chance or ai_will_do + if (!_statements.empty()) { + print_nodeuptr_vector(_statements, stream, indent + 1); + print_newline_indent(stream, indent); + } + return stream << '}'; +}); + +OVDL_AST_LIST_NODE_DEF(DecisionListNode, { + stream << "political_decisions = {"; + if (!_statements.empty()) { + print_nodeuptr_vector(_statements, stream, indent + 1); + print_newline_indent(stream, indent); + } + return stream << '}'; +}); + +#undef OVDL_AST_LIST_NODE_DEF + +EventNode::EventNode(Type type, const std::vector& statements) : _type(type) { + copy_into_node_ptr_vector(statements, _statements); +} +std::ostream& EventNode::print(std::ostream& stream, size_t indent) const { + switch (_type) { + case Type::Country: stream << "country_event = "; break; + case Type::Province: stream << "province_event = "; break; + } + stream << '{'; + if (!_statements.empty()) { + print_nodeuptr_vector(_statements, stream, indent + 1); + print_newline_indent(stream, indent); + } + return stream << '}'; +} + +DecisionNode::DecisionNode(NodePtr name, const std::vector& statements) : _name(std::move(name)) { + copy_into_node_ptr_vector(statements, _statements); +} +std::ostream& DecisionNode::print(std::ostream& stream, size_t indent) const { + print_ptr(stream, _name.get(), indent) << " = {"; + if (!_statements.empty()) { + print_nodeuptr_vector(_statements, stream, indent + 1); + print_newline_indent(stream, indent); + } + return stream << '}'; +} + +ExecutionNode::ExecutionNode(Type type, NodePtr name, NodePtr init) : _type(type), + _name(std::move(name)), + _initializer(std::move(init)) { +} +std::ostream& ExecutionNode::print(std::ostream& stream, size_t indent) const { + print_ptr(stream, _name.get(), indent) << " = "; + if (_initializer) { + Node::print_ptr(stream, _initializer.get(), indent + 1); + } + return stream; +} + +ExecutionListNode::ExecutionListNode(ExecutionNode::Type type, const std::vector& statements) : _type(type) { + copy_into_node_ptr_vector(statements, _statements); +} +std::ostream& ExecutionListNode::print(std::ostream& stream, size_t indent) const { + // Only way to make a valid declared parsable file + stream << "{ "; + switch (_type) { + case ExecutionNode::Type::Effect: stream << "effect = {"; break; + case ExecutionNode::Type::Trigger: stream << "trigger = {"; break; + } + if (!_statements.empty()) { + print_nodeuptr_vector(_statements, stream, indent + 1); + print_newline_indent(stream, indent); + } + return stream << "}}"; +} + +Node::operator std::string() const { + std::stringstream ss; + ss << *this; + return ss.str(); +} + +std::ostream& AssignNode::print(std::ostream& stream, size_t indent) const { + stream << _name << " = "; + return Node::print_ptr(stream, _initializer.get(), indent); +} diff --git a/src/openvic-dataloader/v2script/AiBehaviorGrammar.hpp b/src/openvic-dataloader/v2script/AiBehaviorGrammar.hpp new file mode 100644 index 0000000..012820f --- /dev/null +++ b/src/openvic-dataloader/v2script/AiBehaviorGrammar.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include + +#include "ModifierGrammar.hpp" +#include "SimpleGrammar.hpp" +#include "TriggerGrammar.hpp" + +namespace ovdl::v2script::grammar { + struct AiBehaviorList { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p | lexy::dsl::p); + + static constexpr auto value = + lexy::as_list> >> + lexy::callback( + [](auto&& list) { + return ast::make_node_ptr(LEXY_MOV(list)); + }); + }; + + struct AiBehaviorBlock { + static constexpr auto rule = lexy::dsl::curly_bracketed.opt(lexy::dsl::p); + + static constexpr auto value = lexy::callback( + [](auto&& list) { + return LEXY_MOV(list); + }, + [](lexy::nullopt = {}) { + return lexy::nullopt {}; + }); + }; +} \ No newline at end of file diff --git a/src/openvic-dataloader/v2script/DecisionGrammar.hpp b/src/openvic-dataloader/v2script/DecisionGrammar.hpp new file mode 100644 index 0000000..93ba0f5 --- /dev/null +++ b/src/openvic-dataloader/v2script/DecisionGrammar.hpp @@ -0,0 +1,128 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include "AiBehaviorGrammar.hpp" +#include "EffectGrammar.hpp" +#include "SimpleGrammar.hpp" +#include "TriggerGrammar.hpp" + +// Decision Grammar Definitions // +namespace ovdl::v2script::grammar { + ////////////////// + // Macros + ////////////////// +// Produces _rule and _p +#define OVDL_GRAMMAR_KEYWORD_DEFINE(KW_NAME) \ + struct KW_NAME##_rule { \ + static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_); \ + static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ + static constexpr auto value = lexy::noop; \ + }; \ + static constexpr auto KW_NAME##_p = lexy::dsl::p + +// Produces _rule and _p and _rule::flag and _rule::too_many_error +#define OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(KW_NAME) \ + struct KW_NAME##_rule { \ + static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_); \ + static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ + static constexpr auto value = lexy::noop; \ + static constexpr auto flag = lexy::dsl::context_flag; \ + struct too_many_error { \ + static constexpr auto name = "expected left side " #KW_NAME " to be found once"; \ + }; \ + }; \ + static constexpr auto KW_NAME##_p = lexy::dsl::p >> (lexy::dsl::must(KW_NAME##_rule::flag.is_reset()).error + KW_NAME##_rule::flag.set()) + ////////////////// + // Macros + ////////////////// + struct DecisionStatement { + template + struct _StringStatement { + static constexpr auto rule = Production >> (lexy::dsl::p | lexy::dsl::p); + static constexpr auto value = lexy::forward; + }; + template + static constexpr auto StringStatement = lexy::dsl::p<_StringStatement>; + + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(potential); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(allow); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(effect); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(ai_will_do); + + static constexpr auto rule = [] { + constexpr auto create_flags = + potential_rule::flag.create() + + allow_rule::flag.create() + + effect_rule::flag.create() + + ai_will_do_rule::flag.create(); + + constexpr auto potential_statement = potential_p >> lexy::dsl::p; + constexpr auto allow_statement = allow_p >> lexy::dsl::p; + constexpr auto effect_statement = effect_p >> lexy::dsl::p; + constexpr auto ai_will_do_statement = ai_will_do_p >> lexy::dsl::p; + + return lexy::dsl::p >> + (create_flags + lexy::dsl::equal_sign + + lexy::dsl::curly_bracketed.list( + potential_statement | + allow_statement | + effect_statement | + ai_will_do_statement | + lexy::dsl::p)); + }(); + + static constexpr auto value = + lexy::as_list> >> + lexy::callback( + [](auto&& name, auto&& list) { + return ast::make_node_ptr(LEXY_MOV(name), LEXY_MOV(list)); + }, + [](auto&& name, lexy::nullopt = {}) { + return ast::make_node_ptr(LEXY_MOV(name)); + }); + }; + + struct DecisionList { + static constexpr auto rule = + LEXY_KEYWORD("political_decisions", lexy::dsl::inline_) >> + (lexy::dsl::equal_sign + lexy::dsl::curly_bracketed.opt_list(lexy::dsl::p)); + + static constexpr auto value = + lexy::as_list> >> + lexy::callback( + [](auto&& list) { + return ast::make_node_ptr(LEXY_MOV(list)); + }, + [](lexy::nullopt = {}) { + return lexy::nullopt {}; + }); + }; + + struct DecisionFile { + // Allow arbitrary spaces between individual tokens. + static constexpr auto whitespace = whitespace_specifier | comment_specifier; + + static constexpr auto rule = + lexy::dsl::terminator(lexy::dsl::eof).list( // + lexy::dsl::p | // + lexy::dsl::p); + + static constexpr auto value = lexy::as_list> >> lexy::new_; + }; + +#undef OVDL_GRAMMAR_KEYWORD_DEFINE +#undef OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE +#undef OVDL_GRAMMAR_KEYWORD_STATEMENT +#undef OVDL_GRAMMAR_KEYWORD_FLAG_STATEMENT +} \ No newline at end of file diff --git a/src/openvic-dataloader/v2script/EffectGrammar.hpp b/src/openvic-dataloader/v2script/EffectGrammar.hpp new file mode 100644 index 0000000..9f164b2 --- /dev/null +++ b/src/openvic-dataloader/v2script/EffectGrammar.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include +#include + +#include "SimpleGrammar.hpp" + +namespace ovdl::v2script::grammar { + struct EffectStatement { + static constexpr auto rule = lexy::dsl::inline_; + + static constexpr auto value = lexy::callback( + [](auto name, auto&& initalizer) { + return ast::make_node_ptr(ast::ExecutionNode::Type::Effect, LEXY_MOV(name), LEXY_MOV(initalizer)); + }); + }; + + struct EffectList { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p); + + static constexpr auto value = + lexy::as_list> >> + lexy::callback( + [](auto&& list) { + return ast::make_node_ptr(ast::ExecutionNode::Type::Effect, LEXY_MOV(list)); + }); + }; + + struct EffectBlock { + static constexpr auto rule = lexy::dsl::curly_bracketed.opt(lexy::dsl::p); + + static constexpr auto value = lexy::callback( + [](auto&& list) { + return LEXY_MOV(list); + }, + [](lexy::nullopt = {}) { + return lexy::nullopt {}; + }); + }; +} \ No newline at end of file diff --git a/src/openvic-dataloader/v2script/EventGrammar.hpp b/src/openvic-dataloader/v2script/EventGrammar.hpp new file mode 100644 index 0000000..93a52bf --- /dev/null +++ b/src/openvic-dataloader/v2script/EventGrammar.hpp @@ -0,0 +1,183 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include + +#include "AiBehaviorGrammar.hpp" +#include "EffectGrammar.hpp" +#include "ModifierGrammar.hpp" +#include "SimpleGrammar.hpp" +#include "TriggerGrammar.hpp" + +// Event Grammar Definitions // +namespace ovdl::v2script::grammar { + ////////////////// + // Macros + ////////////////// +// Produces _rule and _p +#define OVDL_GRAMMAR_KEYWORD_DEFINE(KW_NAME) \ + struct KW_NAME##_rule { \ + static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_); \ + static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ + static constexpr auto value = lexy::noop; \ + }; \ + static constexpr auto KW_NAME##_p = lexy::dsl::p + +// Produces _rule and _p and _rule::flag and _rule::too_many_error +#define OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(KW_NAME) \ + struct KW_NAME##_rule { \ + static constexpr auto keyword = LEXY_KEYWORD(#KW_NAME, lexy::dsl::inline_); \ + static constexpr auto rule = keyword >> lexy::dsl::equal_sign; \ + static constexpr auto value = lexy::noop; \ + static constexpr auto flag = lexy::dsl::context_flag; \ + struct too_many_error { \ + static constexpr auto name = "expected left side " #KW_NAME " to be found once"; \ + }; \ + }; \ + static constexpr auto KW_NAME##_p = lexy::dsl::p >> (lexy::dsl::must(KW_NAME##_rule::flag.is_reset()).error + KW_NAME##_rule::flag.set()) + ////////////////// + // Macros + ////////////////// + static constexpr auto event_symbols = lexy::symbol_table // + .map(ast::EventNode::Type::Country) + .map(ast::EventNode::Type::Province); + + struct EventMtthStatement { + OVDL_GRAMMAR_KEYWORD_DEFINE(months); + + struct MonthValue { + static constexpr auto rule = lexy::dsl::inline_; + static constexpr auto value = lexy::as_string | lexy::new_; + }; + + static constexpr auto rule = lexy::dsl::list( + (months_p >> lexy::dsl::p) | + lexy::dsl::p); + + static constexpr auto value = + lexy::as_list> >> + lexy::callback( + [](auto&& list) { + return ast::make_node_ptr(LEXY_MOV(list)); + }); + }; + + template + struct _StringStatement { + static constexpr auto rule = Production >> (lexy::dsl::p | lexy::dsl::p); + static constexpr auto value = + lexy::callback( + [](auto&& value) { + auto result = ast::make_node_ptr(std::move(static_cast(value)->_name)); + delete value; + return result; + }); + }; + template + static constexpr auto StringStatement = lexy::dsl::p<_StringStatement>; + + struct EventOptionList { + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(name); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(ai_chance); + + static constexpr auto rule = [] { + constexpr auto create_flags = name_rule::flag.create() + ai_chance_rule::flag.create(); + + constexpr auto name_statement = StringStatement; + constexpr auto ai_chance_statement = ai_chance_p >> lexy::dsl::curly_bracketed(lexy::dsl::p); + + return create_flags + lexy::dsl::list(name_statement | ai_chance_statement | lexy::dsl::p); + }(); + + static constexpr auto value = + lexy::as_list> >> + lexy::callback( + [](auto&& list) { + return ast::make_node_ptr(LEXY_MOV(list)); + }); + }; + + struct EventStatement { + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(id); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(title); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(desc); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(picture); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(is_triggered_only); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(fire_only_once); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(immediate); + OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE(mean_time_to_happen); + OVDL_GRAMMAR_KEYWORD_DEFINE(trigger); + OVDL_GRAMMAR_KEYWORD_DEFINE(option); + + static constexpr auto rule = [] { + constexpr auto symbol_value = lexy::dsl::symbol(lexy::dsl::inline_); + + constexpr auto create_flags = + id_rule::flag.create() + + title_rule::flag.create() + + desc_rule::flag.create() + + picture_rule::flag.create() + + is_triggered_only_rule::flag.create() + + fire_only_once_rule::flag.create() + + immediate_rule::flag.create() + + mean_time_to_happen_rule::flag.create(); + + constexpr auto id_statement = StringStatement; + constexpr auto title_statement = StringStatement; + constexpr auto desc_statement = StringStatement; + constexpr auto picture_statement = StringStatement; + constexpr auto is_triggered_only_statement = StringStatement; + constexpr auto fire_only_once_statement = StringStatement; + constexpr auto immediate_statement = immediate_p >> lexy::dsl::p; + constexpr auto mean_time_to_happen_statement = mean_time_to_happen_p >> lexy::dsl::curly_bracketed(lexy::dsl::p); + + constexpr auto trigger_statement = trigger_p >> lexy::dsl::curly_bracketed.opt(lexy::dsl::p); + constexpr auto option_statement = option_p >> lexy::dsl::curly_bracketed(lexy::dsl::p); + + return symbol_value >> + (create_flags + lexy::dsl::equal_sign + + lexy::dsl::curly_bracketed.opt_list( + id_statement | + title_statement | + desc_statement | + picture_statement | + is_triggered_only_statement | + fire_only_once_statement | + immediate_statement | + mean_time_to_happen_statement | + trigger_statement | + option_statement | + lexy::dsl::p)); + }(); + + static constexpr auto value = + lexy::as_list> >> + lexy::callback( + [](auto& type, auto&& list) { + return ast::make_node_ptr(type, LEXY_MOV(list)); + }, + [](auto& type, lexy::nullopt = {}) { + return ast::make_node_ptr(type); + }); + }; + + struct EventFile { + // Allow arbitrary spaces between individual tokens. + static constexpr auto whitespace = whitespace_specifier | comment_specifier; + + static constexpr auto rule = lexy::dsl::terminator(lexy::dsl::eof).list(lexy::dsl::p | lexy::dsl::p); + + static constexpr auto value = lexy::as_list> >> lexy::new_; + }; + +#undef OVDL_GRAMMAR_KEYWORD_DEFINE +#undef OVDL_GRAMMAR_KEYWORD_FLAG_DEFINE +#undef OVDL_GRAMMAR_KEYWORD_STATEMENT +#undef OVDL_GRAMMAR_KEYWORD_FLAG_STATEMENT +} \ No newline at end of file diff --git a/src/openvic-dataloader/v2script/Grammar.cpp b/src/openvic-dataloader/v2script/Grammar.cpp deleted file mode 100644 index ec9fac2..0000000 --- a/src/openvic-dataloader/v2script/Grammar.cpp +++ /dev/null @@ -1,74 +0,0 @@ -#include -#include - -using namespace ovdl::v2script; - -// Node Definitions // -namespace dsl = lexy::dsl; - -namespace ovdl::v2script::nodes { - struct StatementListBlock; - - static constexpr auto whitespace_specifier = dsl::code_point.range<0x09, 0x0A>() / dsl::lit_cp<0x0D> / dsl::lit_cp<0x20>; - static constexpr auto comment_specifier = LEXY_LIT("#") >> dsl::until(dsl::newline).or_eof(); - - static constexpr auto data_specifier = - dsl::ascii::alpha_digit_underscore / - dsl::code_point.range<0x25, 0x27>() / dsl::lit_cp<0x2B> / dsl::code_point.range<0x2D, 0x2E>() / - dsl::lit_cp<0x3A> / - dsl::lit_cp<0x8A> / dsl::lit_cp<0x8C> / dsl::lit_cp<0x8E> / - dsl::lit_cp<0x92> / dsl::lit_cp<0x9A> / dsl::lit_cp<0x9C> / dsl::code_point.range<0x9E, 0x9F>() / - dsl::code_point.range<0xC0, 0xD6>() / dsl::code_point.range<0xD8, 0xF6>() / dsl::code_point.range<0xF8, 0xFF>(); - - static constexpr auto data_char_class = LEXY_CHAR_CLASS("DataSpecifier", data_specifier); - - struct Identifier { - static constexpr auto rule = dsl::identifier(data_char_class); - }; - - struct StringExpression { - static constexpr auto escaped_symbols = lexy::symbol_table // - .map<'"'>('"') - .map<'\''>('\'') - .map<'\\'>('\\') - .map<'/'>('/') - .map<'b'>('\b') - .map<'f'>('\f') - .map<'n'>('\n') - .map<'r'>('\r') - .map<'t'>('\t'); - static constexpr auto rule = [] { - // Arbitrary code points that aren't control characters. - auto c = -dsl::unicode::control; - - // Escape sequences start with a backlash. - // They either map one of the symbols, - // or a Unicode code point of the form uXXXX. - auto escape = dsl::backslash_escape // - .symbol() - .rule(dsl::lit_c<'u'> >> dsl::code_point_id<4>); - return dsl::quoted(c, escape); - }(); - }; - - struct AssignmentStatement { - static constexpr auto rule = dsl::p >> - (dsl::equal_sign >> - (dsl::p | dsl::p | dsl::recurse_branch) | - dsl::else_ >> dsl::return_); - }; - - struct StatementListBlock { - static constexpr auto rule = - dsl::curly_bracketed.open() >> - dsl::opt(dsl::list(dsl::p)) + dsl::opt(dsl::semicolon) + - dsl::curly_bracketed.close(); - }; - - struct File { - // Allow arbitrary spaces between individual tokens. - static constexpr auto whitespace = whitespace_specifier | comment_specifier; - - static constexpr auto rule = dsl::terminator(dsl::eof).list(dsl::p); - }; -} diff --git a/src/openvic-dataloader/v2script/ModifierGrammar.hpp b/src/openvic-dataloader/v2script/ModifierGrammar.hpp new file mode 100644 index 0000000..ad0704a --- /dev/null +++ b/src/openvic-dataloader/v2script/ModifierGrammar.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include + +#include "SimpleGrammar.hpp" +#include "TriggerGrammar.hpp" + +namespace ovdl::v2script::grammar { + constexpr auto modifier_keyword = LEXY_KEYWORD("modifier", lexy::dsl::inline_); + constexpr auto factor_keyword = LEXY_KEYWORD("factor", lexy::dsl::inline_); + + struct FactorStatement { + static constexpr auto rule = factor_keyword >> lexy::dsl::equal_sign + lexy::dsl::inline_; + static constexpr auto value = lexy::as_string | lexy::new_; + }; + + struct ModifierStatement { + static constexpr auto rule = + modifier_keyword >> + lexy::dsl::curly_bracketed.list( + lexy::dsl::p | + lexy::dsl::p); + + static constexpr auto value = + lexy::as_list> >> + lexy::callback( + [](auto&& list) { + return make_node_ptr(LEXY_MOV(list)); + }); + }; +} \ No newline at end of file diff --git a/src/openvic-dataloader/v2script/Parser.cpp b/src/openvic-dataloader/v2script/Parser.cpp new file mode 100644 index 0000000..07d6455 --- /dev/null +++ b/src/openvic-dataloader/v2script/Parser.cpp @@ -0,0 +1,224 @@ +#include "openvic-dataloader/v2script/Parser.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "SimpleGrammar.hpp" +#include "detail/BasicBufferHandler.hpp" +#include "detail/DetectUtf8.hpp" +#include "detail/Errors.hpp" +#include "detail/LexyReportError.hpp" +#include "detail/NullBuff.hpp" +#include "detail/OStreamOutputIterator.hpp" +#include "detail/Warnings.hpp" +#include "v2script/DecisionGrammar.hpp" +#include "v2script/EventGrammar.hpp" + +using namespace ovdl; +using namespace ovdl::v2script; + +/// BufferHandler /// + +class Parser::BufferHandler final : public detail::BasicBufferHandler<> { +public: + constexpr bool is_exclusive_utf8() const { + return detail::is_utf8_no_ascii(_buffer); + } + + template + std::optional> parse(const ErrorCallback& callback) { + auto result = lexy::parse(_buffer, callback); + if (!result) { + return result.errors(); + } + // This is mighty frustrating + _root = std::unique_ptr(result.value()); + return std::nullopt; + } + + std::unique_ptr& get_root() { + return _root; + } + +private: + std::unique_ptr _root; +}; + +/// BufferHandler /// + +Parser::Parser() + : _buffer_handler(std::make_unique()) { + set_error_log_to_stderr(); +} + +Parser::Parser(Parser&&) = default; +Parser& Parser::operator=(Parser&&) = default; +Parser::~Parser() = default; + +Parser Parser::from_buffer(const char* data, std::size_t size) { + Parser result; + return std::move(result.load_from_buffer(data, size)); +} + +Parser Parser::from_buffer(const char* start, const char* end) { + Parser result; + return std::move(result.load_from_buffer(start, end)); +} + +Parser Parser::from_string(const std::string_view string) { + Parser result; + return std::move(result.load_from_string(string)); +} + +Parser Parser::from_file(const char* path) { + Parser result; + return std::move(result.load_from_file(path)); +} + +Parser Parser::from_file(const std::filesystem::path& path) { + Parser result; + return std::move(result.load_from_file(path)); +} + +/// +/// @brief Executes a function on _buffer_handler that is expected to load a buffer +/// +/// Expected Use: +/// @code {.cpp} +/// _run_load_func(&BufferHandler::, ); +/// @endcode +/// +/// @tparam Type +/// @tparam Args +/// @param func +/// @param args +/// +template +constexpr void Parser::_run_load_func(detail::LoadCallback auto func, Args... args) { + _warnings.clear(); + _errors.clear(); + _has_fatal_error = false; + if (auto error = func(_buffer_handler.get(), std::forward(args)...); error) { + _has_fatal_error = error.value().type == ParseError::Type::Fatal; + _errors.push_back(error.value()); + _error_stream.get() << "Error: " << _errors.back().message << '\n'; + } +} + +constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size) { + // Type can't be deduced? + _run_load_func(std::mem_fn(&BufferHandler::load_buffer_size), data, size); + return *this; +} + +constexpr Parser& Parser::load_from_buffer(const char* start, const char* end) { + // Type can't be deduced? + _run_load_func(std::mem_fn(&BufferHandler::load_buffer), start, end); + return *this; +} + +constexpr Parser& Parser::load_from_string(const std::string_view string) { + return load_from_buffer(string.data(), string.size()); +} + +constexpr Parser& Parser::load_from_file(const char* path) { + _file_path = path; + // Type can be deduced?? + _run_load_func(std::mem_fn(&BufferHandler::load_file), path); + return *this; +} + +Parser& Parser::load_from_file(const std::filesystem::path& path) { + return load_from_file(path.string().c_str()); +} + +constexpr Parser& Parser::load_from_file(const detail::Has_c_str auto& path) { + return load_from_file(path.c_str()); +} + +bool Parser::simple_parse() { + if (!_buffer_handler->is_valid()) { + return false; + } + + if (_buffer_handler->is_exclusive_utf8()) { + _warnings.push_back(warnings::make_utf8_warning(_file_path)); + } + + auto errors = _buffer_handler->parse(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + if (errors) { + _errors.reserve(errors->size()); + for (auto& err : errors.value()) { + _has_fatal_error |= err.type == ParseError::Type::Fatal; + _errors.push_back(err); + } + return false; + } + _file_node.reset(static_cast(_buffer_handler->get_root().release())); + return true; +} + +bool Parser::event_parse() { + if (!_buffer_handler->is_valid()) { + return false; + } + + if (_buffer_handler->is_exclusive_utf8()) { + _warnings.push_back(warnings::make_utf8_warning(_file_path)); + } + + auto errors = _buffer_handler->parse(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + if (errors) { + _errors.reserve(errors->size()); + for (auto& err : errors.value()) { + _has_fatal_error |= err.type == ParseError::Type::Fatal; + _errors.push_back(err); + } + return false; + } + _file_node.reset(static_cast(_buffer_handler->get_root().release())); + return true; +} + +bool Parser::decision_parse() { + if (!_buffer_handler->is_valid()) { + return false; + } + + if (_buffer_handler->is_exclusive_utf8()) { + _warnings.push_back(warnings::make_utf8_warning(_file_path)); + } + + auto errors = _buffer_handler->parse(ovdl::detail::ReporError.path(_file_path).to(detail::OStreamOutputIterator { _error_stream })); + if (errors) { + _errors.reserve(errors->size()); + for (auto& err : errors.value()) { + _has_fatal_error |= err.type == ParseError::Type::Fatal; + _errors.push_back(err); + } + return false; + } + _file_node.reset(static_cast(_buffer_handler->get_root().release())); + return true; +} + +const FileNode* Parser::get_file_node() const { + return _file_node.get(); +} \ No newline at end of file diff --git a/src/openvic-dataloader/v2script/SimpleGrammar.hpp b/src/openvic-dataloader/v2script/SimpleGrammar.hpp new file mode 100644 index 0000000..c91935e --- /dev/null +++ b/src/openvic-dataloader/v2script/SimpleGrammar.hpp @@ -0,0 +1,120 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include + +#include "detail/LexyLitRange.hpp" + +// Grammar Definitions // +namespace ovdl::v2script::grammar { + struct StatementListBlock; + + static constexpr auto whitespace_specifier = lexy::dsl::ascii::blank / lexy::dsl::ascii::newline; + static constexpr auto comment_specifier = LEXY_LIT("#") >> lexy::dsl::until(lexy::dsl::newline).or_eof(); + + static constexpr auto data_specifier = + lexy::dsl::ascii::alpha_digit_underscore / + LEXY_ASCII_ONE_OF("%&'") / lexy::dsl::lit_c<0x2B> / LEXY_ASCII_ONE_OF("-.") / + lexy::dsl::ascii::digit / lexy::dsl::lit_c<0x3A> / + lexy::dsl::lit_c<0x40> / lexy::dsl::ascii::upper / lexy::dsl::lit_c<0x5F> / + lexy::dsl::ascii::lower / lexy::dsl::lit_b<0x8A> / lexy::dsl::lit_b<0x8C> / lexy::dsl::lit_b<0x8E> / + lexy::dsl::lit_b<0x92> / lexy::dsl::lit_b<0x97> / lexy::dsl::lit_b<0x9A> / lexy::dsl::lit_b<0x9C> / lexy::dsl::lit_b<0x9E> / lexy::dsl::lit_b<0x9F> / + lexy::dsl::lit_b<0xC0> / + ovdl::detail::lexydsl::make_range<0xC0, 0xD6>() / ovdl::detail::lexydsl::make_range<0xD8, 0xF6>() / ovdl::detail::lexydsl::make_range<0xF8, 0xFF>(); + + static constexpr auto data_char_class = LEXY_CHAR_CLASS("DataSpecifier", data_specifier); + + struct Identifier { + static constexpr auto rule = lexy::dsl::identifier(data_char_class); + static constexpr auto value = lexy::as_string | lexy::new_; + }; + + struct StringExpression { + static constexpr auto escaped_symbols = lexy::symbol_table // + .map<'"'>('"') + .map<'\''>('\'') + .map<'\\'>('\\') + .map<'/'>('/') + .map<'b'>('\b') + .map<'f'>('\f') + .map<'n'>('\n') + .map<'r'>('\r') + .map<'t'>('\t'); + static constexpr auto rule = [] { + // Arbitrary code points that aren't control characters. + auto c = ovdl::detail::lexydsl::make_range<0x20, 0xFF>() - lexy::dsl::ascii::control; + + // Escape sequences start with a backlash. + // They either map one of the symbols, + // or a Unicode code point of the form uXXXX. + auto escape = lexy::dsl::backslash_escape // + .symbol(); + return lexy::dsl::quoted(c, escape); + }(); + + static constexpr auto value = lexy::as_string >> lexy::new_; + }; + + struct SimpleAssignmentStatement { + static constexpr auto rule = + lexy::dsl::p >> + lexy::dsl::equal_sign + + (lexy::dsl::p | lexy::dsl::p | lexy::dsl::recurse_branch); + + static constexpr auto value = lexy::callback( + [](auto name, auto&& initalizer) { + return make_node_ptr(LEXY_MOV(name), LEXY_MOV(initalizer)); + }); + }; + + struct AssignmentStatement { + static constexpr auto rule = + lexy::dsl::p >> + (lexy::dsl::equal_sign >> + (lexy::dsl::p | lexy::dsl::p | lexy::dsl::recurse_branch) | + lexy::dsl::else_ >> lexy::dsl::return_) | + lexy::dsl::p; + + static constexpr auto value = lexy::callback( + [](auto name, lexy::nullopt = {}) { + return LEXY_MOV(name); + }, + [](auto name, auto&& initalizer) { + return make_node_ptr(LEXY_MOV(name), LEXY_MOV(initalizer)); + }); + }; + + struct StatementListBlock { + static constexpr auto rule = + lexy::dsl::curly_bracketed( + lexy::dsl::opt(lexy::dsl::list(lexy::dsl::p)) + lexy::dsl::opt(lexy::dsl::semicolon)); + + static constexpr auto value = + lexy::as_list> >> + lexy::callback( + [](lexy::nullopt = {}, lexy::nullopt = {}) { + return ast::make_node_ptr(); + }, + [](auto&& list, lexy::nullopt = {}) { + return make_node_ptr(LEXY_MOV(list)); + }, + [](auto& list) { + return make_node_ptr(list); + }); + }; + + struct File { + // Allow arbitrary spaces between individual tokens. + static constexpr auto whitespace = whitespace_specifier | comment_specifier; + + static constexpr auto rule = lexy::dsl::terminator(lexy::dsl::eof).list(lexy::dsl::p); + + static constexpr auto value = lexy::as_list> >> lexy::new_; + }; +} diff --git a/src/openvic-dataloader/v2script/TriggerGrammar.hpp b/src/openvic-dataloader/v2script/TriggerGrammar.hpp new file mode 100644 index 0000000..290fb70 --- /dev/null +++ b/src/openvic-dataloader/v2script/TriggerGrammar.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include +#include + +#include "SimpleGrammar.hpp" + +namespace ovdl::v2script::grammar { + struct TriggerStatement { + static constexpr auto rule = lexy::dsl::inline_; + + static constexpr auto value = lexy::callback( + [](auto name, auto&& initalizer) { + return ast::make_node_ptr(ast::ExecutionNode::Type::Trigger, LEXY_MOV(name), LEXY_MOV(initalizer)); + }); + }; + + struct TriggerList { + static constexpr auto rule = lexy::dsl::list(lexy::dsl::p); + + static constexpr auto value = + lexy::as_list> >> + lexy::callback( + [](auto&& list) { + return ast::make_node_ptr(ast::ExecutionNode::Type::Trigger, LEXY_MOV(list)); + }); + }; + + struct TriggerBlock { + static constexpr auto rule = lexy::dsl::curly_bracketed.opt(lexy::dsl::p); + + static constexpr auto value = lexy::callback( + [](auto&& list) { + return LEXY_MOV(list); + }, + [](lexy::nullopt = {}) { + return lexy::nullopt {}; + }); + }; +} \ No newline at end of file -- cgit v1.2.3-56-ga3b1