From b0c3ba3f91926b0c95625bdbf4aab69269130b13 Mon Sep 17 00:00:00 2001 From: Spartan322 Date: Thu, 9 May 2024 10:06:02 -0400 Subject: Add runtime encoding detection and conversion Win-1251/1252 detection is a reduced C++ version of https://github.com/hsivonen/chardetng Add manually-specified encoding fallback Add default system encoding fallback Add error recovery to v2script Add unknown encoding detection warning Remove csv::Parser templating Fix lua files dropping data Update lexy to foonathan/lexy@1e5d99fa3826b1c3c8628d3a11117fb4fb4cc0d0 Remove exclusive reliance on lexy::default_encoding for v2script Move internal concepts to src/openvic-detail/InternalConcepts.hpp Move contents of DetectUtf8.hpp to src/detail/Detect.hpp Move openvic-dataloader/AbstractSyntaxTree.hpp to src Move DiagnosticLogger.hpp to src Move File.hpp to src Move openvic-dataloader/detail/utlity files to openvic-dataloader/detail Add ovdl::utility::type_concat Add ovdl::utility::type_prepend Add ovdl::utility::is_instance_of Overhaul parse error messages --- include/openvic-dataloader/File.hpp | 69 ------------------------------------- 1 file changed, 69 deletions(-) delete mode 100644 include/openvic-dataloader/File.hpp (limited to 'include/openvic-dataloader/File.hpp') diff --git a/include/openvic-dataloader/File.hpp b/include/openvic-dataloader/File.hpp deleted file mode 100644 index caa4a0a..0000000 --- a/include/openvic-dataloader/File.hpp +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include - -#include -#include - -#include - -namespace ovdl { - template - concept IsEncoding = requires(T t) { - typename T::char_type; - typename T::int_type; - { T::template is_secondary_char_type() } -> std::same_as; - { T::eof() } -> std::same_as; - { T::to_int_type(typename T::char_type {}) } -> std::same_as; - }; - - struct File { - explicit File(const char* path); - - const char* path() const noexcept; - - protected: - const char* _path; - }; - - template - concept IsFile = - std::derived_from && IsEncoding && - requires(T t, const typename T::node_type* node, NodeLocation location) { - { t.buffer() } -> std::same_as&>; - { t.set_location(node, location) } -> std::same_as; - { t.location_of(node) } -> std::same_as; - }; - - template - struct BasicFile : File { - using encoding_type = EncodingT; - using node_type = NodeT; - - explicit BasicFile(const char* path, lexy::buffer&& buffer) - : File(path), - _buffer(static_cast&&>(buffer)) {} - - explicit BasicFile(lexy::buffer&& buffer) - : File(""), - _buffer(static_cast&&>(buffer)) {} - - const lexy::buffer& buffer() const { - return _buffer; - } - - void set_location(const node_type* n, NodeLocation loc) { - _map.insert(n, loc); - } - - NodeLocation location_of(const node_type* n) const { - auto result = _map.lookup(n); - DRYAD_ASSERT(result != nullptr, "every Node should have a NodeLocation"); - return *result; - } - - protected: - lexy::buffer _buffer; - dryad::node_map _map; - }; -} \ No newline at end of file -- cgit v1.2.3-56-ga3b1