diff options
Diffstat (limited to 'src/openvic-dataloader/detail/ParseHandler.cpp')
| -rw-r--r-- | src/openvic-dataloader/detail/ParseHandler.cpp | 347 |
1 files changed, 347 insertions, 0 deletions
diff --git a/src/openvic-dataloader/detail/ParseHandler.cpp b/src/openvic-dataloader/detail/ParseHandler.cpp new file mode 100644 index 0000000..3818433 --- /dev/null +++ b/src/openvic-dataloader/detail/ParseHandler.cpp @@ -0,0 +1,347 @@ +#include "ParseHandler.hpp" + +#include <algorithm> +#include <cstddef> +#include <cstdlib> +#include <string_view> +#include <type_traits> + +#include <openvic-dataloader/detail/Encoding.hpp> + +using namespace ovdl::detail; + +#ifdef _WIN32 +#include <array> +#include <cstdint> +#include <utility> + +#define WIN32_LEAN_AND_MEAN +#include <Windows.h> +#undef WIN32_LEAN_AND_MEAN +#endif + +template<size_t N> +struct LangCodeLiteral { + char value[N]; + + constexpr LangCodeLiteral(const char (&str)[N]) { + std::copy_n(str, N, value); + } + + static constexpr std::integral_constant<std::size_t, N - 1> size = {}; + + constexpr const char& operator[](std::size_t index) const noexcept { + return value[index]; + } + + constexpr operator std::string_view() const noexcept { + return std::string_view(value, size()); + } + + constexpr bool operator==(const std::string_view view) const noexcept { + return view.size() >= size() + 1 && view.starts_with(*this) && view[size()] == '_'; + } +}; + +struct LangCodeView { + std::string_view view; + bool is_valid; + + constexpr LangCodeView() = default; + + template<std::size_t N> + constexpr LangCodeView(const char (&str)[N]) : view(str), is_valid(true) {} + + constexpr LangCodeView(char* str) : view(str) { + is_valid = view.find('_') != std::string_view::npos; + } + + constexpr std::size_t size() const noexcept { + return view.size(); + } + + constexpr const char& operator[](std::size_t index) const noexcept { + return view[index]; + } + + constexpr operator std::string_view() const noexcept { + return view; + } + + template<std::size_t N> + constexpr bool operator==(const LangCodeLiteral<N>& literal) { + return is_valid && size() >= LangCodeLiteral<N>::size() && view.starts_with(literal); + } +}; + +struct FallbackSetter { + std::optional<Encoding>& fallback; + + template<Encoding _Encoding, LangCodeLiteral LangCode> + constexpr bool encoded(auto&& view) const { + if (view == LangCode) { + fallback = _Encoding; + return true; + } + return false; + }; +}; + +void ParseHandler::_detect_system_fallback_encoding() { + _system_fallback_encoding = Encoding::Unknown; + LangCodeView lang_code; + +#ifdef _WIN32 + using namespace std::string_view_literals; + + // Every Windows language id mapped to a language code according to https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/63d3d639-7fd2-4afb-abbe-0d5b5551eef8 + constexpr std::array lang_id_to_lang_code = std::to_array<std::pair<std::uint8_t, LangCodeView>>({ + { 0x0001, "ar" }, + { 0x0002, "bg" }, + { 0x0003, "ca" }, + { 0x0004, "zh" }, + { 0x0005, "cs" }, + { 0x0006, "da" }, + { 0x0007, "de" }, + { 0x0008, "el" }, + { 0x0009, "en" }, + { 0x000A, "es" }, + { 0x000B, "fi" }, + { 0x000C, "fr" }, + { 0x000D, "he" }, + { 0x000E, "hu" }, + { 0x000F, "is" }, + { 0x0010, "it" }, + { 0x0011, "ja" }, + { 0x0012, "ko" }, + { 0x0013, "nl" }, + { 0x0014, "no" }, + { 0x0015, "pl" }, + { 0x0016, "pt" }, + { 0x0017, "rm" }, + { 0x0018, "ro" }, + { 0x0019, "ru" }, + { 0x001A, "hr" }, + { 0x001B, "sk" }, + { 0x001C, "sq" }, + { 0x001D, "sv" }, + { 0x001E, "th" }, + { 0x001F, "tr" }, + { 0x0020, "ur" }, + { 0x0021, "id" }, + { 0x0022, "uk" }, + { 0x0023, "be" }, + { 0x0024, "sl" }, + { 0x0025, "et" }, + { 0x0026, "lv" }, + { 0x0027, "lt" }, + { 0x0028, "tg" }, + { 0x0029, "fa" }, + { 0x002A, "vi" }, + { 0x002B, "hy" }, + { 0x002C, "az" }, + { 0x002D, "eu" }, + { 0x002E, "hsb" }, + { 0x002F, "mk" }, + { 0x0030, "st" }, + { 0x0031, "ts" }, + { 0x0032, "tn" }, + { 0x0033, "ve" }, + { 0x0034, "xh" }, + { 0x0035, "zu" }, + { 0x0036, "af" }, + { 0x0037, "ka" }, + { 0x0038, "fo" }, + { 0x0039, "hi" }, + { 0x003A, "mt" }, + { 0x003B, "se" }, + { 0x003C, "ga" }, + { 0x003D, "yi" }, + { 0x003E, "ms" }, + { 0x003F, "kk" }, + { 0x0040, "ky" }, + { 0x0041, "sw" }, + { 0x0042, "tk" }, + { 0x0043, "uz" }, + { 0x0044, "tt" }, + { 0x0045, "bn" }, + { 0x0046, "pa" }, + { 0x0047, "gu" }, + { 0x0048, "or" }, + { 0x0049, "ta" }, + { 0x004A, "te" }, + { 0x004B, "kn" }, + { 0x004C, "ml" }, + { 0x004D, "as" }, + { 0x004E, "mr" }, + { 0x004F, "sa" }, + { 0x0050, "mn" }, + { 0x0051, "bo" }, + { 0x0052, "cy" }, + { 0x0053, "km" }, + { 0x0054, "lo" }, + { 0x0055, "my" }, + { 0x0056, "gl" }, + { 0x0057, "kok" }, + { 0x0058, "mni" }, + { 0x0059, "sd" }, + { 0x005A, "syr" }, + { 0x005B, "si" }, + { 0x005C, "chr" }, + { 0x005D, "iu" }, + { 0x005E, "am" }, + { 0x005F, "tzm" }, + { 0x0060, "ks" }, + { 0x0061, "ne" }, + { 0x0062, "fy" }, + { 0x0063, "ps" }, + { 0x0064, "fil" }, + { 0x0065, "dv" }, + { 0x0066, "bin" }, + { 0x0067, "ff" }, + { 0x0068, "ha" }, + { 0x0069, "ibb" }, + { 0x006A, "yo" }, + { 0x006B, "quz" }, + { 0x006C, "nso" }, + { 0x006D, "ba" }, + { 0x006E, "lb" }, + { 0x006F, "kl" }, + { 0x0070, "ig" }, + { 0x0071, "kr" }, + { 0x0072, "om" }, + { 0x0073, "ti" }, + { 0x0074, "gn" }, + { 0x0075, "haw" }, + { 0x0076, "la" }, + { 0x0077, "so" }, + { 0x0078, "ii" }, + { 0x0079, "pap" }, + { 0x007A, "arn" }, + { 0x007C, "moh" }, + { 0x007E, "br" }, + { 0x0080, "ug" }, + { 0x0081, "mi" }, + { 0x0082, "oc" }, + { 0x0083, "co" }, + { 0x0084, "gsw" }, + { 0x0085, "sah" }, + { 0x0086, "qut" }, + { 0x0087, "rw" }, + { 0x0088, "wo" }, + { 0x008C, "prs" }, + { 0x0091, "gd" }, + { 0x0092, "ku" }, + { 0x0093, "quc" } // + }); + +#pragma pack(push, 1) + struct LocaleStruct { + struct { + uint8_t language_id; + uint8_t country_id; + } language_country; + uint8_t sort_id : 4; + uint16_t reserved : 12; + }; +#pragma pack(pop) + + std::uint32_t locale_int = GetSystemDefaultLCID(); + LocaleStruct locale_id; + std::memcpy(&locale_id, &locale_int, sizeof(locale_id)); + // first 16 bytes are language-country id, next 4 are sort id, last 12 bytes are reserved + // first 8 are the language id, last 8 bytes are a country id + const std::uint8_t& lang_id = locale_id.language_country.language_id; + + for (const auto& map : lang_id_to_lang_code) { + if (map.first != lang_id) continue; + lang_code = map.second; + break; + } +#else + lang_code = std::getenv("LANG"); +#endif + + constexpr FallbackSetter setter { _system_fallback_encoding }; + + if (lang_code.size() < 2) { + _system_fallback_encoding = Encoding::Unknown; + return; + } + +#define WIN1251(LANG_CODE) \ + if (setter.encoded<Encoding::Windows1251, #LANG_CODE>(lang_code)) return; + +#define WIN1252(LANG_CODE) \ + if (setter.encoded<Encoding::Windows1252, #LANG_CODE>(lang_code)) return; + + // More common, prefer + WIN1252(en); + WIN1252(es); + WIN1252(fr); + WIN1252(de); + + WIN1251(ru); + + WIN1252(af); + WIN1252(sq); + WIN1252(eu); + WIN1252(br); + WIN1252(co); + WIN1252(fo); + WIN1252(gl); + WIN1252(is); + WIN1252(io); + WIN1252(ga); + WIN1252(id); + WIN1252(in); + WIN1252(it); + WIN1252(lb); + WIN1252(ms); + WIN1252(gv); + WIN1252(no); + WIN1252(oc); + WIN1252(pt); + WIN1252(gd); + WIN1252(sw); + WIN1252(fi); + WIN1252(da); + WIN1252(et); + WIN1252(tn); + WIN1252(ca); + WIN1252(rm); + WIN1252(nl); + WIN1252(sl); + WIN1252(cy); + WIN1252(hu); + + WIN1251(be); + WIN1251(uk); + WIN1251(bg); + WIN1251(kk); + WIN1251(tg); + WIN1251(sr); + WIN1251(ky); + WIN1251(mn); + WIN1251(mk); + WIN1251(mo); + + if (lang_code.size() < 3) { + return; + } + + WIN1251(mol); + + WIN1252(ast); + WIN1252(jbo); + WIN1252(gla); + WIN1252(sco); + WIN1252(sma); + WIN1252(roo); + WIN1252(swa); + WIN1252(tsn); + WIN1252(tok); + +#undef WIN1251 +#undef WIN1252 +}
\ No newline at end of file |