| |
| |
| |
| |
| |
| |
| |
| |
|
|
| #ifndef _LIBCPP___FORMAT_UNICODE_H |
| #define _LIBCPP___FORMAT_UNICODE_H |
|
|
| #include <__assert> |
| #include <__bit/countl.h> |
| #include <__concepts/same_as.h> |
| #include <__config> |
| #include <__format/extended_grapheme_cluster_table.h> |
| #include <__format/indic_conjunct_break_table.h> |
| #include <__iterator/concepts.h> |
| #include <__iterator/readable_traits.h> |
| #include <__utility/unreachable.h> |
| #include <string_view> |
|
|
| #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) |
| # pragma GCC system_header |
| #endif |
|
|
| _LIBCPP_BEGIN_NAMESPACE_STD |
|
|
| #if _LIBCPP_STD_VER >= 20 |
|
|
| namespace __unicode { |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| struct __consume_result { |
| |
| |
| char32_t __code_point : 31; |
|
|
| enum : char32_t { |
| |
| __ok = 0, |
| |
| __error = 1 |
| } __status : 1 {__ok}; |
| }; |
| static_assert(sizeof(__consume_result) == sizeof(char32_t)); |
|
|
| # if _LIBCPP_HAS_UNICODE |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| inline constexpr char32_t __replacement_character = U'\ufffd'; |
|
|
| |
| |
| |
| |
| |
| inline constexpr __consume_result __consume_result_error{__replacement_character, __consume_result::__error}; |
|
|
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_high_surrogate(char32_t __value) { |
| return __value >= 0xd800 && __value <= 0xdbff; |
| } |
|
|
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_low_surrogate(char32_t __value) { |
| return __value >= 0xdc00 && __value <= 0xdfff; |
| } |
|
|
| |
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_surrogate(char32_t __value) { |
| return __value >= 0xd800 && __value <= 0xdfff; |
| } |
|
|
| |
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_code_point(char32_t __value) { |
| return __value <= 0x10ffff; |
| } |
|
|
| |
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_scalar_value(char32_t __value) { |
| return __unicode::__is_code_point(__value) && !__unicode::__is_surrogate(__value); |
| } |
|
|
| template <contiguous_iterator _Iterator> |
| requires same_as<iter_value_t<_Iterator>, char> |
| _LIBCPP_HIDE_FROM_ABI constexpr bool __is_continuation(_Iterator __char, int __count) { |
| do { |
| if ((*__char & 0b1100'0000) != 0b1000'0000) |
| return false; |
| --__count; |
| ++__char; |
| } while (__count); |
| return true; |
| } |
|
|
| |
| |
| |
| |
| template <class _CharT> |
| class __code_point_view; |
|
|
| |
| template <> |
| class __code_point_view<char> { |
| using _Iterator _LIBCPP_NODEBUG = basic_string_view<char>::const_iterator; |
|
|
| public: |
| _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last) |
| : __first_(__first), __last_(__last) {} |
|
|
| _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; } |
| _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept { |
| _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input"); |
|
|
| |
| |
| |
| switch (std::countl_one(static_cast<unsigned char>(*__first_))) { |
| case 0: |
| return {static_cast<unsigned char>(*__first_++)}; |
|
|
| case 2: { |
| if (__last_ - __first_ < 2 || !__unicode::__is_continuation(__first_ + 1, 1)) [[unlikely]] |
| break; |
|
|
| char32_t __value = static_cast<unsigned char>(*__first_++) & 0x1f; |
| __value <<= 6; |
| __value |= static_cast<unsigned char>(*__first_++) & 0x3f; |
|
|
| |
| if (__value < 0x0080) [[unlikely]] |
| return __consume_result_error; |
|
|
| return {__value}; |
| } |
|
|
| case 3: { |
| if (__last_ - __first_ < 3 || !__unicode::__is_continuation(__first_ + 1, 2)) [[unlikely]] |
| break; |
|
|
| char32_t __value = static_cast<unsigned char>(*__first_++) & 0x0f; |
| __value <<= 6; |
| __value |= static_cast<unsigned char>(*__first_++) & 0x3f; |
| __value <<= 6; |
| __value |= static_cast<unsigned char>(*__first_++) & 0x3f; |
|
|
| |
| if (__value < 0x0800) [[unlikely]] |
| return __consume_result_error; |
|
|
| |
| if (__unicode::__is_surrogate(__value)) [[unlikely]] |
| return __consume_result_error; |
|
|
| return {__value}; |
| } |
|
|
| case 4: { |
| if (__last_ - __first_ < 4 || !__unicode::__is_continuation(__first_ + 1, 3)) [[unlikely]] |
| break; |
|
|
| char32_t __value = static_cast<unsigned char>(*__first_++) & 0x07; |
| __value <<= 6; |
| __value |= static_cast<unsigned char>(*__first_++) & 0x3f; |
| __value <<= 6; |
| __value |= static_cast<unsigned char>(*__first_++) & 0x3f; |
| __value <<= 6; |
| __value |= static_cast<unsigned char>(*__first_++) & 0x3f; |
|
|
| |
| if (__value < 0x10000) [[unlikely]] |
| return __consume_result_error; |
|
|
| |
| if (!__unicode::__is_code_point(__value)) [[unlikely]] |
| return __consume_result_error; |
|
|
| return {__value}; |
| } |
| } |
| |
| |
| |
| ++__first_; |
| return __consume_result_error; |
| } |
|
|
| private: |
| _Iterator __first_; |
| _Iterator __last_; |
| }; |
|
|
| # if _LIBCPP_HAS_WIDE_CHARACTERS |
| _LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_high(wchar_t __value) { |
| return __value >= 0xd800 && __value <= 0xdbff; |
| } |
|
|
| _LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_low(wchar_t __value) { |
| return __value >= 0xdc00 && __value <= 0xdfff; |
| } |
|
|
| |
| |
| |
| template <> |
| class __code_point_view<wchar_t> { |
| using _Iterator _LIBCPP_NODEBUG = typename basic_string_view<wchar_t>::const_iterator; |
|
|
| public: |
| static_assert(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4, "sizeof(wchar_t) has a not implemented value"); |
|
|
| _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last) |
| : __first_(__first), __last_(__last) {} |
|
|
| _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; } |
| _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; } |
|
|
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept { |
| _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input"); |
|
|
| char32_t __value = static_cast<char32_t>(*__first_++); |
| if constexpr (sizeof(wchar_t) == 2) { |
| if (__unicode::__is_low_surrogate(__value)) [[unlikely]] |
| return __consume_result_error; |
|
|
| if (__unicode::__is_high_surrogate(__value)) { |
| if (__first_ == __last_ || !__unicode::__is_low_surrogate(static_cast<char32_t>(*__first_))) [[unlikely]] |
| return __consume_result_error; |
|
|
| __value -= 0xd800; |
| __value <<= 10; |
| __value += static_cast<char32_t>(*__first_++) - 0xdc00; |
| __value += 0x10000; |
|
|
| if (!__unicode::__is_code_point(__value)) [[unlikely]] |
| return __consume_result_error; |
| } |
| } else { |
| if (!__unicode::__is_scalar_value(__value)) [[unlikely]] |
| return __consume_result_error; |
| } |
|
|
| return {__value}; |
| } |
|
|
| private: |
| _Iterator __first_; |
| _Iterator __last_; |
| }; |
| # endif |
|
|
| |
| |
| |
| |
| |
| class __extended_grapheme_cluster_break { |
| using __EGC_property _LIBCPP_NODEBUG = __extended_grapheme_custer_property_boundary::__property; |
| using __inCB_property _LIBCPP_NODEBUG = __indic_conjunct_break::__property; |
|
|
| public: |
| _LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_break(char32_t __first_code_point) |
| : __prev_code_point_(__first_code_point), |
| __prev_property_(__extended_grapheme_custer_property_boundary::__get_property(__first_code_point)) { |
| |
| if (__prev_property_ == __EGC_property::__Extended_Pictographic) |
| __active_rule_ = __rule::__GB11_emoji; |
| else if (__prev_property_ == __EGC_property::__Regional_Indicator) |
| __active_rule_ = __rule::__GB12_GB13_regional_indicator; |
| else if (__indic_conjunct_break::__get_property(__first_code_point) == __inCB_property::__Consonant) |
| __active_rule_ = __rule::__GB9c_indic_conjunct_break; |
| } |
|
|
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(char32_t __next_code_point) { |
| __EGC_property __next_property = __extended_grapheme_custer_property_boundary::__get_property(__next_code_point); |
| bool __result = __evaluate(__next_code_point, __next_property); |
| __prev_code_point_ = __next_code_point; |
| __prev_property_ = __next_property; |
| return __result; |
| } |
|
|
| |
| |
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr char32_t __current_code_point() const { return __prev_code_point_; } |
|
|
| private: |
| |
| |
|
|
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool |
| __evaluate(char32_t __next_code_point, __EGC_property __next_property) { |
| switch (__active_rule_) { |
| case __rule::__none: |
| return __evaluate_none(__next_code_point, __next_property); |
| case __rule::__GB9c_indic_conjunct_break: |
| return __evaluate_GB9c_indic_conjunct_break(__next_code_point, __next_property); |
| case __rule::__GB11_emoji: |
| return __evaluate_GB11_emoji(__next_code_point, __next_property); |
| case __rule::__GB12_GB13_regional_indicator: |
| return __evaluate_GB12_GB13_regional_indicator(__next_code_point, __next_property); |
| } |
| __libcpp_unreachable(); |
| } |
|
|
| _LIBCPP_HIDE_FROM_ABI constexpr bool __evaluate_none(char32_t __next_code_point, __EGC_property __next_property) { |
| |
|
|
| _LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__sot, "should be handled in the constructor"); |
| _LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__eot, "should be handled by our caller"); |
|
|
| |
| if (__prev_property_ == __EGC_property::__CR && __next_property == __EGC_property::__LF) |
| return false; |
|
|
| if (__prev_property_ == __EGC_property::__Control || __prev_property_ == __EGC_property::__CR || |
| __prev_property_ == __EGC_property::__LF) |
| return true; |
|
|
| if (__next_property == __EGC_property::__Control || __next_property == __EGC_property::__CR || |
| __next_property == __EGC_property::__LF) |
| return true; |
|
|
| |
| if (__prev_property_ == __EGC_property::__L && |
| (__next_property == __EGC_property::__L || __next_property == __EGC_property::__V || |
| __next_property == __EGC_property::__LV || __next_property == __EGC_property::__LVT)) |
| return false; |
|
|
| if ((__prev_property_ == __EGC_property::__LV || __prev_property_ == __EGC_property::__V) && |
| (__next_property == __EGC_property::__V || __next_property == __EGC_property::__T)) |
| return false; |
|
|
| if ((__prev_property_ == __EGC_property::__LVT || __prev_property_ == __EGC_property::__T) && |
| __next_property == __EGC_property::__T) |
| return false; |
|
|
| |
| if (__next_property == __EGC_property::__Extend || __next_property == __EGC_property::__ZWJ) |
| return false; |
|
|
| |
| if (__next_property == __EGC_property::__SpacingMark) |
| return false; |
|
|
| if (__prev_property_ == __EGC_property::__Prepend) |
| return false; |
|
|
| |
| if (__indic_conjunct_break::__get_property(__next_code_point) == __inCB_property::__Consonant) { |
| __active_rule_ = __rule::__GB9c_indic_conjunct_break; |
| __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant; |
| return true; |
| } |
|
|
| |
| if (__next_property == __EGC_property::__Extended_Pictographic) { |
| __active_rule_ = __rule::__GB11_emoji; |
| __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic; |
| return true; |
| } |
|
|
| |
|
|
| |
| |
| if (__next_property == __EGC_property::__Regional_Indicator) { |
| __active_rule_ = __rule::__GB12_GB13_regional_indicator; |
| return true; |
| } |
|
|
| |
| return true; |
| } |
|
|
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool |
| __evaluate_GB9c_indic_conjunct_break(char32_t __next_code_point, __EGC_property __next_property) { |
| __inCB_property __break = __indic_conjunct_break::__get_property(__next_code_point); |
| if (__break == __inCB_property::__none) { |
| __active_rule_ = __rule::__none; |
| return __evaluate_none(__next_code_point, __next_property); |
| } |
|
|
| switch (__GB9c_indic_conjunct_break_state_) { |
| case __GB9c_indic_conjunct_break_state::__Consonant: |
| if (__break == __inCB_property::__Extend) { |
| return false; |
| } |
| if (__break == __inCB_property::__Linker) { |
| __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Linker; |
| return false; |
| } |
| __active_rule_ = __rule::__none; |
| return __evaluate_none(__next_code_point, __next_property); |
|
|
| case __GB9c_indic_conjunct_break_state::__Linker: |
| if (__break == __inCB_property::__Extend) { |
| return false; |
| } |
| if (__break == __inCB_property::__Linker) { |
| return false; |
| } |
| if (__break == __inCB_property::__Consonant) { |
| __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant; |
| return false; |
| } |
| __active_rule_ = __rule::__none; |
| return __evaluate_none(__next_code_point, __next_property); |
| } |
| __libcpp_unreachable(); |
| } |
|
|
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool |
| __evaluate_GB11_emoji(char32_t __next_code_point, __EGC_property __next_property) { |
| switch (__GB11_emoji_state_) { |
| case __GB11_emoji_state::__Extended_Pictographic: |
| if (__next_property == __EGC_property::__Extend) { |
| __GB11_emoji_state_ = __GB11_emoji_state::__Extend; |
| return false; |
| } |
| [[fallthrough]]; |
| case __GB11_emoji_state::__Extend: |
| if (__next_property == __EGC_property::__ZWJ) { |
| __GB11_emoji_state_ = __GB11_emoji_state::__ZWJ; |
| return false; |
| } |
| if (__next_property == __EGC_property::__Extend) |
| return false; |
| __active_rule_ = __rule::__none; |
| return __evaluate_none(__next_code_point, __next_property); |
|
|
| case __GB11_emoji_state::__ZWJ: |
| if (__next_property == __EGC_property::__Extended_Pictographic) { |
| __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic; |
| return false; |
| } |
| __active_rule_ = __rule::__none; |
| return __evaluate_none(__next_code_point, __next_property); |
| } |
| __libcpp_unreachable(); |
| } |
|
|
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool |
| __evaluate_GB12_GB13_regional_indicator(char32_t __next_code_point, __EGC_property __next_property) { |
| __active_rule_ = __rule::__none; |
| if (__next_property == __EGC_property::__Regional_Indicator) |
| return false; |
| return __evaluate_none(__next_code_point, __next_property); |
| } |
|
|
| char32_t __prev_code_point_; |
| __EGC_property __prev_property_; |
|
|
| enum class __rule { |
| __none, |
| __GB9c_indic_conjunct_break, |
| __GB11_emoji, |
| __GB12_GB13_regional_indicator, |
| }; |
| __rule __active_rule_ = __rule::__none; |
|
|
| enum class __GB11_emoji_state { |
| __Extended_Pictographic, |
| __Extend, |
| __ZWJ, |
| }; |
| __GB11_emoji_state __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic; |
|
|
| enum class __GB9c_indic_conjunct_break_state { |
| __Consonant, |
| __Linker, |
| }; |
|
|
| __GB9c_indic_conjunct_break_state __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant; |
|
|
| |
| }; |
|
|
| |
| |
| |
| |
| |
| template <class _CharT> |
| class __extended_grapheme_cluster_view { |
| using _Iterator _LIBCPP_NODEBUG = typename basic_string_view<_CharT>::const_iterator; |
|
|
| public: |
| _LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_view(_Iterator __first, _Iterator __last) |
| : __code_point_view_(__first, __last), __at_break_(__code_point_view_.__consume().__code_point) {} |
|
|
| struct __cluster { |
| |
| |
| |
| |
| char32_t __code_point_; |
|
|
| |
| |
| |
| |
| _Iterator __last_; |
| }; |
|
|
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __cluster __consume() { |
| char32_t __code_point = __at_break_.__current_code_point(); |
| _Iterator __position = __code_point_view_.__position(); |
| while (!__code_point_view_.__at_end()) { |
| if (__at_break_(__code_point_view_.__consume().__code_point)) |
| break; |
| __position = __code_point_view_.__position(); |
| } |
| return {__code_point, __position}; |
| } |
|
|
| private: |
| __code_point_view<_CharT> __code_point_view_; |
| __extended_grapheme_cluster_break __at_break_; |
| }; |
|
|
| template <contiguous_iterator _Iterator> |
| __extended_grapheme_cluster_view(_Iterator, _Iterator) -> __extended_grapheme_cluster_view<iter_value_t<_Iterator>>; |
|
|
| # else |
|
|
| |
| |
| template <class _CharT> |
| class __code_point_view { |
| using _Iterator _LIBCPP_NODEBUG = typename basic_string_view<_CharT>::const_iterator; |
|
|
| public: |
| _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last) |
| : __first_(__first), __last_(__last) {} |
|
|
| _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; } |
| _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; } |
|
|
| [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept { |
| _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input"); |
| return {static_cast<char32_t>(*__first_++)}; |
| } |
|
|
| private: |
| _Iterator __first_; |
| _Iterator __last_; |
| }; |
|
|
| # endif |
|
|
| } |
|
|
| #endif |
|
|
| _LIBCPP_END_NAMESPACE_STD |
|
|
| #endif |
|
|