Open Chinese Convert: Utf8SkipScan.hpp Source File

Open Chinese Convert 1.4.2+g71621a0e1.dirty A project for conversion between Traditional and Simplified Chinese

Loading...
Searching...
No Matches
Utf8SkipScan.hpp
1/*
2 * Open Chinese Convert
3 *
4 * Copyright 2010-2026 Carbo Kuo and contributors
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19#pragma once
20
21#include <cassert>
22#include <cstddef>
23#include <cstdint>
24#include <cstring>
25#include <vector>
26
27#include "UTF8Util.hpp"
28
29namespace opencc {
30namespace internal {
31
39inline uint32_t DecodeCodePoint23(const char* str, size_t charLength) {
40 const unsigned char lead = static_cast<unsigned char>(str[0]);
41 if (charLength == 2) {
42 return ((lead & 0x1FU) << 6) |
43 (static_cast<unsigned char>(str[1]) & 0x3FU);
44 }
45 return ((lead & 0x0FU) << 12) |
46 ((static_cast<unsigned char>(str[1]) & 0x3FU) << 6) |
47 (static_cast<unsigned char>(str[2]) & 0x3FU);
48}
49
56 struct Utf8SkipTable {
62 bool candidate[256] = {};
67 bool asciiHasCandidates = false;
77 std::vector<uint64_t> bmpCandidates;
78
79 bool CharLevel() const { return !bmpCandidates.empty(); }
80
81 void EnableCharLevel() { bmpCandidates.assign(0x10000 / 64, 0); }
82
88 void DisableCharLevel() { bmpCandidates.clear(); }
89
90 void MarkCharCandidate(uint32_t codePoint) {
91 if (CharLevel() && codePoint < 0x10000) {
92 bmpCandidates[codePoint >> 6] |= uint64_t(1) << (codePoint & 63);
93 }
94 }
95
96 bool IsCharCandidate(uint32_t codePoint) const {
97 assert(CharLevel());
98 return (bmpCandidates[codePoint >> 6] >> (codePoint & 63)) & 1;
99 }
100
101 void MarkAllCandidates() {
102 for (size_t b = 0; b < 256; b++) {
103 candidate[b] = true;
104 }
105 DisableCharLevel();
106 }
107
116 void Finalize() {
117 static_assert(UTF8Util::kFirstIdeographicDescriptionOperator >= 0x0800 &&
118 UTF8Util::kLastIdeographicDescriptionOperator <= 0xFFFF,
119 "IDS operator range must lie in the 3-byte UTF-8 block");
120 for (uint32_t cp = UTF8Util::kFirstIdeographicDescriptionOperator;
121 cp <= UTF8Util::kLastIdeographicDescriptionOperator; cp++) {
122 if (UTF8Util::IdeographicDescriptionOperatorArity(cp) == 0) {
123 continue;
124 }
125 // The operator range lies in the 3-byte UTF-8 block of the BMP.
126 candidate[0xE0 | (cp >> 12)] = true;
127 MarkCharCandidate(cp);
128 }
129 asciiHasCandidates = false;
130 for (size_t b = 0; b < 0x80; b++) {
131 if (candidate[b]) {
132 asciiHasCandidates = true;
133 break;
134 }
135 }
136 }
137};
138
150inline size_t AsciiRunLength(const char* str, size_t len) {
151 size_t pos = 0;
152 for (; pos + sizeof(uint64_t) <= len; pos += sizeof(uint64_t)) {
153 uint64_t word;
154 std::memcpy(&word, str + pos, sizeof(word));
155 if ((word & UINT64_C(0x8080808080808080)) != 0) {
156 break;
157 }
158 }
159 // Resolve the exact position within the word that stopped the loop (at
160 // most 8 iterations), and handle the trailing partial word.
161 for (; pos < len; pos++) {
162 if (static_cast<unsigned char>(str[pos]) >= 0x80) {
163 break;
164 }
165 }
166 return pos;
167}
168
176inline size_t SkipNonCandidateBytes(const Utf8SkipTable& table, const char* str,
177 size_t len) {
178 size_t pos = 0;
179 while (pos < len) {
180 const unsigned char lead = static_cast<unsigned char>(str[pos]);
181 if (lead < 0x80) {
182 if (!table.asciiHasCandidates) {
183 pos += AsciiRunLength(str + pos, len - pos);
184 continue;
185 }
186 if (table.candidate[lead]) {
187 break;
188 }
189 pos++;
190 continue;
191 }
192 const size_t charLength = UTF8Util::NextCharLengthNoException(str + pos);
193 if (charLength == 0 || charLength > len - pos) {
194 break;
195 }
196 if (table.CharLevel() && (charLength == 2 || charLength == 3)) {
197 // DecodeCodePoint23 does not validate continuation bytes; the table
198 // builder decodes key bytes with the same function, so a position is
199 // skipped only when its exact bytes cannot begin any key. Either way
200 // the scan consumes the same charLength bytes the per-character path
201 // would, so behavior stays equivalent even on malformed input.
202 if (table.IsCharCandidate(DecodeCodePoint23(str + pos, charLength))) {
203 break;
204 }
205 } else if (table.candidate[lead]) {
206 break;
207 }
208 pos += charLength;
209 }
210 return pos;
211}
212
213} // namespace internal
214} // namespace opencc
opencc::UTF8Util::NextCharLengthNoException
static size_t NextCharLengthNoException(const char *str)
Returns the length in byte for the next UTF8 character.
Definition UTF8Util.hpp:50
opencc::UTF8Util::kFirstIdeographicDescriptionOperator
static const uint32_t kFirstIdeographicDescriptionOperator
Code point bounds of the ideographic description operators recognized by IdeographicDescriptionOperat...
Definition UTF8Util.hpp:121
opencc::internal::Utf8SkipTable
Per-byte lookup table describing which byte values may begin a dictionary key.
Definition Utf8SkipScan.hpp:56
opencc::internal::Utf8SkipTable::candidate
bool candidate[256]
candidate[b] is true when byte value b may begin a dictionary key (or is the lead byte of an ideograp...
Definition Utf8SkipScan.hpp:62
opencc::internal::Utf8SkipTable::bmpCandidates
std::vector< uint64_t > bmpCandidates
Optional character-level refinement: one bit per BMP code point (U+0000..U+FFFF, 8 KiB).
Definition Utf8SkipScan.hpp:77
opencc::internal::Utf8SkipTable::asciiHasCandidates
bool asciiHasCandidates
True when any ASCII byte value is a candidate; disables the bulk ASCII-run scan.
Definition Utf8SkipScan.hpp:67
opencc::internal::Utf8SkipTable::DisableCharLevel
void DisableCharLevel()
Permanently falls back to lead-byte filtering, e.g.
Definition Utf8SkipScan.hpp:88
opencc::internal::Utf8SkipTable::Finalize
void Finalize()
Must be called after candidate[] is filled and before the table is used.
Definition Utf8SkipScan.hpp:116