Open Chinese Convert: UTF8Util.hpp Source File

Open Chinese Convert 1.4.2+g71621a0e1.dirty A project for conversion between Traditional and Simplified Chinese

Loading...
Searching...
No Matches
UTF8Util.hpp
1/*
2 * Open Chinese Convert
3 *
4 * Copyright 2013 Carbo Kuo <byvoid@byvoid.com>
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19#pragma once
20
21#ifdef _MSC_VER
22#ifndef NOMINMAX
23#define NOMINMAX
24#endif
25#include <windows.h>
26#endif // _MSC_VER
27
28#include <cstdint>
29#include <cstring>
30
31#include "Common.hpp"
32#include "Exception.hpp"
33
34namespace opencc {
39 class OPENCC_EXPORT UTF8Util {
40public:
44 static void SkipUtf8Bom(FILE* fp);
45
50 static size_t NextCharLengthNoException(const char* str) {
51 const unsigned char ch = static_cast<unsigned char>(*str);
52 if ((ch & 0xF0) == 0xE0) {
53 return 3;
54 } else if ((ch & 0x80) == 0x00) {
55 return 1;
56 } else if ((ch & 0xE0) == 0xC0) {
57 return 2;
58 } else if ((ch & 0xF8) == 0xF0) {
59 return 4;
60 } else if ((ch & 0xFC) == 0xF8) {
61 return 5;
62 } else if ((ch & 0xFE) == 0xFC) {
63 return 6;
64 }
65 return 0;
66 }
67
71 static size_t NextCharLength(const char* str) {
72 size_t length = NextCharLengthNoException(str);
73 if (length == 0) {
74 throw InvalidUTF8(str);
75 }
76 return length;
77 }
78
82 static size_t PrevCharLength(const char* str) {
83 const char* candidate = str - 1;
84 size_t distance = 1;
85 while (distance < 6) {
86 const unsigned char ch = static_cast<unsigned char>(*candidate);
87 if ((ch & 0xC0) != 0x80) {
88 break;
89 }
90 candidate--;
91 distance++;
92 }
93
94 const size_t length = NextCharLengthNoException(candidate);
95 if (length == distance) {
96 return length;
97 }
98 throw InvalidUTF8(str);
99 }
100
104 static const char* NextChar(const char* str) {
105 return str + NextCharLength(str);
106 }
107
111 static const char* PrevChar(const char* str) {
112 return str - PrevCharLength(str);
113 }
114
121 static const uint32_t kFirstIdeographicDescriptionOperator = 0x2FF0;
122 static const uint32_t kLastIdeographicDescriptionOperator = 0x2FFF;
123
124 static size_t IdeographicDescriptionOperatorArity(uint32_t codePoint) {
125 switch (codePoint) {
126 case 0x2FF2:
127 case 0x2FF3:
128 return 3;
129 case 0x2FFE:
130 case 0x2FFF:
131 return 1;
132 case 0x2FF0:
133 case 0x2FF1:
134 case 0x2FF4:
135 case 0x2FF5:
136 case 0x2FF6:
137 case 0x2FF7:
138 case 0x2FF8:
139 case 0x2FF9:
140 case 0x2FFA:
141 case 0x2FFB:
142 case 0x2FFC:
143 case 0x2FFD:
144 return 2;
145 default:
146 return 0;
147 }
148 }
149
150 static size_t NextIdeographicDescriptionSequenceLength(const char* str,
151 size_t len) {
152 const size_t kMaxIDSDepth = 16;
153 const size_t kMaxIDSCodePoints = 64;
154 if (len == 0) {
155 return 0;
156 }
157 const size_t charLen = NextCharLengthNoException(str);
158 if (charLen == 0 || charLen > len) {
159 return 0;
160 }
161 const uint32_t codePoint = CodePointNoException(str, charLen);
162 if (IdeographicDescriptionOperatorArity(codePoint) == 0) {
163 return 0;
164 }
165
166 size_t consumed = 0;
167 size_t codePoints = 0;
168 if (ConsumeIdeographicDescriptionSequence(
169 str, len, kMaxIDSDepth, kMaxIDSCodePoints, &consumed,
170 &codePoints) == IDSParseStatus::Complete) {
171 return consumed;
172 }
173 return 0;
174 }
175
176 static bool IsIncompleteIdeographicDescriptionSequencePrefix(const char* str,
177 size_t len) {
178 const size_t kMaxIDSDepth = 16;
179 const size_t kMaxIDSCodePoints = 64;
180 if (len == 0) {
181 return false;
182 }
183 const size_t charLen = NextCharLengthNoException(str);
184 if (charLen == 0 || charLen > len) {
185 return false;
186 }
187 const uint32_t codePoint = CodePointNoException(str, charLen);
188 if (IdeographicDescriptionOperatorArity(codePoint) == 0) {
189 return false;
190 }
191
192 size_t consumed = 0;
193 size_t codePoints = 0;
194 return ConsumeIdeographicDescriptionSequence(
195 str, len, kMaxIDSDepth, kMaxIDSCodePoints, &consumed,
196 &codePoints) == IDSParseStatus::Incomplete;
197 }
198
199 static bool IsVariationSelector(uint32_t codePoint) {
200 return (codePoint >= 0xFE00 && codePoint <= 0xFE0F) ||
201 (codePoint >= 0xE0100 && codePoint <= 0xE01EF);
202 }
203
204 static bool ContainsVariationSelector(const char* str, size_t len) {
205 const char* pStr = str;
206 const char* strEnd = str + len;
207 while (pStr < strEnd) {
208 const size_t remainingLength = strEnd - pStr;
209 const size_t charLen = NextCharLengthNoException(pStr);
210 if (charLen == 0) {
211 ++pStr;
212 continue;
213 }
214 if (charLen > remainingLength) {
215 return false;
216 }
217 if (IsVariationSelector(CodePointNoException(pStr, charLen))) {
218 return true;
219 }
220 pStr += charLen;
221 }
222 return false;
223 }
224
231 static size_t Length(const char* str) {
232 size_t length = 0;
233 while (*str != '\0') {
234 const size_t charLen = NextCharLengthNoException(str);
235 if (charLen == 0) {
236 throw InvalidUTF8(str);
237 }
238 // Verify all continuation bytes are present before the null terminator.
239 // Use a while loop (not a for-with-return) to avoid complex control flow
240 // that triggers MSVC LTCG code-generator bugs.
241 size_t i = 1;
242 while (i < charLen && str[i] != '\0') {
243 ++i;
244 }
245 if (i < charLen) {
246 throw InvalidUTF8(str); // Truncated sequence: throw, don't silently skip
247 }
248 str += charLen;
249 ++length;
250 }
251 return length;
252 }
253
260 static const char* FindNextInline(const char* str, const char ch) {
261 while (!IsLineEndingOrFileEnding(*str) && *str != ch) {
262 str = NextChar(str);
263 }
264 return str;
265 }
266
270 static bool IsLineEndingOrFileEnding(const char ch) {
271 return ch == '\0' || ch == '\n' || ch == '\r';
272 }
273
277 static std::string FromSubstr(const char* str, size_t length) {
278 std::string newStr;
279 newStr.resize(length);
280 memcpy(newStr.data(), str, length);
281 return newStr;
282 }
283
288 static bool NotShorterThan(const char* str, size_t byteLength) {
289 while (byteLength > 0) {
290 if (*str == '\0') {
291 return false;
292 }
293 byteLength--;
294 str++;
295 }
296 return true;
297 }
298
303 static std::string TruncateUTF8(const char* str, size_t maxByteLength) {
304 std::string wordTrunc;
305 if (NotShorterThan(str, maxByteLength)) {
306 size_t len = 0;
307 const char* pStr = str;
308 for (;;) {
309 const size_t charLength = NextCharLength(pStr);
310 if (len + charLength > maxByteLength) {
311 break;
312 }
313 pStr += charLength;
314 len += charLength;
315 }
316 wordTrunc = FromSubstr(str, len);
317 } else {
318 wordTrunc = str;
319 }
320 return wordTrunc;
321 }
322
326 static void ReplaceAll(std::string& str, const char* from, const char* to) {
327 std::string::size_type pos = 0;
328 std::string::size_type fromLen = strlen(from);
329 std::string::size_type toLen = strlen(to);
330 while ((pos = str.find(from, pos)) != std::string::npos) {
331 str.replace(pos, fromLen, to);
332 pos += toLen;
333 }
334 }
335
339 static std::string Join(const std::vector<std::string>& strings,
340 const std::string& separator) {
341 std::ostringstream buffer;
342 bool first = true;
343 for (const auto& str : strings) {
344 if (!first) {
345 buffer << separator;
346 }
347 buffer << str;
348 first = false;
349 }
350 return buffer.str();
351 }
352
356 static std::string Join(const std::vector<std::string>& strings) {
357 std::ostringstream buffer;
358 for (const auto& str : strings) {
359 buffer << str;
360 }
361 return buffer.str();
362 }
363
364 static void GetByteMap(const char* str, const size_t utf8Length,
365 std::vector<size_t>* byteMap) {
366 if (byteMap->size() < utf8Length) {
367 byteMap->resize(utf8Length);
368 }
369 const char* pstr = str;
370 for (size_t i = 0; i < utf8Length; i++) {
371 (*byteMap)[i] = pstr - str;
372 pstr = NextChar(pstr);
373 }
374 }
375
376#ifdef _MSC_VER
377 static std::wstring GetPlatformString(const std::string& str) {
378 return U8ToU16(str);
379 }
380#else
381 static std::string GetPlatformString(const std::string& str) { return str; }
382#endif // _MSC_VER
383
384#ifdef _MSC_VER
385 static std::string U16ToU8(const std::wstring& wstr) {
386 std::string ret;
387 int length = static_cast<int>(wstr.length());
388 int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0,
389 NULL, NULL);
390 if (convcnt > 0) {
391 ret.resize(convcnt);
392 WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt,
393 NULL, NULL);
394 }
395 return ret;
396 }
397
398 static std::wstring U8ToU16(const std::string& str) {
399 std::wstring ret;
400 int length = static_cast<int>(str.length());
401 int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0);
402 if (convcnt > 0) {
403 ret.resize(convcnt);
404 MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt);
405 }
406 return ret;
407 }
408#endif // _MSC_VER
409
410private:
411 enum class IDSParseStatus {
412 Complete,
413 Incomplete,
414 Invalid,
415 };
416
417 static uint32_t CodePointNoException(const char* str, size_t charLen) {
418 const unsigned char first = static_cast<unsigned char>(str[0]);
419 if (charLen == 1) {
420 return first;
421 }
422
423 uint32_t codePoint = first & ((1U << (7 - charLen)) - 1);
424 for (size_t i = 1; i < charLen; i++) {
425 codePoint = (codePoint << 6) |
426 (static_cast<unsigned char>(str[i]) & 0x3FU);
427 }
428 return codePoint;
429 }
430
431 static IDSParseStatus ConsumeIdeographicDescriptionSequence(
432 const char* str, size_t len, size_t depthLeft, size_t maxCodePoints,
433 size_t* consumed, size_t* codePoints) {
434 if (len == 0) {
435 return IDSParseStatus::Incomplete;
436 }
437 if (depthLeft == 0 || *codePoints >= maxCodePoints) {
438 return IDSParseStatus::Invalid;
439 }
440 const size_t charLen = NextCharLengthNoException(str);
441 if (charLen == 0) {
442 return IDSParseStatus::Invalid;
443 }
444 if (charLen > len) {
445 return IDSParseStatus::Incomplete;
446 }
447 ++(*codePoints);
448
449 const uint32_t codePoint = CodePointNoException(str, charLen);
450 const size_t arity = IdeographicDescriptionOperatorArity(codePoint);
451 if (arity == 0) {
452 *consumed = charLen;
453 return IDSParseStatus::Complete;
454 }
455
456 size_t offset = charLen;
457 for (size_t i = 0; i < arity; i++) {
458 if (offset >= len) {
459 return IDSParseStatus::Incomplete;
460 }
461 size_t operandLength = 0;
462 const IDSParseStatus operandStatus = ConsumeIdeographicDescriptionSequence(
463 str + offset, len - offset, depthLeft - 1, maxCodePoints,
464 &operandLength, codePoints);
465 if (operandStatus != IDSParseStatus::Complete) {
466 return operandStatus;
467 }
468 offset += operandLength;
469 }
470 *consumed = offset;
471 return IDSParseStatus::Complete;
472 }
473};
474} // namespace opencc
opencc::InvalidUTF8
Definition Exception.hpp:77
opencc::UTF8Util
UTF8 std::string utilities.
Definition UTF8Util.hpp:39
opencc::UTF8Util::IsLineEndingOrFileEnding
static bool IsLineEndingOrFileEnding(const char ch)
Returns true if the character is a line ending or end of file.
Definition UTF8Util.hpp:270
opencc::UTF8Util::PrevCharLength
static size_t PrevCharLength(const char *str)
Returns the length in byte for the previous UTF8 character.
Definition UTF8Util.hpp:82
opencc::UTF8Util::FromSubstr
static std::string FromSubstr(const char *str, size_t length)
Copies a substring with given length to a new string.
Definition UTF8Util.hpp:277
opencc::UTF8Util::ReplaceAll
static void ReplaceAll(std::string &str, const char *from, const char *to)
Replaces all patterns in a std::string in place.
Definition UTF8Util.hpp:326
opencc::UTF8Util::SkipUtf8Bom
static void SkipUtf8Bom(FILE *fp)
Detect UTF8 BOM and skip it.
Definition UTF8Util.cpp:23
opencc::UTF8Util::NextCharLengthNoException
static size_t NextCharLengthNoException(const char *str)
Returns the length in byte for the next UTF8 character.
Definition UTF8Util.hpp:50
opencc::UTF8Util::NotShorterThan
static bool NotShorterThan(const char *str, size_t byteLength)
Returns true if the given std::string is longer or as long as the given length.
Definition UTF8Util.hpp:288
opencc::UTF8Util::Join
static std::string Join(const std::vector< std::string > &strings)
Joins a std::string vector in to a std::string.
Definition UTF8Util.hpp:356
opencc::UTF8Util::TruncateUTF8
static std::string TruncateUTF8(const char *str, size_t maxByteLength)
Truncates a std::string with a maximal length in byte.
Definition UTF8Util.hpp:303
opencc::UTF8Util::Length
static size_t Length(const char *str)
Returns the UTF8 length of a null-terminated string.
Definition UTF8Util.hpp:231
opencc::UTF8Util::FindNextInline
static const char * FindNextInline(const char *str, const char ch)
Finds a character in the same line.
Definition UTF8Util.hpp:260
opencc::UTF8Util::NextCharLength
static size_t NextCharLength(const char *str)
Returns the length in byte for the next UTF8 character.
Definition UTF8Util.hpp:71
opencc::UTF8Util::kFirstIdeographicDescriptionOperator
static const uint32_t kFirstIdeographicDescriptionOperator
Code point bounds of the ideographic description operators recognized by IdeographicDescriptionOperat...
Definition UTF8Util.hpp:121
opencc::UTF8Util::Join
static std::string Join(const std::vector< std::string > &strings, const std::string &separator)
Joins a std::string vector in to a std::string with a separator.
Definition UTF8Util.hpp:339
opencc::UTF8Util::PrevChar
static const char * PrevChar(const char *str)
Move the char* pointer before the previous UTF8 character.
Definition UTF8Util.hpp:111
opencc::UTF8Util::NextChar
static const char * NextChar(const char *str)
Returns the char* pointer over the next UTF8 character.
Definition UTF8Util.hpp:104