Open Chinese Convert: StreamWindow.hpp Source File

Open Chinese Convert 1.4.2+g71621a0e1.dirty A project for conversion between Traditional and Simplified Chinese

Loading...
Searching...
No Matches
StreamWindow.hpp
1/*
2 * Open Chinese Convert
3 *
4 * Copyright 2010-2026 Carbo Kuo and contributors
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19#pragma once
20
21#include <cstddef>
22#include <string_view>
23
24#include "UTF8Util.hpp"
25
26namespace opencc {
27namespace internal {
28
36inline constexpr size_t kDefaultStreamKeepChars = 16;
37
52inline size_t FlushableByteCount(std::string_view pending,
53 size_t maxKeepChars) {
54 if (pending.empty()) {
55 return 0;
56 }
57
58 const char* bufferBegin = pending.data();
59 const char* bufferEnd = bufferBegin + pending.size();
60 const char* completeEnd = bufferBegin;
61 while (completeEnd < bufferEnd) {
62 const size_t nextCharLen = UTF8Util::NextCharLength(completeEnd);
63 if (completeEnd + nextCharLen > bufferEnd) {
64 break;
65 }
66 completeEnd += nextCharLen;
67 }
68
69 const char* keepStart = completeEnd;
70 size_t charsKept = 0;
71 while (keepStart > bufferBegin && charsKept < maxKeepChars) {
72 const size_t prevCharLen = UTF8Util::PrevCharLength(keepStart);
73 keepStart -= prevCharLen;
74 charsKept++;
75 }
76
77 const char* idsKeepStart = completeEnd;
78 const char* idsCandidate = completeEnd;
79 size_t idsCharsScanned = 0;
80 const size_t kMaxIDSCodePoints = 64;
81 while (idsCandidate > bufferBegin && idsCharsScanned < kMaxIDSCodePoints) {
82 idsCandidate -= UTF8Util::PrevCharLength(idsCandidate);
83 idsCharsScanned++;
84 if (UTF8Util::IsIncompleteIdeographicDescriptionSequencePrefix(
85 idsCandidate, completeEnd - idsCandidate)) {
86 idsKeepStart = idsCandidate;
87 }
88 }
89 if (idsKeepStart < keepStart) {
90 keepStart = idsKeepStart;
91 }
92
93 return static_cast<size_t>(keepStart - bufferBegin);
94}
95
96} // namespace internal
97} // namespace opencc
opencc::UTF8Util::PrevCharLength
static size_t PrevCharLength(const char *str)
Returns the length in byte for the previous UTF8 character.
Definition UTF8Util.hpp:82
opencc::UTF8Util::NextCharLength
static size_t NextCharLength(const char *str)
Returns the length in byte for the next UTF8 character.
Definition UTF8Util.hpp:71