]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | ***************************************************************************************** | |
3 | * Copyright (C) 2006-2008 Apple Inc. All Rights Reserved. | |
4 | ***************************************************************************************** | |
5 | */ | |
6 | ||
7 | #include "unicode/utypes.h" | |
8 | ||
9 | #if !UCONFIG_NO_BREAK_ITERATION | |
10 | ||
11 | #include "unicode/urbtok.h" | |
12 | ||
13 | #include "unicode/ustring.h" | |
14 | #include "unicode/rbbi.h" | |
15 | #include "rbbirb.h" | |
16 | #include "rbbidata.h" | |
17 | #include "rbbidata57.h" | |
18 | #include "rbtok.h" | |
19 | #include "cmemory.h" | |
20 | #include "ucmndata.h" | |
21 | ||
22 | U_NAMESPACE_USE | |
23 | ||
24 | U_CAPI UBreakIterator* U_EXPORT2 | |
25 | urbtok_open(UBreakIteratorType type, | |
26 | const char *locale, | |
27 | UErrorCode *status) | |
28 | { | |
29 | UBreakIterator* result = ubrk_open(type, locale, NULL, 0, status); | |
30 | if(U_SUCCESS(*status)) { | |
31 | ((RuleBasedBreakIterator*)result)->initLatin1Cat(); | |
32 | } | |
33 | return result; | |
34 | } | |
35 | ||
36 | U_CAPI UBreakIterator* U_EXPORT2 | |
37 | urbtok_openRules(const UChar *rules, | |
38 | int32_t rulesLength, | |
39 | UParseError *parseErr, | |
40 | UErrorCode *status) | |
41 | { | |
42 | if (status == NULL || U_FAILURE(*status)){ | |
43 | return 0; | |
44 | } | |
45 | UnicodeString ruleString(rules, rulesLength); | |
46 | BreakIterator *result = RBBIRuleBuilder::createRuleBasedBreakIterator(ruleString, parseErr, *status); | |
47 | if(U_FAILURE(*status)) { | |
48 | return 0; | |
49 | } | |
50 | ((RuleBasedBreakIterator*)result)->initLatin1Cat(); | |
51 | return (UBreakIterator *)result; | |
52 | } | |
53 | ||
54 | U_CAPI UBreakIterator* U_EXPORT2 | |
55 | urbtok_openBinaryRules(const uint8_t *rules, | |
56 | UErrorCode *status) | |
57 | { | |
58 | if (status == NULL || U_FAILURE(*status)){ | |
59 | return 0; | |
60 | } | |
61 | uint32_t length = ((const RBBIDataHeader *)rules)->fLength; | |
62 | uint8_t *ruleCopy = (uint8_t *) uprv_malloc(length); | |
63 | if (ruleCopy == 0) | |
64 | { | |
65 | *status = U_MEMORY_ALLOCATION_ERROR; | |
66 | return 0; | |
67 | } | |
68 | // Copy the rules so they can be adopted by the tokenizer | |
69 | uprv_memcpy(ruleCopy, rules, length); | |
70 | // The following intended-to-be-private constructor does adopt the rules. | |
71 | BreakIterator *result = new RuleBasedBreakIterator((RBBIDataHeader *)ruleCopy, *status); | |
72 | if(U_FAILURE(*status)) { | |
73 | return 0; | |
74 | } | |
75 | ((RuleBasedBreakIterator*)result)->initLatin1Cat(); | |
76 | return (UBreakIterator *)result; | |
77 | } | |
78 | ||
79 | U_CAPI UBreakIterator* U_EXPORT2 | |
80 | urbtok_openBinaryRulesNoCopy(const uint8_t *rules, | |
81 | UErrorCode *status) | |
82 | { | |
83 | if (status == NULL || U_FAILURE(*status)){ | |
84 | return 0; | |
85 | } | |
86 | uint32_t length = ((const RBBIDataHeader *)rules)->fLength; | |
87 | ||
88 | // The following public constructor does not adopt the rules | |
89 | BreakIterator *result = new RuleBasedBreakIterator(rules, length, *status); | |
90 | if(U_FAILURE(*status)) { | |
91 | return 0; | |
92 | } | |
93 | ((RuleBasedBreakIterator*)result)->initLatin1Cat(); | |
94 | return (UBreakIterator *)result; | |
95 | } | |
96 | ||
97 | U_CAPI uint32_t U_EXPORT2 | |
98 | urbtok_getBinaryRules(UBreakIterator *bi, | |
99 | uint8_t *buffer, | |
100 | uint32_t buffSize, | |
101 | UErrorCode *status) | |
102 | { | |
103 | if (status == NULL || U_FAILURE(*status)){ | |
104 | return 0; | |
105 | } | |
106 | if (buffer == NULL && buffSize > 0) { | |
107 | *status = U_ILLEGAL_ARGUMENT_ERROR; | |
108 | return 0; | |
109 | } | |
110 | RuleBasedBreakIterator *rbbi; | |
111 | if ((rbbi = dynamic_cast<RuleBasedBreakIterator*>(reinterpret_cast<BreakIterator*>(bi))) == NULL) { | |
112 | *status = U_ILLEGAL_ARGUMENT_ERROR; | |
113 | return 0; | |
114 | } | |
115 | uint32_t length; | |
116 | const uint8_t *rules = rbbi->getBinaryRules(length); | |
117 | if (buffer != 0) | |
118 | { | |
119 | if (length > buffSize) { | |
120 | *status = U_BUFFER_OVERFLOW_ERROR; | |
121 | } | |
122 | else { | |
123 | uprv_memcpy(buffer, rules, length); | |
124 | } | |
125 | } | |
126 | return length; | |
127 | } | |
128 | ||
129 | U_CAPI int32_t U_EXPORT2 | |
130 | urbtok_tokenize(UBreakIterator *bi, | |
131 | int32_t maxTokens, | |
132 | RuleBasedTokenRange *outTokens, | |
133 | unsigned long *outTokenFlags) | |
134 | { | |
135 | // Using dynamic_cast/reinterpret_cast has a significant performance impact | |
136 | // on calls to urbtok_tokenize with maxTokens=1, use a regular cast instead | |
137 | //RuleBasedBreakIterator *rbbi; | |
138 | //if ((rbbi = dynamic_cast<RuleBasedBreakIterator*>(reinterpret_cast<BreakIterator*>(bi))) == NULL) { | |
139 | // return 0; | |
140 | //} | |
141 | //return rbbi->tokenize(maxTokens, outTokens, outTokenFlags); | |
142 | if (bi == NULL || outTokens == NULL) { | |
143 | return 0; | |
144 | } | |
145 | return ((RuleBasedBreakIterator*)bi)->tokenize(maxTokens, outTokens, outTokenFlags); | |
146 | } | |
147 | ||
148 | U_CAPI void U_EXPORT2 | |
149 | urbtok_swapBinaryRules(const uint8_t *rules, | |
150 | uint8_t *buffer, | |
151 | UBool inIsBigEndian, | |
152 | UBool outIsBigEndian, | |
153 | UErrorCode *status) | |
154 | { | |
155 | DataHeader *outH = NULL; | |
156 | uint32_t outLength = 0; | |
157 | UDataSwapper *ds = udata_openSwapper(inIsBigEndian, U_CHARSET_FAMILY, outIsBigEndian, U_CHARSET_FAMILY, status); | |
158 | ||
159 | if (status == NULL || U_FAILURE(*status)){ | |
160 | return; | |
161 | } | |
162 | ||
163 | uint32_t length = ds->readUInt32(((const RBBIDataHeader *)rules)->fLength); | |
164 | uint32_t totalLength = sizeof(DataHeader) + length; | |
165 | ||
166 | DataHeader *dh = (DataHeader *)uprv_malloc(totalLength); | |
167 | if (dh == 0) | |
168 | { | |
169 | *status = U_MEMORY_ALLOCATION_ERROR; | |
170 | goto closeSwapper; | |
171 | } | |
172 | outH = (DataHeader *)uprv_malloc(totalLength); | |
173 | if (outH == 0) | |
174 | { | |
175 | *status = U_MEMORY_ALLOCATION_ERROR; | |
176 | uprv_free(dh); | |
177 | goto closeSwapper; | |
178 | } | |
179 | dh->dataHeader.headerSize = ds->readUInt16(sizeof(DataHeader)); | |
180 | dh->dataHeader.magic1 = 0xda; | |
181 | dh->dataHeader.magic2 = 0x27; | |
182 | dh->info.size = ds->readUInt16(sizeof(UDataInfo)); | |
183 | dh->info.reservedWord = 0; | |
184 | dh->info.isBigEndian = inIsBigEndian; | |
185 | dh->info.charsetFamily = U_CHARSET_FAMILY; | |
186 | dh->info.sizeofUChar = U_SIZEOF_UCHAR; | |
187 | dh->info.reservedByte = 0; | |
188 | uprv_memcpy(dh->info.dataFormat, "Brk ", sizeof(dh->info.dataFormat)); | |
189 | uprv_memcpy(dh->info.formatVersion, ((const RBBIDataHeader *)rules)->fFormatVersion, sizeof(dh->info.formatVersion)); | |
190 | dh->info.dataVersion[0] = 4; // Unicode version | |
191 | dh->info.dataVersion[1] = 1; | |
192 | dh->info.dataVersion[2] = 0; | |
193 | dh->info.dataVersion[3] = 0; | |
194 | uprv_memcpy(((uint8_t*)dh) + sizeof(DataHeader), rules, length); | |
195 | ||
196 | outLength = (uint32_t)ubrk_swap(ds, dh, totalLength, outH, status); | |
197 | if (U_SUCCESS(*status) && outLength != totalLength) // something went horribly wrong | |
198 | { | |
199 | *status = U_INVALID_FORMAT_ERROR; | |
200 | } | |
201 | ||
202 | if (U_SUCCESS(*status)) | |
203 | { | |
204 | uprv_memcpy(buffer, ((uint8_t *)outH) + sizeof(DataHeader), length); | |
205 | } | |
206 | uprv_free(outH); | |
207 | uprv_free(dh); | |
208 | ||
209 | closeSwapper: | |
210 | udata_closeSwapper(ds); | |
211 | } | |
212 | ||
213 | U_CAPI UBreakIterator* U_EXPORT2 | |
214 | urbtok57_openRules(const UChar *rules, | |
215 | int32_t rulesLength, | |
216 | UParseError *parseErr, | |
217 | UErrorCode *status) | |
218 | { | |
219 | if (status == NULL || U_FAILURE(*status)){ | |
220 | return 0; | |
221 | } | |
222 | ||
223 | BreakIterator *result = 0; | |
224 | UnicodeString ruleString(rules, rulesLength); | |
225 | result = new RuleBasedTokenizer(ruleString, *parseErr, *status); | |
226 | if(U_FAILURE(*status)) { | |
227 | return 0; | |
228 | } | |
229 | ||
230 | UBreakIterator *uBI = (UBreakIterator *)result; | |
231 | return uBI; | |
232 | } | |
233 | ||
234 | U_CAPI UBreakIterator* U_EXPORT2 | |
235 | urbtok57_openBinaryRules(const uint8_t *rules, | |
236 | UErrorCode *status) | |
237 | { | |
238 | if (status == NULL || U_FAILURE(*status)){ | |
239 | return 0; | |
240 | } | |
241 | ||
242 | uint32_t length = ((const RBBIDataHeader57 *)rules)->fLength; | |
243 | uint8_t *ruleCopy = (uint8_t *) uprv_malloc(length); | |
244 | if (ruleCopy == 0) | |
245 | { | |
246 | *status = U_MEMORY_ALLOCATION_ERROR; | |
247 | return 0; | |
248 | } | |
249 | // Copy the rules so they can be adopted by the tokenizer | |
250 | uprv_memcpy(ruleCopy, rules, length); | |
251 | BreakIterator *result = 0; | |
252 | result = new RuleBasedTokenizer(ruleCopy, *status); | |
253 | if(U_FAILURE(*status)) { | |
254 | return 0; | |
255 | } | |
256 | ||
257 | UBreakIterator *uBI = (UBreakIterator *)result; | |
258 | return uBI; | |
259 | } | |
260 | ||
261 | U_CAPI UBreakIterator* U_EXPORT2 | |
262 | urbtok57_openBinaryRulesNoCopy(const uint8_t *rules, | |
263 | UErrorCode *status) | |
264 | { | |
265 | if (status == NULL || U_FAILURE(*status)){ | |
266 | return 0; | |
267 | } | |
268 | ||
269 | BreakIterator *result = 0; | |
270 | result = new RuleBasedTokenizer(rules, RuleBasedTokenizer::kDontAdopt, *status); | |
271 | if(U_FAILURE(*status)) { | |
272 | return 0; | |
273 | } | |
274 | ||
275 | UBreakIterator *uBI = (UBreakIterator *)result; | |
276 | return uBI; | |
277 | } | |
278 | ||
279 | U_CAPI uint32_t U_EXPORT2 | |
280 | urbtok57_getBinaryRules(UBreakIterator *bi, | |
281 | uint8_t *buffer, | |
282 | uint32_t buffSize, | |
283 | UErrorCode *status) | |
284 | { | |
285 | if (status == NULL || U_FAILURE(*status)){ | |
286 | return 0; | |
287 | } | |
288 | if (buffer == NULL && buffSize > 0) { | |
289 | *status = U_ILLEGAL_ARGUMENT_ERROR; | |
290 | return 0; | |
291 | } | |
292 | RuleBasedBreakIterator57 *rbbi57; | |
293 | if ((rbbi57 = dynamic_cast<RuleBasedBreakIterator57*>(reinterpret_cast<BreakIterator*>(bi))) == NULL) { | |
294 | *status = U_ILLEGAL_ARGUMENT_ERROR; | |
295 | return 0; | |
296 | } | |
297 | uint32_t length; | |
298 | const uint8_t *rules = rbbi57->getBinaryRules(length); | |
299 | if (buffer != 0) | |
300 | { | |
301 | if (length > buffSize) { | |
302 | *status = U_BUFFER_OVERFLOW_ERROR; | |
303 | } | |
304 | else { | |
305 | uprv_memcpy(buffer, rules, length); | |
306 | } | |
307 | } | |
308 | return length; | |
309 | } | |
310 | ||
311 | U_CAPI int32_t U_EXPORT2 | |
312 | urbtok57_tokenize(UBreakIterator *bi, | |
313 | int32_t maxTokens, | |
314 | RuleBasedTokenRange *outTokens, | |
315 | unsigned long *outTokenFlags) | |
316 | { | |
317 | return ((RuleBasedTokenizer *)bi)->tokenize(maxTokens, outTokens, outTokenFlags); | |
318 | } | |
319 | ||
320 | #endif /* #if !UCONFIG_NO_BREAK_ITERATION */ |