]>
Commit | Line | Data |
---|---|---|
b75a7d8f A |
1 | /* |
2 | ******************************************************************************* | |
3 | * | |
4388f060 | 4 | * Copyright (C) 2003-2011, International Business Machines |
b75a7d8f A |
5 | * Corporation and others. All Rights Reserved. |
6 | * | |
7 | ******************************************************************************* | |
8 | * file name: testidn.cpp | |
9 | * encoding: US-ASCII | |
10 | * tab size: 8 (not used) | |
11 | * indentation:4 | |
12 | * | |
13 | * created on: 2003-02-06 | |
14 | * created by: Ram Viswanadha | |
15 | * | |
16 | * This program reads the rfc3454_*.txt files, | |
17 | * parses them, and extracts the data for Nameprep conformance. | |
18 | * It then preprocesses it and writes a binary file for efficient use | |
19 | * in various IDNA conversion processes. | |
20 | */ | |
21 | ||
b75a7d8f A |
22 | #include "unicode/utypes.h" |
23 | ||
24 | #if !UCONFIG_NO_IDNA && !UCONFIG_NO_TRANSLITERATION | |
25 | ||
374ca955 A |
26 | #define USPREP_TYPE_NAMES_ARRAY |
27 | ||
b75a7d8f A |
28 | #include "unicode/uchar.h" |
29 | #include "unicode/putil.h" | |
30 | #include "cmemory.h" | |
31 | #include "cstring.h" | |
32 | #include "unicode/udata.h" | |
4388f060 | 33 | #include "unicode/utf16.h" |
b75a7d8f A |
34 | #include "unewdata.h" |
35 | #include "uoptions.h" | |
36 | #include "uparse.h" | |
37 | #include "utrie.h" | |
38 | #include "umutex.h" | |
39 | #include "sprpimpl.h" | |
40 | #include "testidna.h" | |
374ca955 | 41 | #include "punyref.h" |
73c04bcf | 42 | #include <stdlib.h> |
b75a7d8f A |
43 | |
44 | UBool beVerbose=FALSE, haveCopyright=TRUE; | |
45 | ||
46 | /* prototypes --------------------------------------------------------------- */ | |
47 | ||
48 | ||
b75a7d8f | 49 | static void |
374ca955 | 50 | parseMappings(const char *filename, UBool reportError,TestIDNA& test, UErrorCode *pErrorCode); |
b75a7d8f A |
51 | |
52 | static void | |
53 | compareMapping(uint32_t codepoint, uint32_t* mapping, int32_t mapLength, | |
374ca955 | 54 | UStringPrepType option); |
b75a7d8f A |
55 | |
56 | static void | |
374ca955 | 57 | compareFlagsForRange(uint32_t start, uint32_t end,UStringPrepType option); |
b75a7d8f A |
58 | |
59 | static void | |
60 | testAllCodepoints(TestIDNA& test); | |
61 | ||
62 | static TestIDNA* pTestIDNA =NULL; | |
63 | ||
64 | static const char* fileNames[] = { | |
729e4ab9 | 65 | "rfc3491.txt" |
374ca955 | 66 | }; |
374ca955 A |
67 | static const UTrie *idnTrie = NULL; |
68 | static const int32_t *indexes = NULL; | |
69 | static const uint16_t *mappingData = NULL; | |
b75a7d8f A |
70 | /* -------------------------------------------------------------------------- */ |
71 | ||
72 | /* file definitions */ | |
b75a7d8f A |
73 | #define DATA_TYPE "icu" |
74 | ||
729e4ab9 | 75 | #define SPREP_DIR "sprep" |
b75a7d8f A |
76 | |
77 | extern int | |
78 | testData(TestIDNA& test) { | |
b75a7d8f A |
79 | char *basename=NULL; |
80 | UErrorCode errorCode=U_ZERO_ERROR; | |
81 | char *saveBasename =NULL; | |
82 | ||
729e4ab9 | 83 | LocalUStringPrepProfilePointer profile(usprep_openByType(USPREP_RFC3491_NAMEPREP, &errorCode)); |
374ca955 | 84 | if(U_FAILURE(errorCode)){ |
729e4ab9 | 85 | test.errcheckln(errorCode, "Failed to load IDNA data file. " + UnicodeString(u_errorName(errorCode))); |
374ca955 | 86 | return errorCode; |
b75a7d8f A |
87 | } |
88 | ||
46f4442e A |
89 | char* filename = (char*) malloc(strlen(IntlTest::pathToDataDirectory())*1024); |
90 | //TODO get the srcDir dynamically | |
91 | const char *srcDir=IntlTest::pathToDataDirectory(); | |
92 | ||
374ca955 A |
93 | idnTrie = &profile->sprepTrie; |
94 | indexes = profile->indexes; | |
95 | mappingData = profile->mappingData; | |
96 | ||
b75a7d8f A |
97 | //initialize |
98 | pTestIDNA = &test; | |
99 | ||
100 | /* prepare the filename beginning with the source dir */ | |
101 | if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL){ | |
102 | filename[0] = 0x2E; | |
103 | filename[1] = U_FILE_SEP_CHAR; | |
104 | uprv_strcpy(filename+2,srcDir); | |
105 | }else{ | |
106 | uprv_strcpy(filename, srcDir); | |
107 | } | |
108 | basename=filename+uprv_strlen(filename); | |
109 | if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { | |
110 | *basename++=U_FILE_SEP_CHAR; | |
111 | } | |
112 | ||
113 | /* process unassigned */ | |
114 | basename=filename+uprv_strlen(filename); | |
115 | if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { | |
116 | *basename++=U_FILE_SEP_CHAR; | |
117 | } | |
118 | ||
119 | /* first copy misc directory */ | |
120 | saveBasename = basename; | |
729e4ab9 A |
121 | uprv_strcpy(basename,SPREP_DIR); |
122 | basename = basename + uprv_strlen(SPREP_DIR); | |
b75a7d8f A |
123 | *basename++=U_FILE_SEP_CHAR; |
124 | ||
125 | /* process unassigned */ | |
126 | uprv_strcpy(basename,fileNames[0]); | |
374ca955 | 127 | parseMappings(filename,TRUE, test,&errorCode); |
b75a7d8f A |
128 | if(U_FAILURE(errorCode)) { |
129 | test.errln( "Could not open file %s for reading \n", filename); | |
130 | return errorCode; | |
131 | } | |
132 | ||
133 | testAllCodepoints(test); | |
134 | ||
b75a7d8f A |
135 | pTestIDNA = NULL; |
136 | free(filename); | |
137 | return errorCode; | |
138 | } | |
139 | U_CDECL_BEGIN | |
374ca955 | 140 | |
b75a7d8f | 141 | static void U_CALLCONV |
374ca955 A |
142 | strprepProfileLineFn(void * /*context*/, |
143 | char *fields[][2], int32_t fieldCount, | |
b75a7d8f A |
144 | UErrorCode *pErrorCode) { |
145 | uint32_t mapping[40]; | |
374ca955 | 146 | char *end, *map; |
b75a7d8f A |
147 | uint32_t code; |
148 | int32_t length; | |
374ca955 A |
149 | /*UBool* mapWithNorm = (UBool*) context;*/ |
150 | const char* typeName; | |
151 | uint32_t rangeStart=0,rangeEnd =0; | |
729e4ab9 A |
152 | const char *s; |
153 | ||
154 | s = u_skipWhitespace(fields[0][0]); | |
155 | if (*s == '@') { | |
156 | /* a special directive introduced in 4.2 */ | |
157 | return; | |
158 | } | |
159 | ||
374ca955 A |
160 | if(fieldCount != 3){ |
161 | *pErrorCode = U_INVALID_FORMAT_ERROR; | |
162 | return; | |
163 | } | |
b75a7d8f | 164 | |
374ca955 A |
165 | typeName = fields[2][0]; |
166 | map = fields[1][0]; | |
167 | ||
168 | if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){ | |
b75a7d8f | 169 | |
729e4ab9 | 170 | u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode); |
374ca955 A |
171 | |
172 | /* store the range */ | |
173 | compareFlagsForRange(rangeStart,rangeEnd,USPREP_UNASSIGNED); | |
174 | ||
175 | }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){ | |
176 | ||
729e4ab9 | 177 | u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode); |
b75a7d8f | 178 | |
374ca955 A |
179 | /* store the range */ |
180 | compareFlagsForRange(rangeStart,rangeEnd,USPREP_PROHIBITED); | |
b75a7d8f | 181 | |
374ca955 A |
182 | }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){ |
183 | /* get the character code, field 0 */ | |
729e4ab9 | 184 | code=(uint32_t)uprv_strtoul(s, &end, 16); |
374ca955 A |
185 | |
186 | /* parse the mapping string */ | |
187 | length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode); | |
188 | ||
189 | /* store the mapping */ | |
190 | compareMapping(code,mapping, length,USPREP_MAP); | |
191 | ||
192 | }else{ | |
193 | *pErrorCode = U_INVALID_FORMAT_ERROR; | |
194 | } | |
b75a7d8f | 195 | |
b75a7d8f | 196 | } |
374ca955 | 197 | |
b75a7d8f A |
198 | U_CDECL_END |
199 | ||
200 | static void | |
374ca955 | 201 | parseMappings(const char *filename,UBool reportError, TestIDNA& test, UErrorCode *pErrorCode) { |
b75a7d8f A |
202 | char *fields[3][2]; |
203 | ||
204 | if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { | |
205 | return; | |
206 | } | |
207 | ||
374ca955 | 208 | u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode); |
b75a7d8f A |
209 | |
210 | //fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len); | |
211 | ||
212 | if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) { | |
213 | test.errln( "testidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); | |
214 | } | |
215 | } | |
216 | ||
b75a7d8f | 217 | |
374ca955 A |
218 | static inline UStringPrepType |
219 | getValues(uint32_t result, int32_t& value, UBool& isIndex){ | |
b75a7d8f | 220 | |
374ca955 | 221 | UStringPrepType type; |
b75a7d8f | 222 | |
374ca955 A |
223 | if(result == 0){ |
224 | /* | |
225 | * Initial value stored in the mapping table | |
226 | * just return USPREP_TYPE_LIMIT .. so that | |
227 | * the source codepoint is copied to the destination | |
228 | */ | |
229 | type = USPREP_TYPE_LIMIT; | |
73c04bcf A |
230 | isIndex =FALSE; |
231 | value = 0; | |
374ca955 A |
232 | }else if(result >= _SPREP_TYPE_THRESHOLD){ |
233 | type = (UStringPrepType) (result - _SPREP_TYPE_THRESHOLD); | |
73c04bcf A |
234 | isIndex =FALSE; |
235 | value = 0; | |
374ca955 A |
236 | }else{ |
237 | /* get the state */ | |
238 | type = USPREP_MAP; | |
239 | /* ascertain if the value is index or delta */ | |
240 | if(result & 0x02){ | |
241 | isIndex = TRUE; | |
242 | value = result >> 2; //mask off the lower 2 bits and shift | |
b75a7d8f | 243 | |
374ca955 A |
244 | }else{ |
245 | isIndex = FALSE; | |
246 | value = (int16_t)result; | |
247 | value = (value >> 2); | |
b75a7d8f | 248 | |
374ca955 A |
249 | } |
250 | if((result>>2) == _SPREP_MAX_INDEX_VALUE){ | |
251 | type = USPREP_DELETE; | |
252 | isIndex =FALSE; | |
253 | value = 0; | |
254 | } | |
255 | } | |
256 | return type; | |
b75a7d8f A |
257 | } |
258 | ||
374ca955 | 259 | |
b75a7d8f A |
260 | |
261 | static void | |
374ca955 A |
262 | testAllCodepoints(TestIDNA& test){ |
263 | /* | |
264 | { | |
265 | UChar str[19] = { | |
266 | 0xC138, 0xACC4, 0xC758, 0xBAA8, 0xB4E0, 0xC0AC, 0xB78C, 0xB4E4, 0xC774, | |
267 | 0x070F,//prohibited | |
268 | 0xD55C, 0xAD6D, 0xC5B4, 0xB97C, 0xC774, 0xD574, 0xD55C, 0xB2E4, 0xBA74 | |
269 | }; | |
270 | uint32_t in[19] = {0}; | |
271 | UErrorCode status = U_ZERO_ERROR; | |
272 | int32_t inLength=0, outLength=100; | |
273 | char output[100] = {0}; | |
274 | punycode_status error; | |
275 | u_strToUTF32((UChar32*)in,19,&inLength,str,19,&status); | |
276 | ||
277 | error= punycode_encode(inLength, in, NULL, (uint32_t*)&outLength, output); | |
278 | printf(output); | |
279 | ||
b75a7d8f | 280 | } |
374ca955 | 281 | */ |
b75a7d8f | 282 | |
374ca955 A |
283 | uint32_t i = 0; |
284 | int32_t unassigned = 0; | |
285 | int32_t prohibited = 0; | |
286 | int32_t mappedWithNorm = 0; | |
287 | int32_t mapped = 0; | |
288 | int32_t noValueInTrie = 0; | |
b75a7d8f | 289 | |
374ca955 A |
290 | UStringPrepType type; |
291 | int32_t value; | |
292 | UBool isIndex = FALSE; | |
b75a7d8f | 293 | |
374ca955 A |
294 | for(i=0;i<=0x10FFFF;i++){ |
295 | uint32_t result = 0; | |
296 | UTRIE_GET16(idnTrie,i, result); | |
297 | type = getValues(result,value, isIndex); | |
298 | if(type != USPREP_TYPE_LIMIT ){ | |
299 | if(type == USPREP_UNASSIGNED){ | |
300 | unassigned++; | |
301 | } | |
302 | if(type == USPREP_PROHIBITED){ | |
303 | prohibited++; | |
304 | } | |
305 | if(type == USPREP_MAP){ | |
306 | mapped++; | |
307 | } | |
308 | }else{ | |
309 | noValueInTrie++; | |
310 | if(result > 0){ | |
311 | test.errln("The return value for 0x%06X is wrong. %i\n",i,result); | |
b75a7d8f A |
312 | } |
313 | } | |
b75a7d8f | 314 | } |
b75a7d8f | 315 | |
374ca955 A |
316 | test.logln("Number of Unassinged code points : %i \n",unassigned); |
317 | test.logln("Number of Prohibited code points : %i \n",prohibited); | |
318 | test.logln("Number of Mapped code points : %i \n",mapped); | |
319 | test.logln("Number of Mapped with NFKC code points : %i \n",mappedWithNorm); | |
320 | test.logln("Number of code points that have no value in Trie: %i \n",noValueInTrie); | |
321 | ||
322 | ||
b75a7d8f A |
323 | } |
324 | ||
325 | static void | |
326 | compareMapping(uint32_t codepoint, uint32_t* mapping,int32_t mapLength, | |
374ca955 A |
327 | UStringPrepType type){ |
328 | uint32_t result = 0; | |
329 | UTRIE_GET16(idnTrie,codepoint, result); | |
b75a7d8f | 330 | |
374ca955 A |
331 | int32_t length=0; |
332 | UBool isIndex; | |
333 | UStringPrepType retType; | |
334 | int32_t value, index=0, delta=0; | |
335 | ||
336 | retType = getValues(result,value,isIndex); | |
b75a7d8f A |
337 | |
338 | ||
374ca955 A |
339 | if(type != retType && retType != USPREP_DELETE){ |
340 | ||
341 | pTestIDNA->errln( "Did not get the assigned type for codepoint 0x%08X. Expected: %i Got: %i\n",codepoint, USPREP_MAP, type); | |
342 | ||
343 | } | |
344 | ||
345 | if(isIndex){ | |
346 | index = value; | |
347 | if(index >= indexes[_SPREP_ONE_UCHAR_MAPPING_INDEX_START] && | |
348 | index < indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START]){ | |
349 | length = 1; | |
350 | }else if(index >= indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START] && | |
351 | index < indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START]){ | |
352 | length = 2; | |
353 | }else if(index >= indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START] && | |
354 | index < indexes[_SPREP_FOUR_UCHARS_MAPPING_INDEX_START]){ | |
355 | length = 3; | |
b75a7d8f | 356 | }else{ |
374ca955 | 357 | length = mappingData[index++]; |
b75a7d8f | 358 | } |
374ca955 A |
359 | }else{ |
360 | delta = value; | |
361 | length = (retType == USPREP_DELETE)? 0 : 1; | |
362 | } | |
b75a7d8f | 363 | |
374ca955 A |
364 | int32_t realLength =0; |
365 | /* figure out the real length */ | |
366 | for(int32_t j=0; j<mapLength; j++){ | |
367 | if(mapping[j] > 0xFFFF){ | |
368 | realLength +=2; | |
369 | }else{ | |
370 | realLength++; | |
371 | } | |
372 | } | |
b75a7d8f | 373 | |
374ca955 A |
374 | if(realLength != length){ |
375 | pTestIDNA->errln( "Did not get the expected length. Expected: %i Got: %i\n", mapLength, length); | |
376 | } | |
377 | ||
378 | if(isIndex){ | |
b75a7d8f A |
379 | for(int8_t i =0; i< mapLength; i++){ |
380 | if(mapping[i] <= 0xFFFF){ | |
381 | if(mappingData[index+i] != (uint16_t)mapping[i]){ | |
382 | pTestIDNA->errln("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[i], mappingData[index+i]); | |
383 | } | |
384 | }else{ | |
4388f060 A |
385 | UChar lead = U16_LEAD(mapping[i]); |
386 | UChar trail = U16_TRAIL(mapping[i]); | |
b75a7d8f A |
387 | if(mappingData[index+i] != lead || |
388 | mappingData[index+i+1] != trail){ | |
389 | pTestIDNA->errln( "Did not get the expected result. Expected: 0x%04X 0x%04X Got: 0x%04X 0x%04X", lead, trail, mappingData[index+i], mappingData[index+i+1]); | |
390 | } | |
391 | } | |
392 | } | |
374ca955 A |
393 | }else{ |
394 | if(retType!=USPREP_DELETE && (codepoint-delta) != (uint16_t)mapping[0]){ | |
395 | pTestIDNA->errln("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[0],(codepoint-delta)); | |
396 | } | |
b75a7d8f A |
397 | } |
398 | ||
399 | } | |
400 | ||
401 | static void | |
402 | compareFlagsForRange(uint32_t start, uint32_t end, | |
374ca955 A |
403 | UStringPrepType type){ |
404 | ||
405 | uint32_t result =0 ; | |
406 | UStringPrepType retType; | |
407 | UBool isIndex=FALSE; | |
408 | int32_t value=0; | |
4388f060 | 409 | /* |
374ca955 | 410 | // supplementary code point |
4388f060 | 411 | UChar __lead16=U16_LEAD(0x2323E); |
374ca955 A |
412 | int32_t __offset; |
413 | ||
414 | // get data for lead surrogate | |
415 | (result)=_UTRIE_GET_RAW((&idnTrie), index, 0, (__lead16)); | |
416 | __offset=(&idnTrie)->getFoldingOffset(result); | |
417 | ||
418 | // get the real data from the folded lead/trail units | |
419 | if(__offset>0) { | |
420 | (result)=_UTRIE_GET_RAW((&idnTrie), index, __offset, (0x2323E)&0x3ff); | |
b75a7d8f | 421 | } else { |
374ca955 | 422 | (result)=(uint32_t)((&idnTrie)->initialValue); |
b75a7d8f | 423 | } |
b75a7d8f | 424 | |
374ca955 A |
425 | UTRIE_GET16(&idnTrie,0x2323E, result); |
426 | */ | |
427 | while(start < end+1){ | |
428 | UTRIE_GET16(idnTrie,start, result); | |
429 | retType = getValues(result,value,isIndex); | |
430 | if(result > _SPREP_TYPE_THRESHOLD){ | |
431 | if(retType != type){ | |
432 | pTestIDNA->errln( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]); | |
433 | } | |
434 | }else{ | |
435 | if(type == USPREP_PROHIBITED && ((result & 0x01) != 0x01)){ | |
436 | pTestIDNA->errln( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]); | |
437 | } | |
b75a7d8f | 438 | } |
b75a7d8f | 439 | |
374ca955 | 440 | start++; |
b75a7d8f | 441 | } |
374ca955 | 442 | |
b75a7d8f A |
443 | } |
444 | ||
374ca955 | 445 | |
b75a7d8f A |
446 | #endif /* #if !UCONFIG_NO_IDNA */ |
447 | ||
448 | /* | |
449 | * Hey, Emacs, please set the following: | |
450 | * | |
451 | * Local Variables: | |
452 | * indent-tabs-mode: nil | |
453 | * End: | |
454 | * | |
455 | */ |