icuSources/test/intltest/testidn.cpp

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4 *******************************************************************************
   5 *
   6 *   Copyright (C) 2003-2013, International Business Machines
   7 *   Corporation and others.  All Rights Reserved.
   8 *
   9 *******************************************************************************
  10 *   file name:  testidn.cpp
  11 *   encoding:   UTF-8
  12 *   tab size:   8 (not used)
  13 *   indentation:4
  14 *
  15 *   created on: 2003-02-06
  16 *   created by: Ram Viswanadha
  17 *
  18 *   This program reads the rfc3454_*.txt files,
  19 *   parses them, and extracts the data for Nameprep conformance.
  20 *   It then preprocesses it and writes a binary file for efficient use
  21 *   in various IDNA conversion processes.
  22 */
  23
  24 #include "unicode/utypes.h"
  25
  26 #if !UCONFIG_NO_IDNA && !UCONFIG_NO_TRANSLITERATION
  27
  28 #define USPREP_TYPE_NAMES_ARRAY
  29
  30 #include "unicode/uchar.h"
  31 #include "unicode/putil.h"
  32 #include "cmemory.h"
  33 #include "cstring.h"
  34 #include "unicode/udata.h"
  35 #include "unicode/utf16.h"
  36 #include "unewdata.h"
  37 #include "uoptions.h"
  38 #include "uparse.h"
  39 #include "utrie.h"
  40 #include "umutex.h"
  41 #include "sprpimpl.h"
  42 #include "testidna.h"
  43 #include "punyref.h"
  44 #include <stdlib.h>
  45
  46 UBool beVerbose=FALSE, haveCopyright=TRUE;
  47
  48 /* prototypes --------------------------------------------------------------- */
  49
  50
  51 static void
  52 parseMappings(const char *filename, UBool reportError,TestIDNA& test, UErrorCode *pErrorCode);
  53
  54 static void
  55 compareMapping(uint32_t codepoint, uint32_t* mapping, int32_t mapLength,
  56                UStringPrepType option);
  57
  58 static void
  59 compareFlagsForRange(uint32_t start, uint32_t end,UStringPrepType option);
  60
  61 static void
  62 testAllCodepoints(TestIDNA& test);
  63
  64 static TestIDNA* pTestIDNA =NULL;
  65
  66 static const char* fileNames[] = {
  67                                     "rfc3491.txt"
  68                                  };
  69 static const UTrie *idnTrie              = NULL;
  70 static const int32_t *indexes            = NULL;
  71 static const uint16_t *mappingData       = NULL;
  72 /* -------------------------------------------------------------------------- */
  73
  74 /* file definitions */
  75 #define DATA_TYPE "icu"
  76
  77 #define SPREP_DIR "sprep"
  78
  79 extern int
  80 testData(TestIDNA& test) {
  81     char *basename=NULL;
  82     UErrorCode errorCode=U_ZERO_ERROR;
  83     char *saveBasename =NULL;
  84
  85     LocalUStringPrepProfilePointer profile(usprep_openByType(USPREP_RFC3491_NAMEPREP, &errorCode));
  86     if(U_FAILURE(errorCode)){
  87         test.errcheckln(errorCode, "Failed to load IDNA data file. " + UnicodeString(u_errorName(errorCode)));
  88         return errorCode;
  89     }
  90
  91     char* filename = (char*) malloc(strlen(IntlTest::pathToDataDirectory())*1024);
  92     //TODO get the srcDir dynamically
  93     const char *srcDir=IntlTest::pathToDataDirectory();
  94
  95     idnTrie     = &profile->sprepTrie;
  96     indexes     = profile->indexes;
  97     mappingData = profile->mappingData;
  98
  99     //initialize
 100     pTestIDNA = &test;
 101
 102     /* prepare the filename beginning with the source dir */
 103     if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL){
 104         filename[0] = 0x2E;
 105         filename[1] = U_FILE_SEP_CHAR;
 106         uprv_strcpy(filename+2,srcDir);
 107     }else{
 108         uprv_strcpy(filename, srcDir);
 109     }
 110     basename=filename+uprv_strlen(filename);
 111     if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
 112         *basename++=U_FILE_SEP_CHAR;
 113     }
 114
 115     /* process unassigned */
 116     basename=filename+uprv_strlen(filename);
 117     if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
 118         *basename++=U_FILE_SEP_CHAR;
 119     }
 120
 121     /* first copy misc directory */
 122     saveBasename = basename;
 123     (void)saveBasename;    // Suppress set but not used warning.
 124     uprv_strcpy(basename,SPREP_DIR);
 125     basename = basename + uprv_strlen(SPREP_DIR);
 126     *basename++=U_FILE_SEP_CHAR;
 127
 128     /* process unassigned */
 129     uprv_strcpy(basename,fileNames[0]);
 130     parseMappings(filename,TRUE, test,&errorCode);
 131     if(U_FAILURE(errorCode)) {
 132         test.errln( "Could not open file %s for reading \n", filename);
 133         return errorCode;
 134     }
 135
 136     testAllCodepoints(test);
 137
 138     pTestIDNA = NULL;
 139     free(filename);
 140     return errorCode;
 141 }
 142 U_CDECL_BEGIN
 143
 144 static void U_CALLCONV
 145 strprepProfileLineFn(void * /*context*/,
 146               char *fields[][2], int32_t fieldCount,
 147               UErrorCode *pErrorCode) {
 148     uint32_t mapping[40];
 149     char *end, *map;
 150     uint32_t code;
 151     int32_t length;
 152    /*UBool* mapWithNorm = (UBool*) context;*/
 153     const char* typeName;
 154     uint32_t rangeStart=0,rangeEnd =0;
 155     const char *s;
 156
 157     s = u_skipWhitespace(fields[0][0]);
 158     if (*s == '@') {
 159         /* a special directive introduced in 4.2 */
 160         return;
 161     }
 162
 163     if(fieldCount != 3){
 164         *pErrorCode = U_INVALID_FORMAT_ERROR;
 165         return;
 166     }
 167
 168     typeName = fields[2][0];
 169     map = fields[1][0];
 170
 171     if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){
 172
 173         u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode);
 174
 175         /* store the range */
 176         compareFlagsForRange(rangeStart,rangeEnd,USPREP_UNASSIGNED);
 177
 178     }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){
 179
 180         u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode);
 181
 182         /* store the range */
 183         compareFlagsForRange(rangeStart,rangeEnd,USPREP_PROHIBITED);
 184
 185     }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){
 186         /* get the character code, field 0 */
 187         code=(uint32_t)uprv_strtoul(s, &end, 16);
 188
 189         /* parse the mapping string */
 190         length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode);
 191
 192         /* store the mapping */
 193         compareMapping(code,mapping, length,USPREP_MAP);
 194
 195     }else{
 196         *pErrorCode = U_INVALID_FORMAT_ERROR;
 197     }
 198
 199 }
 200
 201 U_CDECL_END
 202
 203 static void
 204 parseMappings(const char *filename,UBool reportError, TestIDNA& test, UErrorCode *pErrorCode) {
 205     char *fields[3][2];
 206
 207     if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
 208         return;
 209     }
 210
 211     u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode);
 212
 213     //fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);
 214
 215     if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) {
 216         test.errln( "testidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
 217     }
 218 }
 219
 220
 221 static inline UStringPrepType
 222 getValues(uint32_t result, int32_t& value, UBool& isIndex){
 223
 224     UStringPrepType type;
 225
 226     if(result == 0){
 227         /*
 228          * Initial value stored in the mapping table
 229          * just return USPREP_TYPE_LIMIT .. so that
 230          * the source codepoint is copied to the destination
 231          */
 232         type = USPREP_TYPE_LIMIT;
 233         isIndex =FALSE;
 234         value = 0;
 235     }else if(result >= _SPREP_TYPE_THRESHOLD){
 236         type = (UStringPrepType) (result - _SPREP_TYPE_THRESHOLD);
 237         isIndex =FALSE;
 238         value = 0;
 239     }else{
 240         /* get the state */
 241         type = USPREP_MAP;
 242         /* ascertain if the value is index or delta */
 243         if(result & 0x02){
 244             isIndex = TRUE;
 245             value = result  >> 2; //mask off the lower 2 bits and shift
 246
 247         }else{
 248             isIndex = FALSE;
 249             value = (int16_t)result;
 250             value =  (value >> 2);
 251
 252         }
 253         if((result>>2) == _SPREP_MAX_INDEX_VALUE){
 254             type = USPREP_DELETE;
 255             isIndex =FALSE;
 256             value = 0;
 257         }
 258     }
 259     return type;
 260 }
 261
 262
 263
 264 static void
 265 testAllCodepoints(TestIDNA& test){
 266     /*
 267     {
 268         UChar str[19] = {
 269                             0xC138, 0xACC4, 0xC758, 0xBAA8, 0xB4E0, 0xC0AC, 0xB78C, 0xB4E4, 0xC774,
 270                             0x070F,//prohibited
 271                             0xD55C, 0xAD6D, 0xC5B4, 0xB97C, 0xC774, 0xD574, 0xD55C, 0xB2E4, 0xBA74
 272                         };
 273         uint32_t in[19] = {0};
 274         UErrorCode status = U_ZERO_ERROR;
 275         int32_t inLength=0, outLength=100;
 276         char output[100] = {0};
 277         punycode_status error;
 278         u_strToUTF32((UChar32*)in,19,&inLength,str,19,&status);
 279
 280         error= punycode_encode(inLength, in, NULL, (uint32_t*)&outLength, output);
 281         printf(output);
 282
 283     }
 284     */
 285
 286     uint32_t i = 0;
 287     int32_t unassigned      = 0;
 288     int32_t prohibited      = 0;
 289     int32_t mappedWithNorm  = 0;
 290     int32_t mapped          = 0;
 291     int32_t noValueInTrie   = 0;
 292
 293     UStringPrepType type;
 294     int32_t value;
 295     UBool isIndex = FALSE;
 296
 297     for(i=0;i<=0x10FFFF;i++){
 298         uint32_t result = 0;
 299         UTRIE_GET16(idnTrie,i, result);
 300         type = getValues(result,value, isIndex);
 301         if(type != USPREP_TYPE_LIMIT ){
 302             if(type == USPREP_UNASSIGNED){
 303                 unassigned++;
 304             }
 305             if(type == USPREP_PROHIBITED){
 306                 prohibited++;
 307             }
 308             if(type == USPREP_MAP){
 309                 mapped++;
 310             }
 311         }else{
 312             noValueInTrie++;
 313             if(result > 0){
 314                 test.errln("The return value for 0x%06X is wrong. %i\n",i,result);
 315             }
 316         }
 317     }
 318
 319     test.logln("Number of Unassinged code points : %i \n",unassigned);
 320     test.logln("Number of Prohibited code points : %i \n",prohibited);
 321     test.logln("Number of Mapped code points : %i \n",mapped);
 322     test.logln("Number of Mapped with NFKC code points : %i \n",mappedWithNorm);
 323     test.logln("Number of code points that have no value in Trie: %i \n",noValueInTrie);
 324
 325
 326 }
 327
 328 static void
 329 compareMapping(uint32_t codepoint, uint32_t* mapping,int32_t mapLength,
 330                UStringPrepType type){
 331     uint32_t result = 0;
 332     UTRIE_GET16(idnTrie,codepoint, result);
 333
 334     int32_t length=0;
 335     UBool isIndex;
 336     UStringPrepType retType;
 337     int32_t value, index=0, delta=0;
 338
 339     retType = getValues(result,value,isIndex);
 340
 341
 342     if(type != retType && retType != USPREP_DELETE){
 343
 344         pTestIDNA->errln( "Did not get the assigned type for codepoint 0x%08X. Expected: %i Got: %i\n",codepoint, USPREP_MAP, type);
 345
 346     }
 347
 348     if(isIndex){
 349         index = value;
 350         if(index >= indexes[_SPREP_ONE_UCHAR_MAPPING_INDEX_START] &&
 351                  index < indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START]){
 352             length = 1;
 353         }else if(index >= indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START] &&
 354                  index < indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START]){
 355             length = 2;
 356         }else if(index >= indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START] &&
 357                  index < indexes[_SPREP_FOUR_UCHARS_MAPPING_INDEX_START]){
 358             length = 3;
 359         }else{
 360             length = mappingData[index++];
 361         }
 362     }else{
 363         delta = value;
 364         length = (retType == USPREP_DELETE)? 0 :  1;
 365     }
 366
 367     int32_t realLength =0;
 368     /* figure out the real length */
 369     for(int32_t j=0; j<mapLength; j++){
 370         if(mapping[j] > 0xFFFF){
 371             realLength +=2;
 372         }else{
 373             realLength++;
 374         }
 375     }
 376
 377     if(realLength != length){
 378         pTestIDNA->errln( "Did not get the expected length. Expected: %i Got: %i\n", mapLength, length);
 379     }
 380
 381     if(isIndex){
 382         for(int8_t i =0; i< mapLength; i++){
 383             if(mapping[i] <= 0xFFFF){
 384                 if(mappingData[index+i] != (uint16_t)mapping[i]){
 385                     pTestIDNA->errln("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[i], mappingData[index+i]);
 386                 }
 387             }else{
 388                 UChar lead  = U16_LEAD(mapping[i]);
 389                 UChar trail = U16_TRAIL(mapping[i]);
 390                 if(mappingData[index+i] != lead ||
 391                     mappingData[index+i+1] != trail){
 392                     pTestIDNA->errln( "Did not get the expected result. Expected: 0x%04X 0x%04X  Got: 0x%04X 0x%04X", lead, trail, mappingData[index+i], mappingData[index+i+1]);
 393                 }
 394             }
 395         }
 396     }else{
 397         if(retType!=USPREP_DELETE && (codepoint-delta) != (uint16_t)mapping[0]){
 398             pTestIDNA->errln("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[0],(codepoint-delta));
 399         }
 400     }
 401
 402 }
 403
 404 static void
 405 compareFlagsForRange(uint32_t start, uint32_t end,
 406                      UStringPrepType type){
 407
 408     uint32_t result =0 ;
 409     UStringPrepType retType;
 410     UBool isIndex=FALSE;
 411     int32_t value=0;
 412 /*
 413     // supplementary code point
 414     UChar __lead16=U16_LEAD(0x2323E);
 415     int32_t __offset;
 416
 417     // get data for lead surrogate
 418     (result)=_UTRIE_GET_RAW((&idnTrie), index, 0, (__lead16));
 419     __offset=(&idnTrie)->getFoldingOffset(result);
 420
 421     // get the real data from the folded lead/trail units
 422     if(__offset>0) {
 423         (result)=_UTRIE_GET_RAW((&idnTrie), index, __offset, (0x2323E)&0x3ff);
 424     } else {
 425         (result)=(uint32_t)((&idnTrie)->initialValue);
 426     }
 427
 428     UTRIE_GET16(&idnTrie,0x2323E, result);
 429 */
 430     while(start < end+1){
 431         UTRIE_GET16(idnTrie,start, result);
 432         retType = getValues(result,value,isIndex);
 433         if(result > _SPREP_TYPE_THRESHOLD){
 434             if(retType != type){
 435                 pTestIDNA->errln( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]);
 436             }
 437         }else{
 438             if(type == USPREP_PROHIBITED && ((result & 0x01) != 0x01)){
 439                 pTestIDNA->errln( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]);
 440             }
 441         }
 442
 443         start++;
 444     }
 445
 446 }
 447
 448
 449 #endif /* #if !UCONFIG_NO_IDNA */
 450
 451 /*
 452  * Hey, Emacs, please set the following:
 453  *
 454  * Local Variables:
 455  * indent-tabs-mode: nil
 456  * End:
 457  *
 458  */