icuSources/test/cintltst/sprpdata.c

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4  *******************************************************************************
   5  *
   6  *   Copyright (C) 2003-2012, International Business Machines
   7  *   Corporation and others.  All Rights Reserved.
   8  *
   9  *******************************************************************************
  10  *   file name:  spreptst.c
  11  *   encoding:   UTF-8
  12  *   tab size:   8 (not used)
  13  *   indentation:4
  14  *
  15  *   created on: 2003jul11
  16  *   created by: Ram Viswanadha
  17  */
  18 #define USPREP_TYPE_NAMES_ARRAY
  19
  20 #include "unicode/utypes.h"
  21
  22 #if !UCONFIG_NO_IDNA
  23
  24 #include "unicode/ustring.h"
  25 #include "unicode/putil.h"
  26 #include "cintltst.h"
  27 #include "unicode/usprep.h"
  28 #include "unicode/utf16.h"
  29 #include "sprpimpl.h"
  30 #include "uparse.h"
  31 #include "cmemory.h"
  32 #include "ustr_imp.h"
  33 #include "cstring.h"
  34
  35 static void
  36 parseMappings(const char *filename, UStringPrepProfile* data, UBool reportError, UErrorCode *pErrorCode);
  37
  38 static void
  39 compareMapping(UStringPrepProfile* data, uint32_t codepoint, uint32_t* mapping, int32_t mapLength,
  40                UStringPrepType option);
  41
  42 static void
  43 compareFlagsForRange(UStringPrepProfile* data, uint32_t start, uint32_t end,UStringPrepType option);
  44
  45 void
  46 doStringPrepTest(const char* binFileName, const char* txtFileName, int32_t options, UErrorCode* errorCode);
  47
  48 static void U_CALLCONV
  49 strprepProfileLineFn(void *context,
  50               char *fields[][2], int32_t fieldCount,
  51               UErrorCode *pErrorCode) {
  52     uint32_t mapping[40];
  53     char *end, *map;
  54     uint32_t code;
  55     int32_t length;
  56     UStringPrepProfile* data = (UStringPrepProfile*) context;
  57     const char* typeName;
  58     uint32_t rangeStart=0,rangeEnd =0;
  59
  60     typeName = fields[2][0];
  61     map = fields[1][0];
  62
  63     if(strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){
  64
  65         u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode);
  66
  67         /* store the range */
  68         compareFlagsForRange(data, rangeStart,rangeEnd,USPREP_UNASSIGNED);
  69
  70     }else if(strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){
  71
  72         u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode);
  73
  74         /* store the range */
  75         compareFlagsForRange(data, rangeStart,rangeEnd,USPREP_PROHIBITED);
  76
  77     }else if(strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){
  78         /* get the character code, field 0 */
  79         code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
  80
  81         /* parse the mapping string */
  82         length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode);
  83
  84         /* compare the mapping */
  85         compareMapping(data, code,mapping, length,USPREP_MAP);
  86     }else{
  87         *pErrorCode = U_INVALID_FORMAT_ERROR;
  88     }
  89
  90 }
  91
  92
  93
  94 static void
  95 parseMappings(const char *filename, UStringPrepProfile* data, UBool reportError, UErrorCode *pErrorCode) {
  96     char *fields[3][2];
  97
  98     if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
  99         return;
 100     }
 101
 102     u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)data, pErrorCode);
 103
 104     /*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/
 105
 106     if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) {
 107         log_err( "testidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
 108     }
 109 }
 110
 111
 112 static UStringPrepType
 113 getValues(uint32_t result, int32_t* value, UBool* isIndex){
 114
 115     UStringPrepType type;
 116     if(result == 0){
 117         /*
 118          * Initial value stored in the mapping table
 119          * just return USPREP_TYPE_LIMIT .. so that
 120          * the source codepoint is copied to the destination
 121          */
 122         type = USPREP_TYPE_LIMIT;
 123     }else if(result >= _SPREP_TYPE_THRESHOLD){
 124         type = (UStringPrepType) (result - _SPREP_TYPE_THRESHOLD);
 125     }else{
 126         /* get the type */
 127         type = USPREP_MAP;
 128         /* ascertain if the value is index or delta */
 129         if(result & 0x02){
 130             *isIndex = TRUE;
 131             *value = result  >> 2;
 132
 133         }else{
 134             *isIndex = FALSE;
 135             *value = (int16_t)result;
 136             *value =  (*value >> 2);
 137
 138         }
 139         if((result>>2) == _SPREP_MAX_INDEX_VALUE){
 140             type = USPREP_DELETE;
 141             isIndex =FALSE;
 142             value = 0;
 143         }
 144     }
 145     return type;
 146 }
 147
 148 static void
 149 compareMapping(UStringPrepProfile* data, uint32_t codepoint, uint32_t* mapping,int32_t mapLength,
 150                UStringPrepType type){
 151     uint32_t result = 0;
 152     int32_t length=0;
 153     UBool isIndex = FALSE;
 154     UStringPrepType retType;
 155     int32_t value=0, idx=0, delta=0;
 156     int32_t* indexes = data->indexes;
 157     UTrie trie = data->sprepTrie;
 158     const uint16_t* mappingData = data->mappingData;
 159     int32_t realLength =0;
 160     int32_t j=0;
 161     int8_t i=0;
 162
 163     UTRIE_GET16(&trie, codepoint, result);
 164     retType = getValues(result,&value,&isIndex);
 165
 166
 167     if(type != retType && retType != USPREP_DELETE){
 168
 169         log_err( "Did not get the assigned type for codepoint 0x%08X. Expected: %i Got: %i\n",codepoint, USPREP_MAP, type);
 170
 171     }
 172
 173     if(isIndex){
 174         idx = value;
 175         if(idx >= indexes[_SPREP_ONE_UCHAR_MAPPING_INDEX_START] &&
 176                  idx < indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START]){
 177             length = 1;
 178         }else if(idx >= indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START] &&
 179                  idx < indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START]){
 180             length = 2;
 181         }else if(idx >= indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START] &&
 182                  idx < indexes[_SPREP_FOUR_UCHARS_MAPPING_INDEX_START]){
 183             length = 3;
 184         }else{
 185             length = mappingData[idx++];
 186         }
 187     }else{
 188         delta = value;
 189         length = (retType == USPREP_DELETE)? 0 :  1;
 190     }
 191
 192     /* figure out the real length */
 193     for(j=0; j<mapLength; j++){
 194         if(mapping[j] > 0xFFFF){
 195             realLength +=2;
 196         }else{
 197             realLength++;
 198         }
 199     }
 200
 201     if(realLength != length){
 202         log_err( "Did not get the expected length. Expected: %i Got: %i\n", mapLength, length);
 203     }
 204
 205     if(isIndex){
 206         for(i =0; i< mapLength; i++){
 207             if(mapping[i] <= 0xFFFF){
 208                 if(mappingData[idx+i] != (uint16_t)mapping[i]){
 209                     log_err("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[i], mappingData[idx+i]);
 210                 }
 211             }else{
 212                 UChar lead  = U16_LEAD(mapping[i]);
 213                 UChar trail = U16_TRAIL(mapping[i]);
 214                 if(mappingData[idx+i] != lead ||
 215                     mappingData[idx+i+1] != trail){
 216                     log_err( "Did not get the expected result. Expected: 0x%04X 0x%04X  Got: 0x%04X 0x%04X\n", lead, trail, mappingData[idx+i], mappingData[idx+i+1]);
 217                 }
 218             }
 219         }
 220     }else{
 221         if(retType!=USPREP_DELETE && (codepoint-delta) != (uint16_t)mapping[0]){
 222            log_err("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[0],(codepoint-delta));
 223         }
 224     }
 225
 226 }
 227
 228 static void
 229 compareFlagsForRange(UStringPrepProfile* data,
 230                      uint32_t start, uint32_t end,
 231                      UStringPrepType type){
 232
 233     uint32_t result =0 ;
 234     UStringPrepType retType;
 235     UBool isIndex=FALSE;
 236     int32_t value=0;
 237     UTrie trie = data->sprepTrie;
 238 /*
 239     // supplementary code point
 240     UChar __lead16=U16_LEAD(0x2323E);
 241     int32_t __offset;
 242
 243     // get data for lead surrogate
 244     (result)=_UTRIE_GET_RAW((&idnTrie), index, 0, (__lead16));
 245     __offset=(&idnTrie)->getFoldingOffset(result);
 246
 247     // get the real data from the folded lead/trail units
 248     if(__offset>0) {
 249         (result)=_UTRIE_GET_RAW((&idnTrie), index, __offset, (0x2323E)&0x3ff);
 250     } else {
 251         (result)=(uint32_t)((&idnTrie)->initialValue);
 252     }
 253
 254     UTRIE_GET16(&idnTrie,0x2323E, result);
 255 */
 256     while(start < end+1){
 257         UTRIE_GET16(&trie,start, result);
 258         retType = getValues(result, &value, &isIndex);
 259         if(result > _SPREP_TYPE_THRESHOLD){
 260             if(retType != type){
 261                 log_err( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]);
 262             }
 263         }else{
 264             if(type == USPREP_PROHIBITED && ((result & 0x01) != 0x01)){
 265                 log_err( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]);
 266             }
 267         }
 268
 269         start++;
 270     }
 271
 272 }
 273
 274 void
 275 doStringPrepTest(const char* binFileName, const char* txtFileName, int32_t options, UErrorCode* errorCode){
 276
 277     const char *testdatapath = loadTestData(errorCode);
 278     const char *srcdatapath = NULL;
 279     const char *relativepath = NULL;
 280     char *filename = NULL;
 281     UStringPrepProfile* profile = NULL;
 282
 283 #ifdef U_TOPSRCDIR
 284     srcdatapath = U_TOPSRCDIR;
 285     relativepath = U_FILE_SEP_STRING"test"U_FILE_SEP_STRING"testdata"U_FILE_SEP_STRING;
 286 #else
 287     srcdatapath = ctest_dataOutDir();
 288     relativepath = ".."U_FILE_SEP_STRING".."U_FILE_SEP_STRING"test"U_FILE_SEP_STRING"testdata"U_FILE_SEP_STRING;
 289 #endif
 290
 291     profile = usprep_open(testdatapath, binFileName, errorCode);
 292
 293     if(*errorCode == U_FILE_ACCESS_ERROR) {
 294         log_data_err("Failed to load %s data file. Error: %s \n", binFileName, u_errorName(*errorCode));
 295         return;
 296     } else if(U_FAILURE(*errorCode)){
 297         log_err("Failed to load %s data file. Error: %s \n", binFileName, u_errorName(*errorCode));
 298         return;
 299     }
 300     filename = (char*) malloc(strlen(srcdatapath)+strlen(relativepath)+strlen(txtFileName)+10 );
 301     /* open and load the txt file */
 302     strcpy(filename,srcdatapath);
 303     strcat(filename,relativepath);
 304     strcat(filename,txtFileName);
 305
 306     parseMappings(filename,profile, TRUE,errorCode);
 307
 308     free(filename);
 309 }
 310 #endif
 311 /*
 312  * Hey, Emacs, please set the following:
 313  *
 314  * Local Variables:
 315  * indent-tabs-mode: nil
 316  * End:
 317  *
 318  */