icuSources/i18n/collationdatareader.cpp

   1 /*
   2 *******************************************************************************
   3 * Copyright (C) 2013-2015, International Business Machines
   4 * Corporation and others.  All Rights Reserved.
   5 *******************************************************************************
   6 * collationdatareader.cpp
   7 *
   8 * created on: 2013feb07
   9 * created by: Markus W. Scherer
  10 */
  11
  12 #include "unicode/utypes.h"
  13
  14 #if !UCONFIG_NO_COLLATION
  15
  16 #include "unicode/ucol.h"
  17 #include "unicode/udata.h"
  18 #include "unicode/uscript.h"
  19 #include "cmemory.h"
  20 #include "collation.h"
  21 #include "collationdata.h"
  22 #include "collationdatareader.h"
  23 #include "collationfastlatin.h"
  24 #include "collationkeys.h"
  25 #include "collationrootelements.h"
  26 #include "collationsettings.h"
  27 #include "collationtailoring.h"
  28 #include "collunsafe.h"
  29 #include "normalizer2impl.h"
  30 #include "uassert.h"
  31 #include "ucmndata.h"
  32 #include "utrie2.h"
  33
  34 U_NAMESPACE_BEGIN
  35
  36 namespace {
  37
  38 int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) {
  39     return (i < length) ? indexes[i] : -1;
  40 }
  41
  42 }  // namespace
  43
  44 void
  45 CollationDataReader::read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength,
  46                           CollationTailoring &tailoring, UErrorCode &errorCode) {
  47     if(U_FAILURE(errorCode)) { return; }
  48     if(base != NULL) {
  49         if(inBytes == NULL || (0 <= inLength && inLength < 24)) {
  50             errorCode = U_ILLEGAL_ARGUMENT_ERROR;
  51             return;
  52         }
  53         const DataHeader *header = reinterpret_cast<const DataHeader *>(inBytes);
  54         if(!(header->dataHeader.magic1 == 0xda && header->dataHeader.magic2 == 0x27 &&
  55                 isAcceptable(tailoring.version, NULL, NULL, &header->info))) {
  56             errorCode = U_INVALID_FORMAT_ERROR;
  57             return;
  58         }
  59         if(base->getUCAVersion() != tailoring.getUCAVersion()) {
  60             errorCode = U_COLLATOR_VERSION_MISMATCH;
  61             return;
  62         }
  63         int32_t headerLength = header->dataHeader.headerSize;
  64         inBytes += headerLength;
  65         if(inLength >= 0) {
  66             inLength -= headerLength;
  67         }
  68     }
  69
  70     if(inBytes == NULL || (0 <= inLength && inLength < 8)) {
  71         errorCode = U_ILLEGAL_ARGUMENT_ERROR;
  72         return;
  73     }
  74     const int32_t *inIndexes = reinterpret_cast<const int32_t *>(inBytes);
  75     int32_t indexesLength = inIndexes[IX_INDEXES_LENGTH];
  76     if(indexesLength < 2 || (0 <= inLength && inLength < indexesLength * 4)) {
  77         errorCode = U_INVALID_FORMAT_ERROR;  // Not enough indexes.
  78         return;
  79     }
  80
  81     // Assume that the tailoring data is in initial state,
  82     // with NULL pointers and 0 lengths.
  83
  84     // Set pointers to non-empty data parts.
  85     // Do this in order of their byte offsets. (Should help porting to Java.)
  86
  87     int32_t index;  // one of the indexes[] slots
  88     int32_t offset;  // byte offset for the index part
  89     int32_t length;  // number of bytes in the index part
  90
  91     if(indexesLength > IX_TOTAL_SIZE) {
  92         length = inIndexes[IX_TOTAL_SIZE];
  93     } else if(indexesLength > IX_REORDER_CODES_OFFSET) {
  94         length = inIndexes[indexesLength - 1];
  95     } else {
  96         length = 0;  // only indexes, and inLength was already checked for them
  97     }
  98     if(0 <= inLength && inLength < length) {
  99         errorCode = U_INVALID_FORMAT_ERROR;
 100         return;
 101     }
 102
 103     const CollationData *baseData = base == NULL ? NULL : base->data;
 104     const int32_t *reorderCodes = NULL;
 105     int32_t reorderCodesLength = 0;
 106     const uint32_t *reorderRanges = NULL;
 107     int32_t reorderRangesLength = 0;
 108     index = IX_REORDER_CODES_OFFSET;
 109     offset = getIndex(inIndexes, indexesLength, index);
 110     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 111     if(length >= 4) {
 112         if(baseData == NULL) {
 113             // We assume for collation settings that
 114             // the base data does not have a reordering.
 115             errorCode = U_INVALID_FORMAT_ERROR;
 116             return;
 117         }
 118         reorderCodes = reinterpret_cast<const int32_t *>(inBytes + offset);
 119         reorderCodesLength = length / 4;
 120
 121         // The reorderRanges (if any) are the trailing reorderCodes entries.
 122         // Split the array at the boundary.
 123         // Script or reorder codes do not exceed 16-bit values.
 124         // Range limits are stored in the upper 16 bits, and are never 0.
 125         while(reorderRangesLength < reorderCodesLength &&
 126                 (reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) {
 127             ++reorderRangesLength;
 128         }
 129         U_ASSERT(reorderRangesLength < reorderCodesLength);
 130         if(reorderRangesLength != 0) {
 131             reorderCodesLength -= reorderRangesLength;
 132             reorderRanges = reinterpret_cast<const uint32_t *>(reorderCodes + reorderCodesLength);
 133         }
 134     }
 135
 136     // There should be a reorder table only if there are reorder codes.
 137     // However, when there are reorder codes the reorder table may be omitted to reduce
 138     // the data size.
 139     const uint8_t *reorderTable = NULL;
 140     index = IX_REORDER_TABLE_OFFSET;
 141     offset = getIndex(inIndexes, indexesLength, index);
 142     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 143     if(length >= 256) {
 144         if(reorderCodesLength == 0) {
 145             errorCode = U_INVALID_FORMAT_ERROR;  // Reordering table without reordering codes.
 146             return;
 147         }
 148         reorderTable = inBytes + offset;
 149     } else {
 150         // If we have reorder codes, then build the reorderTable at the end,
 151         // when the CollationData is otherwise complete.
 152     }
 153
 154     if(baseData != NULL && baseData->numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000)) {
 155         errorCode = U_INVALID_FORMAT_ERROR;
 156         return;
 157     }
 158     CollationData *data = NULL;  // Remains NULL if there are no mappings.
 159
 160     index = IX_TRIE_OFFSET;
 161     offset = getIndex(inIndexes, indexesLength, index);
 162     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 163     if(length >= 8) {
 164         if(!tailoring.ensureOwnedData(errorCode)) { return; }
 165         data = tailoring.ownedData;
 166         data->base = baseData;
 167         data->numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000;
 168         data->trie = tailoring.trie = utrie2_openFromSerialized(
 169             UTRIE2_32_VALUE_BITS, inBytes + offset, length, NULL,
 170             &errorCode);
 171         if(U_FAILURE(errorCode)) { return; }
 172     } else if(baseData != NULL) {
 173         // Use the base data. Only the settings are tailored.
 174         tailoring.data = baseData;
 175     } else {
 176         errorCode = U_INVALID_FORMAT_ERROR;  // No mappings.
 177         return;
 178     }
 179
 180     index = IX_CES_OFFSET;
 181     offset = getIndex(inIndexes, indexesLength, index);
 182     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 183     if(length >= 8) {
 184         if(data == NULL) {
 185             errorCode = U_INVALID_FORMAT_ERROR;  // Tailored ces without tailored trie.
 186             return;
 187         }
 188         data->ces = reinterpret_cast<const int64_t *>(inBytes + offset);
 189         data->cesLength = length / 8;
 190     }
 191
 192     index = IX_CE32S_OFFSET;
 193     offset = getIndex(inIndexes, indexesLength, index);
 194     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 195     if(length >= 4) {
 196         if(data == NULL) {
 197             errorCode = U_INVALID_FORMAT_ERROR;  // Tailored ce32s without tailored trie.
 198             return;
 199         }
 200         data->ce32s = reinterpret_cast<const uint32_t *>(inBytes + offset);
 201         data->ce32sLength = length / 4;
 202     }
 203
 204     int32_t jamoCE32sStart = getIndex(inIndexes, indexesLength, IX_JAMO_CE32S_START);
 205     if(jamoCE32sStart >= 0) {
 206         if(data == NULL || data->ce32s == NULL) {
 207             errorCode = U_INVALID_FORMAT_ERROR;  // Index into non-existent ce32s[].
 208             return;
 209         }
 210         data->jamoCE32s = data->ce32s + jamoCE32sStart;
 211     } else if(data == NULL) {
 212         // Nothing to do.
 213     } else if(baseData != NULL) {
 214         data->jamoCE32s = baseData->jamoCE32s;
 215     } else {
 216         errorCode = U_INVALID_FORMAT_ERROR;  // No Jamo CE32s for Hangul processing.
 217         return;
 218     }
 219
 220     index = IX_ROOT_ELEMENTS_OFFSET;
 221     offset = getIndex(inIndexes, indexesLength, index);
 222     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 223     if(length >= 4) {
 224         length /= 4;
 225         if(data == NULL || length <= CollationRootElements::IX_SEC_TER_BOUNDARIES) {
 226             errorCode = U_INVALID_FORMAT_ERROR;
 227             return;
 228         }
 229         data->rootElements = reinterpret_cast<const uint32_t *>(inBytes + offset);
 230         data->rootElementsLength = length;
 231         uint32_t commonSecTer = data->rootElements[CollationRootElements::IX_COMMON_SEC_AND_TER_CE];
 232         if(commonSecTer != Collation::COMMON_SEC_AND_TER_CE) {
 233             errorCode = U_INVALID_FORMAT_ERROR;
 234             return;
 235         }
 236         uint32_t secTerBoundaries = data->rootElements[CollationRootElements::IX_SEC_TER_BOUNDARIES];
 237         if((secTerBoundaries >> 24) < CollationKeys::SEC_COMMON_HIGH) {
 238             // [fixed last secondary common byte] is too low,
 239             // and secondary weights would collide with compressed common secondaries.
 240             errorCode = U_INVALID_FORMAT_ERROR;
 241             return;
 242         }
 243     }
 244
 245     index = IX_CONTEXTS_OFFSET;
 246     offset = getIndex(inIndexes, indexesLength, index);
 247     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 248     if(length >= 2) {
 249         if(data == NULL) {
 250             errorCode = U_INVALID_FORMAT_ERROR;  // Tailored contexts without tailored trie.
 251             return;
 252         }
 253         data->contexts = reinterpret_cast<const UChar *>(inBytes + offset);
 254         data->contextsLength = length / 2;
 255     }
 256
 257     index = IX_UNSAFE_BWD_OFFSET;
 258     offset = getIndex(inIndexes, indexesLength, index);
 259     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 260     if(length >= 2) {
 261         if(data == NULL) {
 262             errorCode = U_INVALID_FORMAT_ERROR;
 263             return;
 264         }
 265         if(baseData == NULL) {
 266 #if defined(COLLUNSAFE_COLL_VERSION) && defined (COLLUNSAFE_SERIALIZE)
 267           tailoring.unsafeBackwardSet = new UnicodeSet(unsafe_serializedData, unsafe_serializedCount, UnicodeSet::kSerialized, errorCode);
 268           if(tailoring.unsafeBackwardSet == NULL) {
 269             errorCode = U_MEMORY_ALLOCATION_ERROR;
 270             return;
 271           } else if (U_FAILURE(errorCode)) {
 272             return;
 273           }
 274 #else
 275             // Create the unsafe-backward set for the root collator.
 276             // Include all non-zero combining marks and trail surrogates.
 277             // We do this at load time, rather than at build time,
 278             // to simplify Unicode version bootstrapping:
 279             // The root data builder only needs the new FractionalUCA.txt data,
 280             // but it need not be built with a version of ICU already updated to
 281             // the corresponding new Unicode Character Database.
 282             //
 283             // The following is an optimized version of
 284             // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]").
 285             // It is faster and requires fewer code dependencies.
 286             tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff);  // trail surrogates
 287             if(tailoring.unsafeBackwardSet == NULL) {
 288                 errorCode = U_MEMORY_ALLOCATION_ERROR;
 289                 return;
 290             }
 291             data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet);
 292 #endif // !COLLUNSAFE_SERIALIZE || !COLLUNSAFE_COLL_VERSION
 293         } else {
 294             // Clone the root collator's set contents.
 295             tailoring.unsafeBackwardSet = static_cast<UnicodeSet *>(
 296                 baseData->unsafeBackwardSet->cloneAsThawed());
 297             if(tailoring.unsafeBackwardSet == NULL) {
 298                 errorCode = U_MEMORY_ALLOCATION_ERROR;
 299                 return;
 300             }
 301         }
 302         // Add the ranges from the data file to the unsafe-backward set.
 303         USerializedSet sset;
 304         const uint16_t *unsafeData = reinterpret_cast<const uint16_t *>(inBytes + offset);
 305         if(!uset_getSerializedSet(&sset, unsafeData, length / 2)) {
 306             errorCode = U_INVALID_FORMAT_ERROR;
 307             return;
 308         }
 309         int32_t count = uset_getSerializedRangeCount(&sset);
 310         for(int32_t i = 0; i < count; ++i) {
 311             UChar32 start, end;
 312             uset_getSerializedRange(&sset, i, &start, &end);
 313             tailoring.unsafeBackwardSet->add(start, end);
 314         }
 315         // Mark each lead surrogate as "unsafe"
 316         // if any of its 1024 associated supplementary code points is "unsafe".
 317         UChar32 c = 0x10000;
 318         for(UChar lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) {
 319             if(!tailoring.unsafeBackwardSet->containsNone(c, c + 0x3ff)) {
 320                 tailoring.unsafeBackwardSet->add(lead);
 321             }
 322         }
 323         tailoring.unsafeBackwardSet->freeze();
 324         data->unsafeBackwardSet = tailoring.unsafeBackwardSet;
 325     } else if(data == NULL) {
 326         // Nothing to do.
 327     } else if(baseData != NULL) {
 328         // No tailoring-specific data: Alias the root collator's set.
 329         data->unsafeBackwardSet = baseData->unsafeBackwardSet;
 330     } else {
 331         errorCode = U_INVALID_FORMAT_ERROR;  // No unsafeBackwardSet.
 332         return;
 333     }
 334
 335     // If the fast Latin format version is different,
 336     // or the version is set to 0 for "no fast Latin table",
 337     // then just always use the normal string comparison path.
 338     if(data != NULL) {
 339         data->fastLatinTable = NULL;
 340         data->fastLatinTableLength = 0;
 341         if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin::VERSION) {
 342             index = IX_FAST_LATIN_TABLE_OFFSET;
 343             offset = getIndex(inIndexes, indexesLength, index);
 344             length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 345             if(length >= 2) {
 346                 data->fastLatinTable = reinterpret_cast<const uint16_t *>(inBytes + offset);
 347                 data->fastLatinTableLength = length / 2;
 348                 if((*data->fastLatinTable >> 8) != CollationFastLatin::VERSION) {
 349                     errorCode = U_INVALID_FORMAT_ERROR;  // header vs. table version mismatch
 350                     return;
 351                 }
 352             } else if(baseData != NULL) {
 353                 data->fastLatinTable = baseData->fastLatinTable;
 354                 data->fastLatinTableLength = baseData->fastLatinTableLength;
 355             }
 356         }
 357     }
 358
 359     index = IX_SCRIPTS_OFFSET;
 360     offset = getIndex(inIndexes, indexesLength, index);
 361     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 362     if(length >= 2) {
 363         if(data == NULL) {
 364             errorCode = U_INVALID_FORMAT_ERROR;
 365             return;
 366         }
 367         const uint16_t *scripts = reinterpret_cast<const uint16_t *>(inBytes + offset);
 368         int32_t scriptsLength = length / 2;
 369         data->numScripts = scripts[0];
 370         // There must be enough entries for both arrays, including more than two range starts.
 371         data->scriptStartsLength = scriptsLength - (1 + data->numScripts + 16);
 372         if(data->scriptStartsLength <= 2 ||
 373                 CollationData::MAX_NUM_SCRIPT_RANGES < data->scriptStartsLength) {
 374             errorCode = U_INVALID_FORMAT_ERROR;
 375             return;
 376         }
 377         data->scriptsIndex = scripts + 1;
 378         data->scriptStarts = scripts + 1 + data->numScripts + 16;
 379         if(!(data->scriptStarts[0] == 0 &&
 380                 data->scriptStarts[1] == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8) &&
 381                 data->scriptStarts[data->scriptStartsLength - 1] ==
 382                         (Collation::TRAIL_WEIGHT_BYTE << 8))) {
 383             errorCode = U_INVALID_FORMAT_ERROR;
 384             return;
 385         }
 386     } else if(data == NULL) {
 387         // Nothing to do.
 388     } else if(baseData != NULL) {
 389         data->numScripts = baseData->numScripts;
 390         data->scriptsIndex = baseData->scriptsIndex;
 391         data->scriptStarts = baseData->scriptStarts;
 392         data->scriptStartsLength = baseData->scriptStartsLength;
 393     }
 394
 395     index = IX_COMPRESSIBLE_BYTES_OFFSET;
 396     offset = getIndex(inIndexes, indexesLength, index);
 397     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
 398     if(length >= 256) {
 399         if(data == NULL) {
 400             errorCode = U_INVALID_FORMAT_ERROR;
 401             return;
 402         }
 403         data->compressibleBytes = reinterpret_cast<const UBool *>(inBytes + offset);
 404     } else if(data == NULL) {
 405         // Nothing to do.
 406     } else if(baseData != NULL) {
 407         data->compressibleBytes = baseData->compressibleBytes;
 408     } else {
 409         errorCode = U_INVALID_FORMAT_ERROR;  // No compressibleBytes[].
 410         return;
 411     }
 412
 413     const CollationSettings &ts = *tailoring.settings;
 414     int32_t options = inIndexes[IX_OPTIONS] & 0xffff;
 415     uint16_t fastLatinPrimaries[CollationFastLatin::LATIN_LIMIT];
 416     int32_t fastLatinOptions = CollationFastLatin::getOptions(
 417             tailoring.data, ts, fastLatinPrimaries, UPRV_LENGTHOF(fastLatinPrimaries));
 418     if(options == ts.options && ts.variableTop != 0 &&
 419             reorderCodesLength == ts.reorderCodesLength &&
 420             uprv_memcmp(reorderCodes, ts.reorderCodes, reorderCodesLength * 4) == 0 &&
 421             fastLatinOptions == ts.fastLatinOptions &&
 422             (fastLatinOptions < 0 ||
 423                 uprv_memcmp(fastLatinPrimaries, ts.fastLatinPrimaries,
 424                             sizeof(fastLatinPrimaries)) == 0)) {
 425         return;
 426     }
 427
 428     CollationSettings *settings = SharedObject::copyOnWrite(tailoring.settings);
 429     if(settings == NULL) {
 430         errorCode = U_MEMORY_ALLOCATION_ERROR;
 431         return;
 432     }
 433     settings->options = options;
 434     // Set variableTop from options and scripts data.
 435     settings->variableTop = tailoring.data->getLastPrimaryForGroup(
 436             UCOL_REORDER_CODE_FIRST + settings->getMaxVariable());
 437     if(settings->variableTop == 0) {
 438         errorCode = U_INVALID_FORMAT_ERROR;
 439         return;
 440     }
 441
 442     if(reorderCodesLength != 0) {
 443         settings->aliasReordering(*baseData, reorderCodes, reorderCodesLength,
 444                                   reorderRanges, reorderRangesLength,
 445                                   reorderTable, errorCode);
 446     }
 447
 448     settings->fastLatinOptions = CollationFastLatin::getOptions(
 449         tailoring.data, *settings,
 450         settings->fastLatinPrimaries, UPRV_LENGTHOF(settings->fastLatinPrimaries));
 451 }
 452
 453 UBool U_CALLCONV
 454 CollationDataReader::isAcceptable(void *context,
 455                                   const char * /* type */, const char * /*name*/,
 456                                   const UDataInfo *pInfo) {
 457     if(
 458         pInfo->size >= 20 &&
 459         pInfo->isBigEndian == U_IS_BIG_ENDIAN &&
 460         pInfo->charsetFamily == U_CHARSET_FAMILY &&
 461         pInfo->dataFormat[0] == 0x55 &&  // dataFormat="UCol"
 462         pInfo->dataFormat[1] == 0x43 &&
 463         pInfo->dataFormat[2] == 0x6f &&
 464         pInfo->dataFormat[3] == 0x6c &&
 465         pInfo->formatVersion[0] == 5
 466     ) {
 467         UVersionInfo *version = static_cast<UVersionInfo *>(context);
 468         if(version != NULL) {
 469             uprv_memcpy(version, pInfo->dataVersion, 4);
 470         }
 471         return TRUE;
 472     } else {
 473         return FALSE;
 474     }
 475 }
 476
 477 U_NAMESPACE_END
 478
 479 #endif  // !UCONFIG_NO_COLLATION