[apple/icu.git] / icuSources / i18n / collationdatareader.cpp

// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others.  All Rights Reserved.
*******************************************************************************
* collationdatareader.cpp
*
* created on: 2013feb07
* created by: Markus W. Scherer
*/

#include "unicode/utypes.h"

#if !UCONFIG_NO_COLLATION

#include "unicode/ucol.h"
#include "unicode/udata.h"
#include "unicode/uscript.h"
#include "cmemory.h"
#include "collation.h"
#include "collationdata.h"
#include "collationdatareader.h"
#include "collationfastlatin.h"
#include "collationkeys.h"
#include "collationrootelements.h"
#include "collationsettings.h"
#include "collationtailoring.h"
#include "collunsafe.h"
#include "normalizer2impl.h"
#include "uassert.h"
#include "ucmndata.h"
#include "utrie2.h"

U_NAMESPACE_BEGIN

namespace {

int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) {
    return (i < length) ? indexes[i] : -1;
}

}  // namespace

void
CollationDataReader::read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength,
                          CollationTailoring &tailoring, UErrorCode &errorCode) {
    if(U_FAILURE(errorCode)) { return; }
    if(base != NULL) {
        if(inBytes == NULL || (0 <= inLength && inLength < 24)) {
            errorCode = U_ILLEGAL_ARGUMENT_ERROR;
            return;
        }
        const DataHeader *header = reinterpret_cast<const DataHeader *>(inBytes);
        if(!(header->dataHeader.magic1 == 0xda && header->dataHeader.magic2 == 0x27 &&
                isAcceptable(tailoring.version, NULL, NULL, &header->info))) {
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
        if(base->getUCAVersion() != tailoring.getUCAVersion()) {
            errorCode = U_COLLATOR_VERSION_MISMATCH;
            return;
        }
        int32_t headerLength = header->dataHeader.headerSize;
        inBytes += headerLength;
        if(inLength >= 0) {
            inLength -= headerLength;
        }
    }

    if(inBytes == NULL || (0 <= inLength && inLength < 8)) {
        errorCode = U_ILLEGAL_ARGUMENT_ERROR;
        return;
    }
    const int32_t *inIndexes = reinterpret_cast<const int32_t *>(inBytes);
    int32_t indexesLength = inIndexes[IX_INDEXES_LENGTH];
    if(indexesLength < 2 || (0 <= inLength && inLength < indexesLength * 4)) {
        errorCode = U_INVALID_FORMAT_ERROR;  // Not enough indexes.
        return;
    }

    // Assume that the tailoring data is in initial state,
    // with NULL pointers and 0 lengths.

    // Set pointers to non-empty data parts.
    // Do this in order of their byte offsets. (Should help porting to Java.)

    int32_t index;  // one of the indexes[] slots
    int32_t offset;  // byte offset for the index part
    int32_t length;  // number of bytes in the index part

    if(indexesLength > IX_TOTAL_SIZE) {
        length = inIndexes[IX_TOTAL_SIZE];
    } else if(indexesLength > IX_REORDER_CODES_OFFSET) {
        length = inIndexes[indexesLength - 1];
    } else {
        length = 0;  // only indexes, and inLength was already checked for them
    }
    if(0 <= inLength && inLength < length) {
        errorCode = U_INVALID_FORMAT_ERROR;
        return;
    }

    const CollationData *baseData = base == NULL ? NULL : base->data;
    const int32_t *reorderCodes = NULL;
    int32_t reorderCodesLength = 0;
    const uint32_t *reorderRanges = NULL;
    int32_t reorderRangesLength = 0;
    index = IX_REORDER_CODES_OFFSET;
    offset = getIndex(inIndexes, indexesLength, index);
    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
    if(length >= 4) {
        if(baseData == NULL) {
            // We assume for collation settings that
            // the base data does not have a reordering.
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
        reorderCodes = reinterpret_cast<const int32_t *>(inBytes + offset);
        reorderCodesLength = length / 4;

        // The reorderRanges (if any) are the trailing reorderCodes entries.
        // Split the array at the boundary.
        // Script or reorder codes do not exceed 16-bit values.
        // Range limits are stored in the upper 16 bits, and are never 0.
        while(reorderRangesLength < reorderCodesLength &&
                (reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) {
            ++reorderRangesLength;
        }
        U_ASSERT(reorderRangesLength < reorderCodesLength);
        if(reorderRangesLength != 0) {
            reorderCodesLength -= reorderRangesLength;
            reorderRanges = reinterpret_cast<const uint32_t *>(reorderCodes + reorderCodesLength);
        }
    }

    // There should be a reorder table only if there are reorder codes.
    // However, when there are reorder codes the reorder table may be omitted to reduce
    // the data size.
    const uint8_t *reorderTable = NULL;
    index = IX_REORDER_TABLE_OFFSET;
    offset = getIndex(inIndexes, indexesLength, index);
    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
    if(length >= 256) {
        if(reorderCodesLength == 0) {
            errorCode = U_INVALID_FORMAT_ERROR;  // Reordering table without reordering codes.
            return;
        }
        reorderTable = inBytes + offset;
    } else {
        // If we have reorder codes, then build the reorderTable at the end,
        // when the CollationData is otherwise complete.
    }

    if(baseData != NULL && baseData->numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000)) {
        errorCode = U_INVALID_FORMAT_ERROR;
        return;
    }
    CollationData *data = NULL;  // Remains NULL if there are no mappings.

    index = IX_TRIE_OFFSET;
    offset = getIndex(inIndexes, indexesLength, index);
    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
    if(length >= 8) {
        if(!tailoring.ensureOwnedData(errorCode)) { return; }
        data = tailoring.ownedData;
        data->base = baseData;
        data->numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000;
        data->trie = tailoring.trie = utrie2_openFromSerialized(
            UTRIE2_32_VALUE_BITS, inBytes + offset, length, NULL,
            &errorCode);
        if(U_FAILURE(errorCode)) { return; }
    } else if(baseData != NULL) {
        // Use the base data. Only the settings are tailored.
        tailoring.data = baseData;
    } else {
        errorCode = U_INVALID_FORMAT_ERROR;  // No mappings.
        return;
    }

    index = IX_CES_OFFSET;
    offset = getIndex(inIndexes, indexesLength, index);
    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
    if(length >= 8) {
        if(data == NULL) {
            errorCode = U_INVALID_FORMAT_ERROR;  // Tailored ces without tailored trie.
            return;
        }
        data->ces = reinterpret_cast<const int64_t *>(inBytes + offset);
        data->cesLength = length / 8;
    }

    index = IX_CE32S_OFFSET;
    offset = getIndex(inIndexes, indexesLength, index);
    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
    if(length >= 4) {
        if(data == NULL) {
            errorCode = U_INVALID_FORMAT_ERROR;  // Tailored ce32s without tailored trie.
            return;
        }
        data->ce32s = reinterpret_cast<const uint32_t *>(inBytes + offset);
        data->ce32sLength = length / 4;
    }

    int32_t jamoCE32sStart = getIndex(inIndexes, indexesLength, IX_JAMO_CE32S_START);
    if(jamoCE32sStart >= 0) {
        if(data == NULL || data->ce32s == NULL) {
            errorCode = U_INVALID_FORMAT_ERROR;  // Index into non-existent ce32s[].
            return;
        }
        data->jamoCE32s = data->ce32s + jamoCE32sStart;
    } else if(data == NULL) {
        // Nothing to do.
    } else if(baseData != NULL) {
        data->jamoCE32s = baseData->jamoCE32s;
    } else {
        errorCode = U_INVALID_FORMAT_ERROR;  // No Jamo CE32s for Hangul processing.
        return;
    }

    index = IX_ROOT_ELEMENTS_OFFSET;
    offset = getIndex(inIndexes, indexesLength, index);
    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
    if(length >= 4) {
        length /= 4;
        if(data == NULL || length <= CollationRootElements::IX_SEC_TER_BOUNDARIES) {
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
        data->rootElements = reinterpret_cast<const uint32_t *>(inBytes + offset);
        data->rootElementsLength = length;
        uint32_t commonSecTer = data->rootElements[CollationRootElements::IX_COMMON_SEC_AND_TER_CE];
        if(commonSecTer != Collation::COMMON_SEC_AND_TER_CE) {
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
        uint32_t secTerBoundaries = data->rootElements[CollationRootElements::IX_SEC_TER_BOUNDARIES];
        if((secTerBoundaries >> 24) < CollationKeys::SEC_COMMON_HIGH) {
            // [fixed last secondary common byte] is too low,
            // and secondary weights would collide with compressed common secondaries.
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
    }

    index = IX_CONTEXTS_OFFSET;
    offset = getIndex(inIndexes, indexesLength, index);
    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
    if(length >= 2) {
        if(data == NULL) {
            errorCode = U_INVALID_FORMAT_ERROR;  // Tailored contexts without tailored trie.
            return;
        }
        data->contexts = reinterpret_cast<const UChar *>(inBytes + offset);
        data->contextsLength = length / 2;
    }

    index = IX_UNSAFE_BWD_OFFSET;
    offset = getIndex(inIndexes, indexesLength, index);
    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
    if(length >= 2) {
        if(data == NULL) {
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
        if(baseData == NULL) {
#if defined(COLLUNSAFE_COLL_VERSION) && defined (COLLUNSAFE_SERIALIZE)
          tailoring.unsafeBackwardSet = new UnicodeSet(unsafe_serializedData, unsafe_serializedCount, UnicodeSet::kSerialized, errorCode);
          if(tailoring.unsafeBackwardSet == NULL) {
            errorCode = U_MEMORY_ALLOCATION_ERROR;
            return;
          } else if (U_FAILURE(errorCode)) {
            return;
          }
#else
            // Create the unsafe-backward set for the root collator.
            // Include all non-zero combining marks and trail surrogates.
            // We do this at load time, rather than at build time,
            // to simplify Unicode version bootstrapping:
            // The root data builder only needs the new FractionalUCA.txt data,
            // but it need not be built with a version of ICU already updated to
            // the corresponding new Unicode Character Database.
            //
            // The following is an optimized version of
            // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]").
            // It is faster and requires fewer code dependencies.
            tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff);  // trail surrogates
            if(tailoring.unsafeBackwardSet == NULL) {
                errorCode = U_MEMORY_ALLOCATION_ERROR;
                return;
            }
            data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet);
#endif // !COLLUNSAFE_SERIALIZE || !COLLUNSAFE_COLL_VERSION
        } else {
            // Clone the root collator's set contents.
            tailoring.unsafeBackwardSet = static_cast<UnicodeSet *>(
                baseData->unsafeBackwardSet->cloneAsThawed());
            if(tailoring.unsafeBackwardSet == NULL) {
                errorCode = U_MEMORY_ALLOCATION_ERROR;
                return;
            }
        }
        // Add the ranges from the data file to the unsafe-backward set.
        USerializedSet sset;
        const uint16_t *unsafeData = reinterpret_cast<const uint16_t *>(inBytes + offset);
        if(!uset_getSerializedSet(&sset, unsafeData, length / 2)) {
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
        int32_t count = uset_getSerializedRangeCount(&sset);
        for(int32_t i = 0; i < count; ++i) {
            UChar32 start, end;
            uset_getSerializedRange(&sset, i, &start, &end);
            tailoring.unsafeBackwardSet->add(start, end);
        }
        // Mark each lead surrogate as "unsafe"
        // if any of its 1024 associated supplementary code points is "unsafe".
        UChar32 c = 0x10000;
        for(UChar lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) {
            if(!tailoring.unsafeBackwardSet->containsNone(c, c + 0x3ff)) {
                tailoring.unsafeBackwardSet->add(lead);
            }
        }
        tailoring.unsafeBackwardSet->freeze();
        data->unsafeBackwardSet = tailoring.unsafeBackwardSet;
    } else if(data == NULL) {
        // Nothing to do.
    } else if(baseData != NULL) {
        // No tailoring-specific data: Alias the root collator's set.
        data->unsafeBackwardSet = baseData->unsafeBackwardSet;
    } else {
        errorCode = U_INVALID_FORMAT_ERROR;  // No unsafeBackwardSet.
        return;
    }

    // If the fast Latin format version is different,
    // or the version is set to 0 for "no fast Latin table",
    // then just always use the normal string comparison path.
    if(data != NULL) {
        data->fastLatinTable = NULL;
        data->fastLatinTableLength = 0;
        if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin::VERSION) {
            index = IX_FAST_LATIN_TABLE_OFFSET;
            offset = getIndex(inIndexes, indexesLength, index);
            length = getIndex(inIndexes, indexesLength, index + 1) - offset;
            if(length >= 2) {
                data->fastLatinTable = reinterpret_cast<const uint16_t *>(inBytes + offset);
                data->fastLatinTableLength = length / 2;
                if((*data->fastLatinTable >> 8) != CollationFastLatin::VERSION) {
                    errorCode = U_INVALID_FORMAT_ERROR;  // header vs. table version mismatch
                    return;
                }
            } else if(baseData != NULL) {
                data->fastLatinTable = baseData->fastLatinTable;
                data->fastLatinTableLength = baseData->fastLatinTableLength;
            }
        }
    }

    index = IX_SCRIPTS_OFFSET;
    offset = getIndex(inIndexes, indexesLength, index);
    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
    if(length >= 2) {
        if(data == NULL) {
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
        const uint16_t *scripts = reinterpret_cast<const uint16_t *>(inBytes + offset);
        int32_t scriptsLength = length / 2;
        data->numScripts = scripts[0];
        // There must be enough entries for both arrays, including more than two range starts.
        data->scriptStartsLength = scriptsLength - (1 + data->numScripts + 16);
        if(data->scriptStartsLength <= 2 ||
                CollationData::MAX_NUM_SCRIPT_RANGES < data->scriptStartsLength) {
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
        data->scriptsIndex = scripts + 1;
        data->scriptStarts = scripts + 1 + data->numScripts + 16;
        if(!(data->scriptStarts[0] == 0 &&
                data->scriptStarts[1] == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8) &&
                data->scriptStarts[data->scriptStartsLength - 1] ==
                        (Collation::TRAIL_WEIGHT_BYTE << 8))) {
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
    } else if(data == NULL) {
        // Nothing to do.
    } else if(baseData != NULL) {
        data->numScripts = baseData->numScripts;
        data->scriptsIndex = baseData->scriptsIndex;
        data->scriptStarts = baseData->scriptStarts;
        data->scriptStartsLength = baseData->scriptStartsLength;
    }

    index = IX_COMPRESSIBLE_BYTES_OFFSET;
    offset = getIndex(inIndexes, indexesLength, index);
    length = getIndex(inIndexes, indexesLength, index + 1) - offset;
    if(length >= 256) {
        if(data == NULL) {
            errorCode = U_INVALID_FORMAT_ERROR;
            return;
        }
        data->compressibleBytes = reinterpret_cast<const UBool *>(inBytes + offset);
    } else if(data == NULL) {
        // Nothing to do.
    } else if(baseData != NULL) {
        data->compressibleBytes = baseData->compressibleBytes;
    } else {
        errorCode = U_INVALID_FORMAT_ERROR;  // No compressibleBytes[].
        return;
    }

    const CollationSettings &ts = *tailoring.settings;
    int32_t options = inIndexes[IX_OPTIONS] & 0xffff;
    uint16_t fastLatinPrimaries[CollationFastLatin::LATIN_LIMIT];
    int32_t fastLatinOptions = CollationFastLatin::getOptions(
            tailoring.data, ts, fastLatinPrimaries, UPRV_LENGTHOF(fastLatinPrimaries));
    if(options == ts.options && ts.variableTop != 0 &&
            reorderCodesLength == ts.reorderCodesLength &&
            (reorderCodesLength == 0 ||
                uprv_memcmp(reorderCodes, ts.reorderCodes, reorderCodesLength * 4) == 0) &&
            fastLatinOptions == ts.fastLatinOptions &&
            (fastLatinOptions < 0 ||
                uprv_memcmp(fastLatinPrimaries, ts.fastLatinPrimaries,
                            sizeof(fastLatinPrimaries)) == 0)) {
        return;
    }

    CollationSettings *settings = SharedObject::copyOnWrite(tailoring.settings);
    if(settings == NULL) {
        errorCode = U_MEMORY_ALLOCATION_ERROR;
        return;
    }
    settings->options = options;
    // Set variableTop from options and scripts data.
    settings->variableTop = tailoring.data->getLastPrimaryForGroup(
            UCOL_REORDER_CODE_FIRST + settings->getMaxVariable());
    if(settings->variableTop == 0) {
        errorCode = U_INVALID_FORMAT_ERROR;
        return;
    }

    if(reorderCodesLength != 0) {
        settings->aliasReordering(*baseData, reorderCodes, reorderCodesLength,
                                  reorderRanges, reorderRangesLength,
                                  reorderTable, errorCode);
    }

    settings->fastLatinOptions = CollationFastLatin::getOptions(
        tailoring.data, *settings,
        settings->fastLatinPrimaries, UPRV_LENGTHOF(settings->fastLatinPrimaries));
}

UBool U_CALLCONV
CollationDataReader::isAcceptable(void *context,
                                  const char * /* type */, const char * /*name*/,
                                  const UDataInfo *pInfo) {
    if(
        pInfo->size >= 20 &&
        pInfo->isBigEndian == U_IS_BIG_ENDIAN &&
        pInfo->charsetFamily == U_CHARSET_FAMILY &&
        pInfo->dataFormat[0] == 0x55 &&  // dataFormat="UCol"
        pInfo->dataFormat[1] == 0x43 &&
        pInfo->dataFormat[2] == 0x6f &&
        pInfo->dataFormat[3] == 0x6c &&
        pInfo->formatVersion[0] == 5
    ) {
        UVersionInfo *version = static_cast<UVersionInfo *>(context);
        if(version != NULL) {
            uprv_memcpy(version, pInfo->dataVersion, 4);
        }
        return TRUE;
    } else {
        return FALSE;
    }
}

U_NAMESPACE_END

#endif  // !UCONFIG_NO_COLLATION
Commit	Line	Data
f3c0d7a5 A	1	// © 2016 and later: Unicode, Inc. and others.
f3c0d7a5 A	2	// License & terms of use: http://www.unicode.org/copyright.html
57a6839d A	3	/*
57a6839d A	4	*******************************************************************************
b331163b	5	* Copyright (C) 2013-2015, International Business Machines
57a6839d A	6	* Corporation and others. All Rights Reserved.
	7	*******************************************************************************
	8	* collationdatareader.cpp
	9	*
	10	* created on: 2013feb07
	11	* created by: Markus W. Scherer
	12	*/
	13
	14	#include "unicode/utypes.h"
	15
	16	#if !UCONFIG_NO_COLLATION
	17
	18	#include "unicode/ucol.h"
	19	#include "unicode/udata.h"
	20	#include "unicode/uscript.h"
	21	#include "cmemory.h"
	22	#include "collation.h"
	23	#include "collationdata.h"
	24	#include "collationdatareader.h"
	25	#include "collationfastlatin.h"
	26	#include "collationkeys.h"
	27	#include "collationrootelements.h"
	28	#include "collationsettings.h"
	29	#include "collationtailoring.h"
2ca993e8	30	#include "collunsafe.h"
57a6839d A	31	#include "normalizer2impl.h"
	32	#include "uassert.h"
	33	#include "ucmndata.h"
	34	#include "utrie2.h"
	35
57a6839d A	36	U_NAMESPACE_BEGIN
	37
	38	namespace {
	39
	40	int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) {
	41	return (i < length) ? indexes[i] : -1;
	42	}
	43
	44	} // namespace
	45
	46	void
	47	CollationDataReader::read(const CollationTailoring base, const uint8_t inBytes, int32_t inLength,
	48	CollationTailoring &tailoring, UErrorCode &errorCode) {
	49	if(U_FAILURE(errorCode)) { return; }
	50	if(base != NULL) {
	51	if(inBytes == NULL \|\| (0 <= inLength && inLength < 24)) {
	52	errorCode = U_ILLEGAL_ARGUMENT_ERROR;
	53	return;
	54	}
	55	const DataHeader header = reinterpret_cast<const DataHeader >(inBytes);
	56	if(!(header->dataHeader.magic1 == 0xda && header->dataHeader.magic2 == 0x27 &&
	57	isAcceptable(tailoring.version, NULL, NULL, &header->info))) {
	58	errorCode = U_INVALID_FORMAT_ERROR;
	59	return;
	60	}
	61	if(base->getUCAVersion() != tailoring.getUCAVersion()) {
	62	errorCode = U_COLLATOR_VERSION_MISMATCH;
	63	return;
	64	}
	65	int32_t headerLength = header->dataHeader.headerSize;
	66	inBytes += headerLength;
	67	if(inLength >= 0) {
	68	inLength -= headerLength;
	69	}
	70	}
	71
	72	if(inBytes == NULL \|\| (0 <= inLength && inLength < 8)) {
	73	errorCode = U_ILLEGAL_ARGUMENT_ERROR;
	74	return;
	75	}
	76	const int32_t inIndexes = reinterpret_cast<const int32_t >(inBytes);
	77	int32_t indexesLength = inIndexes[IX_INDEXES_LENGTH];
	78	if(indexesLength < 2 \|\| (0 <= inLength && inLength < indexesLength * 4)) {
	79	errorCode = U_INVALID_FORMAT_ERROR; // Not enough indexes.
	80	return;
	81	}
	82
	83	// Assume that the tailoring data is in initial state,
	84	// with NULL pointers and 0 lengths.
	85
	86	// Set pointers to non-empty data parts.
	87	// Do this in order of their byte offsets. (Should help porting to Java.)
	88
	89	int32_t index; // one of the indexes[] slots
	90	int32_t offset; // byte offset for the index part
	91	int32_t length; // number of bytes in the index part
	92
	93	if(indexesLength > IX_TOTAL_SIZE) {
	94	length = inIndexes[IX_TOTAL_SIZE];
	95	} else if(indexesLength > IX_REORDER_CODES_OFFSET) {
	96	length = inIndexes[indexesLength - 1];
	97	} else {
	98	length = 0; // only indexes, and inLength was already checked for them
	99	}
100	if(0 <= inLength && inLength < length) {
101	errorCode = U_INVALID_FORMAT_ERROR;
102	return;
103	}
104
105	const CollationData *baseData = base == NULL ? NULL : base->data;
106	const int32_t *reorderCodes = NULL;
107	int32_t reorderCodesLength = 0;
b331163b A	108	const uint32_t *reorderRanges = NULL;
b331163b A	109	int32_t reorderRangesLength = 0;
57a6839d A	110	index = IX_REORDER_CODES_OFFSET;
	111	offset = getIndex(inIndexes, indexesLength, index);
	112	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
	113	if(length >= 4) {
	114	if(baseData == NULL) {
	115	// We assume for collation settings that
	116	// the base data does not have a reordering.
	117	errorCode = U_INVALID_FORMAT_ERROR;
	118	return;
	119	}
	120	reorderCodes = reinterpret_cast<const int32_t *>(inBytes + offset);
	121	reorderCodesLength = length / 4;
b331163b A	122
	123	// The reorderRanges (if any) are the trailing reorderCodes entries.
	124	// Split the array at the boundary.
	125	// Script or reorder codes do not exceed 16-bit values.
	126	// Range limits are stored in the upper 16 bits, and are never 0.
	127	while(reorderRangesLength < reorderCodesLength &&
	128	(reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) {
	129	++reorderRangesLength;
	130	}
	131	U_ASSERT(reorderRangesLength < reorderCodesLength);
	132	if(reorderRangesLength != 0) {
	133	reorderCodesLength -= reorderRangesLength;
	134	reorderRanges = reinterpret_cast<const uint32_t *>(reorderCodes + reorderCodesLength);
	135	}
57a6839d A	136	}
	137
	138	// There should be a reorder table only if there are reorder codes.
	139	// However, when there are reorder codes the reorder table may be omitted to reduce
	140	// the data size.
	141	const uint8_t *reorderTable = NULL;
	142	index = IX_REORDER_TABLE_OFFSET;
	143	offset = getIndex(inIndexes, indexesLength, index);
	144	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
	145	if(length >= 256) {
	146	if(reorderCodesLength == 0) {
	147	errorCode = U_INVALID_FORMAT_ERROR; // Reordering table without reordering codes.
	148	return;
	149	}
	150	reorderTable = inBytes + offset;
	151	} else {
	152	// If we have reorder codes, then build the reorderTable at the end,
	153	// when the CollationData is otherwise complete.
	154	}
	155
	156	if(baseData != NULL && baseData->numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000)) {
	157	errorCode = U_INVALID_FORMAT_ERROR;
	158	return;
	159	}
	160	CollationData *data = NULL; // Remains NULL if there are no mappings.
	161
	162	index = IX_TRIE_OFFSET;
	163	offset = getIndex(inIndexes, indexesLength, index);
	164	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
	165	if(length >= 8) {
	166	if(!tailoring.ensureOwnedData(errorCode)) { return; }
	167	data = tailoring.ownedData;
	168	data->base = baseData;
	169	data->numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000;
	170	data->trie = tailoring.trie = utrie2_openFromSerialized(
	171	UTRIE2_32_VALUE_BITS, inBytes + offset, length, NULL,
	172	&errorCode);
	173	if(U_FAILURE(errorCode)) { return; }
	174	} else if(baseData != NULL) {
	175	// Use the base data. Only the settings are tailored.
	176	tailoring.data = baseData;
	177	} else {
	178	errorCode = U_INVALID_FORMAT_ERROR; // No mappings.
	179	return;
	180	}
	181
	182	index = IX_CES_OFFSET;
	183	offset = getIndex(inIndexes, indexesLength, index);
	184	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
	185	if(length >= 8) {
	186	if(data == NULL) {
	187	errorCode = U_INVALID_FORMAT_ERROR; // Tailored ces without tailored trie.
	188	return;
	189	}
	190	data->ces = reinterpret_cast<const int64_t *>(inBytes + offset);
	191	data->cesLength = length / 8;
	192	}
	193
	194	index = IX_CE32S_OFFSET;
	195	offset = getIndex(inIndexes, indexesLength, index);
	196	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
	197	if(length >= 4) {
	198	if(data == NULL) {
	199	errorCode = U_INVALID_FORMAT_ERROR; // Tailored ce32s without tailored trie.
200	return;
201	}
202	data->ce32s = reinterpret_cast<const uint32_t *>(inBytes + offset);
203	data->ce32sLength = length / 4;
204	}
205
206	int32_t jamoCE32sStart = getIndex(inIndexes, indexesLength, IX_JAMO_CE32S_START);
207	if(jamoCE32sStart >= 0) {
208	if(data == NULL \|\| data->ce32s == NULL) {
209	errorCode = U_INVALID_FORMAT_ERROR; // Index into non-existent ce32s[].
210	return;
211	}
212	data->jamoCE32s = data->ce32s + jamoCE32sStart;
213	} else if(data == NULL) {
214	// Nothing to do.
215	} else if(baseData != NULL) {
216	data->jamoCE32s = baseData->jamoCE32s;
217	} else {
218	errorCode = U_INVALID_FORMAT_ERROR; // No Jamo CE32s for Hangul processing.
219	return;
220	}
221
222	index = IX_ROOT_ELEMENTS_OFFSET;
223	offset = getIndex(inIndexes, indexesLength, index);
224	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
225	if(length >= 4) {
226	length /= 4;
227	if(data == NULL \|\| length <= CollationRootElements::IX_SEC_TER_BOUNDARIES) {
228	errorCode = U_INVALID_FORMAT_ERROR;
229	return;
230	}
231	data->rootElements = reinterpret_cast<const uint32_t *>(inBytes + offset);
232	data->rootElementsLength = length;
233	uint32_t commonSecTer = data->rootElements[CollationRootElements::IX_COMMON_SEC_AND_TER_CE];
234	if(commonSecTer != Collation::COMMON_SEC_AND_TER_CE) {
235	errorCode = U_INVALID_FORMAT_ERROR;
236	return;
237	}
238	uint32_t secTerBoundaries = data->rootElements[CollationRootElements::IX_SEC_TER_BOUNDARIES];
239	if((secTerBoundaries >> 24) < CollationKeys::SEC_COMMON_HIGH) {
240	// [fixed last secondary common byte] is too low,
241	// and secondary weights would collide with compressed common secondaries.
242	errorCode = U_INVALID_FORMAT_ERROR;
243	return;
244	}
245	}
246
247	index = IX_CONTEXTS_OFFSET;
248	offset = getIndex(inIndexes, indexesLength, index);
249	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
250	if(length >= 2) {
251	if(data == NULL) {
252	errorCode = U_INVALID_FORMAT_ERROR; // Tailored contexts without tailored trie.
253	return;
254	}
255	data->contexts = reinterpret_cast<const UChar *>(inBytes + offset);
256	data->contextsLength = length / 2;
257	}
258
259	index = IX_UNSAFE_BWD_OFFSET;
260	offset = getIndex(inIndexes, indexesLength, index);
261	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
262	if(length >= 2) {
263	if(data == NULL) {
264	errorCode = U_INVALID_FORMAT_ERROR;
265	return;
266	}
267	if(baseData == NULL) {
2ca993e8 A	268	#if defined(COLLUNSAFE_COLL_VERSION) && defined (COLLUNSAFE_SERIALIZE)
	269	tailoring.unsafeBackwardSet = new UnicodeSet(unsafe_serializedData, unsafe_serializedCount, UnicodeSet::kSerialized, errorCode);
	270	if(tailoring.unsafeBackwardSet == NULL) {
	271	errorCode = U_MEMORY_ALLOCATION_ERROR;
	272	return;
	273	} else if (U_FAILURE(errorCode)) {
	274	return;
	275	}
	276	#else
57a6839d A	277	// Create the unsafe-backward set for the root collator.
	278	// Include all non-zero combining marks and trail surrogates.
	279	// We do this at load time, rather than at build time,
	280	// to simplify Unicode version bootstrapping:
	281	// The root data builder only needs the new FractionalUCA.txt data,
	282	// but it need not be built with a version of ICU already updated to
	283	// the corresponding new Unicode Character Database.
	284	//
	285	// The following is an optimized version of
	286	// new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]").
	287	// It is faster and requires fewer code dependencies.
	288	tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff); // trail surrogates
	289	if(tailoring.unsafeBackwardSet == NULL) {
	290	errorCode = U_MEMORY_ALLOCATION_ERROR;
	291	return;
	292	}
	293	data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet);
2ca993e8	294	#endif // !COLLUNSAFE_SERIALIZE \|\| !COLLUNSAFE_COLL_VERSION
57a6839d A	295	} else {
	296	// Clone the root collator's set contents.
	297	tailoring.unsafeBackwardSet = static_cast<UnicodeSet *>(
	298	baseData->unsafeBackwardSet->cloneAsThawed());
	299	if(tailoring.unsafeBackwardSet == NULL) {
	300	errorCode = U_MEMORY_ALLOCATION_ERROR;
	301	return;
	302	}
	303	}
	304	// Add the ranges from the data file to the unsafe-backward set.
	305	USerializedSet sset;
	306	const uint16_t unsafeData = reinterpret_cast<const uint16_t >(inBytes + offset);
	307	if(!uset_getSerializedSet(&sset, unsafeData, length / 2)) {
	308	errorCode = U_INVALID_FORMAT_ERROR;
	309	return;
	310	}
	311	int32_t count = uset_getSerializedRangeCount(&sset);
	312	for(int32_t i = 0; i < count; ++i) {
	313	UChar32 start, end;
	314	uset_getSerializedRange(&sset, i, &start, &end);
	315	tailoring.unsafeBackwardSet->add(start, end);
	316	}
	317	// Mark each lead surrogate as "unsafe"
	318	// if any of its 1024 associated supplementary code points is "unsafe".
	319	UChar32 c = 0x10000;
	320	for(UChar lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) {
	321	if(!tailoring.unsafeBackwardSet->containsNone(c, c + 0x3ff)) {
	322	tailoring.unsafeBackwardSet->add(lead);
	323	}
	324	}
	325	tailoring.unsafeBackwardSet->freeze();
	326	data->unsafeBackwardSet = tailoring.unsafeBackwardSet;
	327	} else if(data == NULL) {
	328	// Nothing to do.
	329	} else if(baseData != NULL) {
	330	// No tailoring-specific data: Alias the root collator's set.
	331	data->unsafeBackwardSet = baseData->unsafeBackwardSet;
	332	} else {
	333	errorCode = U_INVALID_FORMAT_ERROR; // No unsafeBackwardSet.
	334	return;
	335	}
	336
	337	// If the fast Latin format version is different,
	338	// or the version is set to 0 for "no fast Latin table",
	339	// then just always use the normal string comparison path.
	340	if(data != NULL) {
	341	data->fastLatinTable = NULL;
	342	data->fastLatinTableLength = 0;
	343	if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin::VERSION) {
	344	index = IX_FAST_LATIN_TABLE_OFFSET;
	345	offset = getIndex(inIndexes, indexesLength, index);
	346	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
	347	if(length >= 2) {
	348	data->fastLatinTable = reinterpret_cast<const uint16_t *>(inBytes + offset);
	349	data->fastLatinTableLength = length / 2;
	350	if((*data->fastLatinTable >> 8) != CollationFastLatin::VERSION) {
	351	errorCode = U_INVALID_FORMAT_ERROR; // header vs. table version mismatch
	352	return;
	353	}
	354	} else if(baseData != NULL) {
	355	data->fastLatinTable = baseData->fastLatinTable;
	356	data->fastLatinTableLength = baseData->fastLatinTableLength;
	357	}
	358	}
359	}
360
361	index = IX_SCRIPTS_OFFSET;
362	offset = getIndex(inIndexes, indexesLength, index);
363	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
364	if(length >= 2) {
365	if(data == NULL) {
366	errorCode = U_INVALID_FORMAT_ERROR;
367	return;
368	}
b331163b A	369	const uint16_t scripts = reinterpret_cast<const uint16_t >(inBytes + offset);
	370	int32_t scriptsLength = length / 2;
	371	data->numScripts = scripts[0];
	372	// There must be enough entries for both arrays, including more than two range starts.
	373	data->scriptStartsLength = scriptsLength - (1 + data->numScripts + 16);
	374	if(data->scriptStartsLength <= 2 \|\|
	375	CollationData::MAX_NUM_SCRIPT_RANGES < data->scriptStartsLength) {
	376	errorCode = U_INVALID_FORMAT_ERROR;
	377	return;
	378	}
	379	data->scriptsIndex = scripts + 1;
	380	data->scriptStarts = scripts + 1 + data->numScripts + 16;
	381	if(!(data->scriptStarts[0] == 0 &&
	382	data->scriptStarts[1] == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8) &&
	383	data->scriptStarts[data->scriptStartsLength - 1] ==
	384	(Collation::TRAIL_WEIGHT_BYTE << 8))) {
	385	errorCode = U_INVALID_FORMAT_ERROR;
	386	return;
	387	}
57a6839d A	388	} else if(data == NULL) {
	389	// Nothing to do.
	390	} else if(baseData != NULL) {
b331163b A	391	data->numScripts = baseData->numScripts;
	392	data->scriptsIndex = baseData->scriptsIndex;
	393	data->scriptStarts = baseData->scriptStarts;
	394	data->scriptStartsLength = baseData->scriptStartsLength;
57a6839d A	395	}
	396
	397	index = IX_COMPRESSIBLE_BYTES_OFFSET;
	398	offset = getIndex(inIndexes, indexesLength, index);
	399	length = getIndex(inIndexes, indexesLength, index + 1) - offset;
	400	if(length >= 256) {
	401	if(data == NULL) {
	402	errorCode = U_INVALID_FORMAT_ERROR;
	403	return;
	404	}
	405	data->compressibleBytes = reinterpret_cast<const UBool *>(inBytes + offset);
	406	} else if(data == NULL) {
	407	// Nothing to do.
	408	} else if(baseData != NULL) {
	409	data->compressibleBytes = baseData->compressibleBytes;
	410	} else {
	411	errorCode = U_INVALID_FORMAT_ERROR; // No compressibleBytes[].
	412	return;
	413	}
	414
	415	const CollationSettings &ts = *tailoring.settings;
	416	int32_t options = inIndexes[IX_OPTIONS] & 0xffff;
	417	uint16_t fastLatinPrimaries[CollationFastLatin::LATIN_LIMIT];
	418	int32_t fastLatinOptions = CollationFastLatin::getOptions(
b331163b	419	tailoring.data, ts, fastLatinPrimaries, UPRV_LENGTHOF(fastLatinPrimaries));
57a6839d A	420	if(options == ts.options && ts.variableTop != 0 &&
57a6839d A	421	reorderCodesLength == ts.reorderCodesLength &&
0f5d89e8 A	422	(reorderCodesLength == 0 \|\|
0f5d89e8 A	423	uprv_memcmp(reorderCodes, ts.reorderCodes, reorderCodesLength * 4) == 0) &&
57a6839d A	424	fastLatinOptions == ts.fastLatinOptions &&
	425	(fastLatinOptions < 0 \|\|
	426	uprv_memcmp(fastLatinPrimaries, ts.fastLatinPrimaries,
	427	sizeof(fastLatinPrimaries)) == 0)) {
	428	return;
	429	}
	430
	431	CollationSettings *settings = SharedObject::copyOnWrite(tailoring.settings);
	432	if(settings == NULL) {
	433	errorCode = U_MEMORY_ALLOCATION_ERROR;
	434	return;
	435	}
	436	settings->options = options;
	437	// Set variableTop from options and scripts data.
	438	settings->variableTop = tailoring.data->getLastPrimaryForGroup(
	439	UCOL_REORDER_CODE_FIRST + settings->getMaxVariable());
	440	if(settings->variableTop == 0) {
	441	errorCode = U_INVALID_FORMAT_ERROR;
	442	return;
	443	}
	444
b331163b A	445	if(reorderCodesLength != 0) {
	446	settings->aliasReordering(*baseData, reorderCodes, reorderCodesLength,
	447	reorderRanges, reorderRangesLength,
	448	reorderTable, errorCode);
57a6839d A	449	}
	450
	451	settings->fastLatinOptions = CollationFastLatin::getOptions(
	452	tailoring.data, *settings,
b331163b	453	settings->fastLatinPrimaries, UPRV_LENGTHOF(settings->fastLatinPrimaries));
57a6839d A	454	}
	455
	456	UBool U_CALLCONV
	457	CollationDataReader::isAcceptable(void *context,
	458	const char * /* type /, const char /name/,
	459	const UDataInfo *pInfo) {
	460	if(
	461	pInfo->size >= 20 &&
	462	pInfo->isBigEndian == U_IS_BIG_ENDIAN &&
	463	pInfo->charsetFamily == U_CHARSET_FAMILY &&
	464	pInfo->dataFormat[0] == 0x55 && // dataFormat="UCol"
	465	pInfo->dataFormat[1] == 0x43 &&
	466	pInfo->dataFormat[2] == 0x6f &&
	467	pInfo->dataFormat[3] == 0x6c &&
b331163b	468	pInfo->formatVersion[0] == 5
57a6839d A	469	) {
	470	UVersionInfo version = static_cast<UVersionInfo >(context);
	471	if(version != NULL) {
	472	uprv_memcpy(version, pInfo->dataVersion, 4);
	473	}
	474	return TRUE;
	475	} else {
	476	return FALSE;
	477	}
	478	}
	479
	480	U_NAMESPACE_END
	481
	482	#endif // !UCONFIG_NO_COLLATION