[apple/icu.git] / icuSources / i18n / collationdatawriter.cpp

// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2013-2015, International Business Machines
* Corporation and others.  All Rights Reserved.
*******************************************************************************
* collationdatawriter.cpp
*
* created on: 2013aug06
* created by: Markus W. Scherer
*/

#include "unicode/utypes.h"

#if !UCONFIG_NO_COLLATION

#include "unicode/tblcoll.h"
#include "unicode/udata.h"
#include "unicode/uniset.h"
#include "cmemory.h"
#include "collationdata.h"
#include "collationdatabuilder.h"
#include "collationdatareader.h"
#include "collationdatawriter.h"
#include "collationfastlatin.h"
#include "collationsettings.h"
#include "collationtailoring.h"
#include "uassert.h"
#include "ucmndata.h"

U_NAMESPACE_BEGIN

uint8_t *
RuleBasedCollator::cloneRuleData(int32_t &length, UErrorCode &errorCode) const {
    if(U_FAILURE(errorCode)) { return NULL; }
    LocalMemory<uint8_t> buffer((uint8_t *)uprv_malloc(20000));
    if(buffer.isNull()) {
        errorCode = U_MEMORY_ALLOCATION_ERROR;
        return NULL;
    }
    length = cloneBinary(buffer.getAlias(), 20000, errorCode);
    if(errorCode == U_BUFFER_OVERFLOW_ERROR) {
        if(buffer.allocateInsteadAndCopy(length, 0) == NULL) {
            errorCode = U_MEMORY_ALLOCATION_ERROR;
            return NULL;
        }
        errorCode = U_ZERO_ERROR;
        length = cloneBinary(buffer.getAlias(), length, errorCode);
    }
    if(U_FAILURE(errorCode)) { return NULL; }
    return buffer.orphan();
}

int32_t
RuleBasedCollator::cloneBinary(uint8_t *dest, int32_t capacity, UErrorCode &errorCode) const {
    int32_t indexes[CollationDataReader::IX_TOTAL_SIZE + 1];
    return CollationDataWriter::writeTailoring(
            *tailoring, *settings, indexes, dest, capacity,
            errorCode);
}

static const UDataInfo dataInfo = {
    sizeof(UDataInfo),
    0,

    U_IS_BIG_ENDIAN,
    U_CHARSET_FAMILY,
    U_SIZEOF_UCHAR,
    0,

    { 0x55, 0x43, 0x6f, 0x6c },         // dataFormat="UCol"
    { 5, 0, 0, 0 },                     // formatVersion
    { 6, 3, 0, 0 }                      // dataVersion
};

int32_t
CollationDataWriter::writeBase(const CollationData &data, const CollationSettings &settings,
                               const void *rootElements, int32_t rootElementsLength,
                               int32_t indexes[], uint8_t *dest, int32_t capacity,
                               UErrorCode &errorCode) {
    return write(TRUE, NULL,
                 data, settings,
                 rootElements, rootElementsLength,
                 indexes, dest, capacity, errorCode);
}

int32_t
CollationDataWriter::writeTailoring(const CollationTailoring &t, const CollationSettings &settings,
                                    int32_t indexes[], uint8_t *dest, int32_t capacity,
                                    UErrorCode &errorCode) {
    return write(FALSE, t.version,
                 *t.data, settings,
                 NULL, 0,
                 indexes, dest, capacity, errorCode);
}

int32_t
CollationDataWriter::write(UBool isBase, const UVersionInfo dataVersion,
                           const CollationData &data, const CollationSettings &settings,
                           const void *rootElements, int32_t rootElementsLength,
                           int32_t indexes[], uint8_t *dest, int32_t capacity,
                           UErrorCode &errorCode) {
    if(U_FAILURE(errorCode)) { return 0; }
    if(capacity < 0 || (capacity > 0 && dest == NULL)) {
        errorCode = U_ILLEGAL_ARGUMENT_ERROR;
        return 0;
    }

    // Figure out which data items to write before settling on
    // the indexes length and writing offsets.
    // For any data item, we need to write the start and limit offsets,
    // so the indexes length must be at least index-of-start-offset + 2.
    int32_t indexesLength;
    UBool hasMappings;
    UnicodeSet unsafeBackwardSet;
    const CollationData *baseData = data.base;

    int32_t fastLatinVersion;
    if(data.fastLatinTable != NULL) {
        fastLatinVersion = (int32_t)CollationFastLatin::VERSION << 16;
    } else {
        fastLatinVersion = 0;
    }
    int32_t fastLatinTableLength = 0;

    if(isBase) {
        // For the root collator, we write an even number of indexes
        // so that we start with an 8-aligned offset.
        indexesLength = CollationDataReader::IX_TOTAL_SIZE + 1;
        U_ASSERT(settings.reorderCodesLength == 0);
        hasMappings = TRUE;
        unsafeBackwardSet = *data.unsafeBackwardSet;
        fastLatinTableLength = data.fastLatinTableLength;
    } else if(baseData == NULL) {
        hasMappings = FALSE;
        if(settings.reorderCodesLength == 0) {
            // only options
            indexesLength = CollationDataReader::IX_OPTIONS + 1;  // no limit offset here
        } else {
            // only options, reorder codes, and the reorder table
            indexesLength = CollationDataReader::IX_REORDER_TABLE_OFFSET + 2;
        }
    } else {
        hasMappings = TRUE;
        // Tailored mappings, and what else?
        // Check in ascending order of optional tailoring data items.
        indexesLength = CollationDataReader::IX_CE32S_OFFSET + 2;
        if(data.contextsLength != 0) {
            indexesLength = CollationDataReader::IX_CONTEXTS_OFFSET + 2;
        }
        unsafeBackwardSet.addAll(*data.unsafeBackwardSet).removeAll(*baseData->unsafeBackwardSet);
        if(!unsafeBackwardSet.isEmpty()) {
            indexesLength = CollationDataReader::IX_UNSAFE_BWD_OFFSET + 2;
        }
        if(data.fastLatinTable != baseData->fastLatinTable) {
            fastLatinTableLength = data.fastLatinTableLength;
            indexesLength = CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET + 2;
        }
    }

    UVector32 codesAndRanges(errorCode);
    const int32_t *reorderCodes = settings.reorderCodes;
    int32_t reorderCodesLength = settings.reorderCodesLength;
    if(settings.hasReordering() &&
            CollationSettings::reorderTableHasSplitBytes(settings.reorderTable)) {
        // Rebuild the full list of reorder ranges.
        // The list in the settings is truncated for efficiency.
        data.makeReorderRanges(reorderCodes, reorderCodesLength, codesAndRanges, errorCode);
        // Write the codes, then the ranges.
        for(int32_t i = 0; i < reorderCodesLength; ++i) {
            codesAndRanges.insertElementAt(reorderCodes[i], i, errorCode);
        }
        if(U_FAILURE(errorCode)) { return 0; }
        reorderCodes = codesAndRanges.getBuffer();
        reorderCodesLength = codesAndRanges.size();
    }

    int32_t headerSize;
    if(isBase) {
        headerSize = 0;  // udata_create() writes the header
    } else {
        DataHeader header;
        header.dataHeader.magic1 = 0xda;
        header.dataHeader.magic2 = 0x27;
        uprv_memcpy(&header.info, &dataInfo, sizeof(UDataInfo));
        uprv_memcpy(header.info.dataVersion, dataVersion, sizeof(UVersionInfo));
        headerSize = (int32_t)sizeof(header);
        U_ASSERT((headerSize & 3) == 0);  // multiple of 4 bytes
        if(hasMappings && data.cesLength != 0) {
            // Sum of the sizes of the data items which are
            // not automatically multiples of 8 bytes and which are placed before the CEs.
            int32_t sum = headerSize + (indexesLength + reorderCodesLength) * 4;
            if((sum & 7) != 0) {
                // We need to add padding somewhere so that the 64-bit CEs are 8-aligned.
                // We add to the header size here.
                // Alternatively, we could increment the indexesLength
                // or add a few bytes to the reorderTable.
                headerSize += 4;
            }
        }
        header.dataHeader.headerSize = (uint16_t)headerSize;
        if(headerSize <= capacity) {
            uprv_memcpy(dest, &header, sizeof(header));
            // Write 00 bytes so that the padding is not mistaken for a copyright string.
            uprv_memset(dest + sizeof(header), 0, headerSize - (int32_t)sizeof(header));
            dest += headerSize;
            capacity -= headerSize;
        } else {
            dest = NULL;
            capacity = 0;
        }
    }

    indexes[CollationDataReader::IX_INDEXES_LENGTH] = indexesLength;
    U_ASSERT((settings.options & ~0xffff) == 0);
    indexes[CollationDataReader::IX_OPTIONS] =
            data.numericPrimary | fastLatinVersion | settings.options;
    indexes[CollationDataReader::IX_RESERVED2] = 0;
    indexes[CollationDataReader::IX_RESERVED3] = 0;

    // Byte offsets of data items all start from the start of the indexes.
    // We add the headerSize at the very end.
    int32_t totalSize = indexesLength * 4;

    if(hasMappings && (isBase || data.jamoCE32s != baseData->jamoCE32s)) {
        indexes[CollationDataReader::IX_JAMO_CE32S_START] = data.jamoCE32s - data.ce32s;
    } else {
        indexes[CollationDataReader::IX_JAMO_CE32S_START] = -1;
    }

    indexes[CollationDataReader::IX_REORDER_CODES_OFFSET] = totalSize;
    totalSize += reorderCodesLength * 4;

    indexes[CollationDataReader::IX_REORDER_TABLE_OFFSET] = totalSize;
    if(settings.reorderTable != NULL) {
        totalSize += 256;
    }

    indexes[CollationDataReader::IX_TRIE_OFFSET] = totalSize;
    if(hasMappings) {
        UErrorCode errorCode2 = U_ZERO_ERROR;
        int32_t length;
        if(totalSize < capacity) {
            length = utrie2_serialize(data.trie, dest + totalSize,
                                      capacity - totalSize, &errorCode2);
        } else {
            length = utrie2_serialize(data.trie, NULL, 0, &errorCode2);
        }
        if(U_FAILURE(errorCode2) && errorCode2 != U_BUFFER_OVERFLOW_ERROR) {
            errorCode = errorCode2;
            return 0;
        }
        // The trie size should be a multiple of 8 bytes due to the way
        // compactIndex2(UNewTrie2 *trie) currently works.
        U_ASSERT((length & 7) == 0);
        totalSize += length;
    }

    indexes[CollationDataReader::IX_RESERVED8_OFFSET] = totalSize;
    indexes[CollationDataReader::IX_CES_OFFSET] = totalSize;
    if(hasMappings && data.cesLength != 0) {
        U_ASSERT(((headerSize + totalSize) & 7) == 0);
        totalSize += data.cesLength * 8;
    }

    indexes[CollationDataReader::IX_RESERVED10_OFFSET] = totalSize;
    indexes[CollationDataReader::IX_CE32S_OFFSET] = totalSize;
    if(hasMappings) {
        totalSize += data.ce32sLength * 4;
    }

    indexes[CollationDataReader::IX_ROOT_ELEMENTS_OFFSET] = totalSize;
    totalSize += rootElementsLength * 4;

    indexes[CollationDataReader::IX_CONTEXTS_OFFSET] = totalSize;
    if(hasMappings) {
        totalSize += data.contextsLength * 2;
    }

    indexes[CollationDataReader::IX_UNSAFE_BWD_OFFSET] = totalSize;
    if(hasMappings && !unsafeBackwardSet.isEmpty()) {
        UErrorCode errorCode2 = U_ZERO_ERROR;
        int32_t length;
        if(totalSize < capacity) {
            uint16_t *p = reinterpret_cast<uint16_t *>(dest + totalSize);
            length = unsafeBackwardSet.serialize(
                    p, (capacity - totalSize) / 2, errorCode2);
        } else {
            length = unsafeBackwardSet.serialize(NULL, 0, errorCode2);
        }
        if(U_FAILURE(errorCode2) && errorCode2 != U_BUFFER_OVERFLOW_ERROR) {
            errorCode = errorCode2;
            return 0;
        }
        totalSize += length * 2;
    }

    indexes[CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET] = totalSize;
    totalSize += fastLatinTableLength * 2;

    UnicodeString scripts;
    indexes[CollationDataReader::IX_SCRIPTS_OFFSET] = totalSize;
    if(isBase) {
        scripts.append((UChar)data.numScripts);
        scripts.append(reinterpret_cast<const UChar *>(data.scriptsIndex), data.numScripts + 16);
        scripts.append(reinterpret_cast<const UChar *>(data.scriptStarts), data.scriptStartsLength);
        totalSize += scripts.length() * 2;
    }

    indexes[CollationDataReader::IX_COMPRESSIBLE_BYTES_OFFSET] = totalSize;
    if(isBase) {
        totalSize += 256;
    }

    indexes[CollationDataReader::IX_RESERVED18_OFFSET] = totalSize;
    indexes[CollationDataReader::IX_TOTAL_SIZE] = totalSize;

    if(totalSize > capacity) {
        errorCode = U_BUFFER_OVERFLOW_ERROR;
        return headerSize + totalSize;
    }

    uprv_memcpy(dest, indexes, indexesLength * 4);
    copyData(indexes, CollationDataReader::IX_REORDER_CODES_OFFSET, reorderCodes, dest);
    copyData(indexes, CollationDataReader::IX_REORDER_TABLE_OFFSET, settings.reorderTable, dest);
    // The trie has already been serialized into the dest buffer.
    copyData(indexes, CollationDataReader::IX_CES_OFFSET, data.ces, dest);
    copyData(indexes, CollationDataReader::IX_CE32S_OFFSET, data.ce32s, dest);
    copyData(indexes, CollationDataReader::IX_ROOT_ELEMENTS_OFFSET, rootElements, dest);
    copyData(indexes, CollationDataReader::IX_CONTEXTS_OFFSET, data.contexts, dest);
    // The unsafeBackwardSet has already been serialized into the dest buffer.
    copyData(indexes, CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET, data.fastLatinTable, dest);
    copyData(indexes, CollationDataReader::IX_SCRIPTS_OFFSET, scripts.getBuffer(), dest);
    copyData(indexes, CollationDataReader::IX_COMPRESSIBLE_BYTES_OFFSET, data.compressibleBytes, dest);

    return headerSize + totalSize;
}

void
CollationDataWriter::copyData(const int32_t indexes[], int32_t startIndex,
                              const void *src, uint8_t *dest) {
    int32_t start = indexes[startIndex];
    int32_t limit = indexes[startIndex + 1];
    if(start < limit) {
        uprv_memcpy(dest + start, src, limit - start);
    }
}

U_NAMESPACE_END

#endif  // !UCONFIG_NO_COLLATION
Commit	Line	Data
f3c0d7a5 A	1	// © 2016 and later: Unicode, Inc. and others.
f3c0d7a5 A	2	// License & terms of use: http://www.unicode.org/copyright.html
57a6839d A	3	/*
57a6839d A	4	*******************************************************************************
b331163b	5	* Copyright (C) 2013-2015, International Business Machines
57a6839d A	6	* Corporation and others. All Rights Reserved.
	7	*******************************************************************************
	8	* collationdatawriter.cpp
	9	*
	10	* created on: 2013aug06
	11	* created by: Markus W. Scherer
	12	*/
	13
	14	#include "unicode/utypes.h"
	15
	16	#if !UCONFIG_NO_COLLATION
	17
	18	#include "unicode/tblcoll.h"
	19	#include "unicode/udata.h"
	20	#include "unicode/uniset.h"
	21	#include "cmemory.h"
	22	#include "collationdata.h"
	23	#include "collationdatabuilder.h"
	24	#include "collationdatareader.h"
	25	#include "collationdatawriter.h"
	26	#include "collationfastlatin.h"
	27	#include "collationsettings.h"
	28	#include "collationtailoring.h"
	29	#include "uassert.h"
	30	#include "ucmndata.h"
	31
	32	U_NAMESPACE_BEGIN
	33
	34	uint8_t *
	35	RuleBasedCollator::cloneRuleData(int32_t &length, UErrorCode &errorCode) const {
	36	if(U_FAILURE(errorCode)) { return NULL; }
	37	LocalMemory<uint8_t> buffer((uint8_t *)uprv_malloc(20000));
	38	if(buffer.isNull()) {
	39	errorCode = U_MEMORY_ALLOCATION_ERROR;
	40	return NULL;
	41	}
	42	length = cloneBinary(buffer.getAlias(), 20000, errorCode);
	43	if(errorCode == U_BUFFER_OVERFLOW_ERROR) {
	44	if(buffer.allocateInsteadAndCopy(length, 0) == NULL) {
	45	errorCode = U_MEMORY_ALLOCATION_ERROR;
	46	return NULL;
	47	}
	48	errorCode = U_ZERO_ERROR;
	49	length = cloneBinary(buffer.getAlias(), length, errorCode);
	50	}
	51	if(U_FAILURE(errorCode)) { return NULL; }
	52	return buffer.orphan();
	53	}
	54
	55	int32_t
	56	RuleBasedCollator::cloneBinary(uint8_t *dest, int32_t capacity, UErrorCode &errorCode) const {
	57	int32_t indexes[CollationDataReader::IX_TOTAL_SIZE + 1];
	58	return CollationDataWriter::writeTailoring(
	59	tailoring, settings, indexes, dest, capacity,
	60	errorCode);
	61	}
	62
	63	static const UDataInfo dataInfo = {
	64	sizeof(UDataInfo),
	65	0,
	66
	67	U_IS_BIG_ENDIAN,
	68	U_CHARSET_FAMILY,
	69	U_SIZEOF_UCHAR,
70	0,
71
72	{ 0x55, 0x43, 0x6f, 0x6c }, // dataFormat="UCol"
b331163b	73	{ 5, 0, 0, 0 }, // formatVersion
57a6839d A	74	{ 6, 3, 0, 0 } // dataVersion
	75	};
	76
	77	int32_t
	78	CollationDataWriter::writeBase(const CollationData &data, const CollationSettings &settings,
	79	const void *rootElements, int32_t rootElementsLength,
	80	int32_t indexes[], uint8_t *dest, int32_t capacity,
	81	UErrorCode &errorCode) {
	82	return write(TRUE, NULL,
	83	data, settings,
	84	rootElements, rootElementsLength,
	85	indexes, dest, capacity, errorCode);
	86	}
	87
	88	int32_t
	89	CollationDataWriter::writeTailoring(const CollationTailoring &t, const CollationSettings &settings,
	90	int32_t indexes[], uint8_t *dest, int32_t capacity,
	91	UErrorCode &errorCode) {
	92	return write(FALSE, t.version,
	93	*t.data, settings,
	94	NULL, 0,
	95	indexes, dest, capacity, errorCode);
	96	}
	97
	98	int32_t
	99	CollationDataWriter::write(UBool isBase, const UVersionInfo dataVersion,
	100	const CollationData &data, const CollationSettings &settings,
	101	const void *rootElements, int32_t rootElementsLength,
	102	int32_t indexes[], uint8_t *dest, int32_t capacity,
	103	UErrorCode &errorCode) {
	104	if(U_FAILURE(errorCode)) { return 0; }
	105	if(capacity < 0 \|\| (capacity > 0 && dest == NULL)) {
	106	errorCode = U_ILLEGAL_ARGUMENT_ERROR;
	107	return 0;
	108	}
	109
	110	// Figure out which data items to write before settling on
	111	// the indexes length and writing offsets.
	112	// For any data item, we need to write the start and limit offsets,
	113	// so the indexes length must be at least index-of-start-offset + 2.
	114	int32_t indexesLength;
	115	UBool hasMappings;
	116	UnicodeSet unsafeBackwardSet;
	117	const CollationData *baseData = data.base;
	118
	119	int32_t fastLatinVersion;
	120	if(data.fastLatinTable != NULL) {
	121	fastLatinVersion = (int32_t)CollationFastLatin::VERSION << 16;
	122	} else {
	123	fastLatinVersion = 0;
	124	}
	125	int32_t fastLatinTableLength = 0;
	126
	127	if(isBase) {
	128	// For the root collator, we write an even number of indexes
	129	// so that we start with an 8-aligned offset.
	130	indexesLength = CollationDataReader::IX_TOTAL_SIZE + 1;
	131	U_ASSERT(settings.reorderCodesLength == 0);
	132	hasMappings = TRUE;
	133	unsafeBackwardSet = *data.unsafeBackwardSet;
	134	fastLatinTableLength = data.fastLatinTableLength;
	135	} else if(baseData == NULL) {
	136	hasMappings = FALSE;
	137	if(settings.reorderCodesLength == 0) {
138	// only options
139	indexesLength = CollationDataReader::IX_OPTIONS + 1; // no limit offset here
140	} else {
141	// only options, reorder codes, and the reorder table
142	indexesLength = CollationDataReader::IX_REORDER_TABLE_OFFSET + 2;
143	}
144	} else {
145	hasMappings = TRUE;
146	// Tailored mappings, and what else?
147	// Check in ascending order of optional tailoring data items.
148	indexesLength = CollationDataReader::IX_CE32S_OFFSET + 2;
149	if(data.contextsLength != 0) {
150	indexesLength = CollationDataReader::IX_CONTEXTS_OFFSET + 2;
151	}
152	unsafeBackwardSet.addAll(data.unsafeBackwardSet).removeAll(baseData->unsafeBackwardSet);
153	if(!unsafeBackwardSet.isEmpty()) {
154	indexesLength = CollationDataReader::IX_UNSAFE_BWD_OFFSET + 2;
155	}
156	if(data.fastLatinTable != baseData->fastLatinTable) {
157	fastLatinTableLength = data.fastLatinTableLength;
158	indexesLength = CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET + 2;
159	}
160	}
161
b331163b A	162	UVector32 codesAndRanges(errorCode);
	163	const int32_t *reorderCodes = settings.reorderCodes;
	164	int32_t reorderCodesLength = settings.reorderCodesLength;
	165	if(settings.hasReordering() &&
	166	CollationSettings::reorderTableHasSplitBytes(settings.reorderTable)) {
	167	// Rebuild the full list of reorder ranges.
	168	// The list in the settings is truncated for efficiency.
	169	data.makeReorderRanges(reorderCodes, reorderCodesLength, codesAndRanges, errorCode);
	170	// Write the codes, then the ranges.
	171	for(int32_t i = 0; i < reorderCodesLength; ++i) {
	172	codesAndRanges.insertElementAt(reorderCodes[i], i, errorCode);
	173	}
	174	if(U_FAILURE(errorCode)) { return 0; }
	175	reorderCodes = codesAndRanges.getBuffer();
	176	reorderCodesLength = codesAndRanges.size();
	177	}
	178
57a6839d A	179	int32_t headerSize;
	180	if(isBase) {
	181	headerSize = 0; // udata_create() writes the header
	182	} else {
	183	DataHeader header;
	184	header.dataHeader.magic1 = 0xda;
	185	header.dataHeader.magic2 = 0x27;
	186	uprv_memcpy(&header.info, &dataInfo, sizeof(UDataInfo));
	187	uprv_memcpy(header.info.dataVersion, dataVersion, sizeof(UVersionInfo));
	188	headerSize = (int32_t)sizeof(header);
	189	U_ASSERT((headerSize & 3) == 0); // multiple of 4 bytes
	190	if(hasMappings && data.cesLength != 0) {
	191	// Sum of the sizes of the data items which are
	192	// not automatically multiples of 8 bytes and which are placed before the CEs.
b331163b	193	int32_t sum = headerSize + (indexesLength + reorderCodesLength) * 4;
57a6839d A	194	if((sum & 7) != 0) {
	195	// We need to add padding somewhere so that the 64-bit CEs are 8-aligned.
	196	// We add to the header size here.
	197	// Alternatively, we could increment the indexesLength
	198	// or add a few bytes to the reorderTable.
	199	headerSize += 4;
	200	}
	201	}
	202	header.dataHeader.headerSize = (uint16_t)headerSize;
	203	if(headerSize <= capacity) {
	204	uprv_memcpy(dest, &header, sizeof(header));
	205	// Write 00 bytes so that the padding is not mistaken for a copyright string.
	206	uprv_memset(dest + sizeof(header), 0, headerSize - (int32_t)sizeof(header));
	207	dest += headerSize;
	208	capacity -= headerSize;
	209	} else {
	210	dest = NULL;
	211	capacity = 0;
	212	}
	213	}
	214
	215	indexes[CollationDataReader::IX_INDEXES_LENGTH] = indexesLength;
	216	U_ASSERT((settings.options & ~0xffff) == 0);
	217	indexes[CollationDataReader::IX_OPTIONS] =
	218	data.numericPrimary \| fastLatinVersion \| settings.options;
	219	indexes[CollationDataReader::IX_RESERVED2] = 0;
	220	indexes[CollationDataReader::IX_RESERVED3] = 0;
	221
	222	// Byte offsets of data items all start from the start of the indexes.
	223	// We add the headerSize at the very end.
	224	int32_t totalSize = indexesLength * 4;
	225
	226	if(hasMappings && (isBase \|\| data.jamoCE32s != baseData->jamoCE32s)) {
	227	indexes[CollationDataReader::IX_JAMO_CE32S_START] = data.jamoCE32s - data.ce32s;
	228	} else {
	229	indexes[CollationDataReader::IX_JAMO_CE32S_START] = -1;
	230	}
	231
	232	indexes[CollationDataReader::IX_REORDER_CODES_OFFSET] = totalSize;
b331163b	233	totalSize += reorderCodesLength * 4;
57a6839d A	234
	235	indexes[CollationDataReader::IX_REORDER_TABLE_OFFSET] = totalSize;
	236	if(settings.reorderTable != NULL) {
	237	totalSize += 256;
	238	}
	239
	240	indexes[CollationDataReader::IX_TRIE_OFFSET] = totalSize;
	241	if(hasMappings) {
	242	UErrorCode errorCode2 = U_ZERO_ERROR;
	243	int32_t length;
	244	if(totalSize < capacity) {
	245	length = utrie2_serialize(data.trie, dest + totalSize,
	246	capacity - totalSize, &errorCode2);
	247	} else {
	248	length = utrie2_serialize(data.trie, NULL, 0, &errorCode2);
	249	}
	250	if(U_FAILURE(errorCode2) && errorCode2 != U_BUFFER_OVERFLOW_ERROR) {
	251	errorCode = errorCode2;
	252	return 0;
	253	}
	254	// The trie size should be a multiple of 8 bytes due to the way
	255	// compactIndex2(UNewTrie2 *trie) currently works.
	256	U_ASSERT((length & 7) == 0);
	257	totalSize += length;
	258	}
	259
	260	indexes[CollationDataReader::IX_RESERVED8_OFFSET] = totalSize;
	261	indexes[CollationDataReader::IX_CES_OFFSET] = totalSize;
	262	if(hasMappings && data.cesLength != 0) {
	263	U_ASSERT(((headerSize + totalSize) & 7) == 0);
	264	totalSize += data.cesLength * 8;
	265	}
	266
	267	indexes[CollationDataReader::IX_RESERVED10_OFFSET] = totalSize;
	268	indexes[CollationDataReader::IX_CE32S_OFFSET] = totalSize;
	269	if(hasMappings) {
	270	totalSize += data.ce32sLength * 4;
	271	}
	272
	273	indexes[CollationDataReader::IX_ROOT_ELEMENTS_OFFSET] = totalSize;
	274	totalSize += rootElementsLength * 4;
	275
	276	indexes[CollationDataReader::IX_CONTEXTS_OFFSET] = totalSize;
	277	if(hasMappings) {
	278	totalSize += data.contextsLength * 2;
	279	}
	280
	281	indexes[CollationDataReader::IX_UNSAFE_BWD_OFFSET] = totalSize;
	282	if(hasMappings && !unsafeBackwardSet.isEmpty()) {
	283	UErrorCode errorCode2 = U_ZERO_ERROR;
	284	int32_t length;
	285	if(totalSize < capacity) {
	286	uint16_t p = reinterpret_cast<uint16_t >(dest + totalSize);
	287	length = unsafeBackwardSet.serialize(
	288	p, (capacity - totalSize) / 2, errorCode2);
	289	} else {
	290	length = unsafeBackwardSet.serialize(NULL, 0, errorCode2);
	291	}
	292	if(U_FAILURE(errorCode2) && errorCode2 != U_BUFFER_OVERFLOW_ERROR) {
	293	errorCode = errorCode2;
	294	return 0;
	295	}
	296	totalSize += length * 2;
	297	}
298
299	indexes[CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET] = totalSize;
300	totalSize += fastLatinTableLength * 2;
301
b331163b	302	UnicodeString scripts;
57a6839d A	303	indexes[CollationDataReader::IX_SCRIPTS_OFFSET] = totalSize;
57a6839d A	304	if(isBase) {
b331163b A	305	scripts.append((UChar)data.numScripts);
	306	scripts.append(reinterpret_cast<const UChar *>(data.scriptsIndex), data.numScripts + 16);
	307	scripts.append(reinterpret_cast<const UChar *>(data.scriptStarts), data.scriptStartsLength);
	308	totalSize += scripts.length() * 2;
57a6839d A	309	}
	310
	311	indexes[CollationDataReader::IX_COMPRESSIBLE_BYTES_OFFSET] = totalSize;
	312	if(isBase) {
	313	totalSize += 256;
	314	}
	315
	316	indexes[CollationDataReader::IX_RESERVED18_OFFSET] = totalSize;
	317	indexes[CollationDataReader::IX_TOTAL_SIZE] = totalSize;
	318
	319	if(totalSize > capacity) {
	320	errorCode = U_BUFFER_OVERFLOW_ERROR;
	321	return headerSize + totalSize;
	322	}
	323
	324	uprv_memcpy(dest, indexes, indexesLength * 4);
b331163b	325	copyData(indexes, CollationDataReader::IX_REORDER_CODES_OFFSET, reorderCodes, dest);
57a6839d A	326	copyData(indexes, CollationDataReader::IX_REORDER_TABLE_OFFSET, settings.reorderTable, dest);
	327	// The trie has already been serialized into the dest buffer.
	328	copyData(indexes, CollationDataReader::IX_CES_OFFSET, data.ces, dest);
	329	copyData(indexes, CollationDataReader::IX_CE32S_OFFSET, data.ce32s, dest);
	330	copyData(indexes, CollationDataReader::IX_ROOT_ELEMENTS_OFFSET, rootElements, dest);
	331	copyData(indexes, CollationDataReader::IX_CONTEXTS_OFFSET, data.contexts, dest);
	332	// The unsafeBackwardSet has already been serialized into the dest buffer.
	333	copyData(indexes, CollationDataReader::IX_FAST_LATIN_TABLE_OFFSET, data.fastLatinTable, dest);
b331163b	334	copyData(indexes, CollationDataReader::IX_SCRIPTS_OFFSET, scripts.getBuffer(), dest);
57a6839d A	335	copyData(indexes, CollationDataReader::IX_COMPRESSIBLE_BYTES_OFFSET, data.compressibleBytes, dest);
	336
	337	return headerSize + totalSize;
	338	}
	339
	340	void
	341	CollationDataWriter::copyData(const int32_t indexes[], int32_t startIndex,
	342	const void src, uint8_t dest) {
	343	int32_t start = indexes[startIndex];
	344	int32_t limit = indexes[startIndex + 1];
	345	if(start < limit) {
	346	uprv_memcpy(dest + start, src, limit - start);
	347	}
	348	}
	349
	350	U_NAMESPACE_END
	351
	352	#endif // !UCONFIG_NO_COLLATION