icuSources/i18n/collationdatareader.h

   1 /*
   2 *******************************************************************************
   3 * Copyright (C) 2013-2014, International Business Machines
   4 * Corporation and others.  All Rights Reserved.
   5 *******************************************************************************
   6 * collationdatareader.h
   7 *
   8 * created on: 2013feb07
   9 * created by: Markus W. Scherer
  10 */
  11
  12 #ifndef __COLLATIONDATAREADER_H__
  13 #define __COLLATIONDATAREADER_H__
  14
  15 #include "unicode/utypes.h"
  16
  17 #if !UCONFIG_NO_COLLATION
  18
  19 #include "unicode/udata.h"
  20
  21 struct UDataMemory;
  22
  23 U_NAMESPACE_BEGIN
  24
  25 struct CollationTailoring;
  26
  27 /**
  28  * Collation binary data reader.
  29  */
  30 struct U_I18N_API CollationDataReader /* all static */ {
  31     // The following constants are also copied into source/common/ucol_swp.cpp.
  32     // Keep them in sync!
  33     enum {
  34         /**
  35          * Number of int32_t indexes.
  36          *
  37          * Can be 2 if there are only options.
  38          * Can be 7 or 8 if there are only options and a script reordering.
  39          * The loader treats any index>=indexes[IX_INDEXES_LENGTH] as 0.
  40          */
  41         IX_INDEXES_LENGTH,  // 0
  42         /**
  43          * Bits 31..24: numericPrimary, for numeric collation
  44          *      23..16: fast Latin format version (0 = no fast Latin table)
  45          *      15.. 0: options bit set
  46          */
  47         IX_OPTIONS,
  48         IX_RESERVED2,
  49         IX_RESERVED3,
  50
  51         /** Array offset to Jamo CE32s in ce32s[], or <0 if none. */
  52         IX_JAMO_CE32S_START,  // 4
  53
  54         // Byte offsets from the start of the data, after the generic header.
  55         // The indexes[] are at byte offset 0, other data follows.
  56         // Each data item is aligned properly.
  57         // The data items should be in descending order of unit size,
  58         // to minimize the need for padding.
  59         // Each item's byte length is given by the difference between its offset and
  60         // the next index/offset value.
  61         /** Byte offset to int32_t reorderCodes[]. */
  62         IX_REORDER_CODES_OFFSET,
  63         /**
  64          * Byte offset to uint8_t reorderTable[].
  65          * Empty table if <256 bytes (padding only).
  66          * Otherwise 256 bytes or more (with padding).
  67          */
  68         IX_REORDER_TABLE_OFFSET,
  69         /** Byte offset to the collation trie. Its length is a multiple of 8 bytes. */
  70         IX_TRIE_OFFSET,
  71
  72         IX_RESERVED8_OFFSET,  // 8
  73         /** Byte offset to int64_t ces[]. */
  74         IX_CES_OFFSET,
  75         IX_RESERVED10_OFFSET,
  76         /** Byte offset to uint32_t ce32s[]. */
  77         IX_CE32S_OFFSET,
  78
  79         /** Byte offset to uint32_t rootElements[]. */
  80         IX_ROOT_ELEMENTS_OFFSET,  // 12
  81         /** Byte offset to UChar *contexts[]. */
  82         IX_CONTEXTS_OFFSET,
  83         /** Byte offset to uint16_t [] with serialized unsafeBackwardSet. */
  84         IX_UNSAFE_BWD_OFFSET,
  85         /** Byte offset to uint16_t fastLatinTable[]. */
  86         IX_FAST_LATIN_TABLE_OFFSET,
  87
  88         /** Byte offset to uint16_t scripts[]. */
  89         IX_SCRIPTS_OFFSET,  // 16
  90         /**
  91          * Byte offset to UBool compressibleBytes[].
  92          * Empty table if <256 bytes (padding only).
  93          * Otherwise 256 bytes or more (with padding).
  94          */
  95         IX_COMPRESSIBLE_BYTES_OFFSET,
  96         IX_RESERVED18_OFFSET,
  97         IX_TOTAL_SIZE
  98     };
  99
 100     static void read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength,
 101                      CollationTailoring &tailoring, UErrorCode &errorCode);
 102
 103     static UBool U_CALLCONV
 104     isAcceptable(void *context, const char *type, const char *name, const UDataInfo *pInfo);
 105
 106 private:
 107     CollationDataReader();  // no constructor
 108 };
 109
 110 /*
 111  * Format of collation data (ucadata.icu, binary data in coll/ *.res files).
 112  * Format version 4.0.
 113  *
 114  * The root collation data is stored in the ucadata.icu file.
 115  * Tailorings are stored inside .res resource bundle files, with a complete file header.
 116  *
 117  * Collation data begins with a standard ICU data file header
 118  * (DataHeader, see ucmndata.h and unicode/udata.h).
 119  * The UDataInfo.dataVersion field contains the UCA and other version numbers,
 120  * see the comments for CollationTailoring.version.
 121  *
 122  * After the header, the file contains the following parts.
 123  * Constants are defined as enum values of the CollationDataReader class.
 124  * See also the Collation class.
 125  *
 126  * int32_t indexes[indexesLength];
 127  *      The indexes array has variable length.
 128  *      Some tailorings only need the length and the options,
 129  *      others only add reorderCodes and the reorderTable,
 130  *      some need to store mappings.
 131  *      Only as many indexes are stored as needed to read all of the data.
 132  *
 133  *      Index 0: indexesLength
 134  *      Index 1: numericPrimary, CollationFastLatin::VERSION, and options: see IX_OPTIONS
 135  *      Index 2..3: Unused/reserved/0.
 136  *      Index 4: Index into the ce32s array where the CE32s of the conjoining Jamo
 137  *               are stored in a short, contiguous part of the ce32s array.
 138  *
 139  *      Indexes 5..19 are byte offsets in ascending order.
 140  *      Each byte offset marks the start of the next part in the data file,
 141  *      and the end of the previous one.
 142  *      When two consecutive byte offsets are the same (or too short),
 143  *      then the corresponding part is empty.
 144  *      Byte offsets are offsets from after the header,
 145  *      that is, from the beginning of the indexes[].
 146  *      Each part starts at an offset with proper alignment for its data.
 147  *      If necessary, the previous part may include padding bytes to achieve this alignment.
 148  *      The last byte offset that is stored in the indexes indicates the total size of the data
 149  *      (starting with the indexes).
 150  *
 151  * int32_t reorderCodes[]; -- empty in root
 152  *      The list of script and reordering codes.
 153  *
 154  * uint8_t reorderTable[256]; -- empty in root; can be longer to include padding bytes
 155  *      Primary-weight lead byte permutation table.
 156  *      Normally present when the reorderCodes are, but can be built at load time.
 157  *
 158  * UTrie2 trie; -- see utrie2_impl.h and utrie2.h
 159  *      The trie holds the main collation data. Each code point is mapped to a 32-bit value.
 160  *      It encodes a simple collation element (CE) in compact form, unless bits 7..6 are both set,
 161  *      in which case it is a special CE32 and contains a 4-bit tag and further data.
 162  *      See the Collation class for details.
 163  *
 164  *      The trie has a value for each lead surrogate code unit with some bits encoding
 165  *      collective properties of the 1024 supplementary characters whose UTF-16 form starts with
 166  *      the lead surrogate. See Collation::LEAD_SURROGATE_TAG..
 167  *
 168  * int64_t ces[];
 169  *      64-bit CEs and expansions that cannot be stored in a more compact form.
 170  *
 171  * uint32_t ce32s[];
 172  *      CE32s for expansions in compact form, and for characters whose trie values
 173  *      contain special data.
 174  *
 175  * uint32_t rootElements[]; -- empty in all tailorings
 176  *      Compact storage for all of the CEs that occur in the root collation.
 177  *      See the CollationRootElements class.
 178  *
 179  * UChar *contexts[];
 180  *      Serialized UCharsTrie structures with prefix (pre-context) and contraction mappings.
 181  *
 182  * uint16_t unsafeBackwardSet[]; -- see UnicodeSet::serialize()
 183  *      Serialized form of characters that are unsafe when iterating backwards,
 184  *      and at the end of an identical string prefix.
 185  *      Back up to a safe character.
 186  *      Lead surrogates are "unsafe" when any of their corresponding supplementary
 187  *      code points are unsafe.
 188  *      Does not include [:^lccc=0:][:^tccc=0:].
 189  *      For each tailoring, the root unsafeBackwardSet is subtracted.
 190  *      (As a result, in many tailorings no set needs to be stored.)
 191  *
 192  * uint16_t fastLatinTable[];
 193  *      Optional optimization for Latin text.
 194  *      See the CollationFastLatin class.
 195  *
 196  * uint16_t scripts[]; -- empty in all tailorings
 197  *      Table of the reordering groups with their first and last lead bytes,
 198  *      and their script and reordering codes.
 199  *      See CollationData::scripts.
 200  *
 201  * UBool compressibleBytes[]; -- empty in all tailorings
 202  *      Flag for getSortKey(), indicating primary weight lead bytes that are compressible.
 203  */
 204
 205 U_NAMESPACE_END
 206
 207 #endif  // !UCONFIG_NO_COLLATION
 208 #endif  // __COLLATIONDATAREADER_H__