icuSources/tools/toolutil/ucm.h

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 2003-2005, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  ucm.h
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2003jun20
  14 *   created by: Markus W. Scherer
  15 *
  16 *   Definitions for the .ucm file parser and handler module ucm.c.
  17 */
  18
  19 #ifndef __UCM_H__
  20 #define __UCM_H__
  21
  22 #include "unicode/utypes.h"
  23 #include "ucnvmbcs.h"
  24 #include "ucnv_ext.h"
  25 #include "filestrm.h"
  26 #include <stdio.h>
  27
  28 #if !UCONFIG_NO_CONVERSION
  29
  30 U_CDECL_BEGIN
  31
  32 /*
  33  * Per-mapping data structure
  34  *
  35  * u if uLen==1: Unicode code point
  36  *   else index to uLen code points
  37  * b if bLen<=4: up to 4 bytes
  38  *   else index to bLen bytes
  39  * uLen number of code points
  40  * bLen number of words containing left-justified bytes
  41  * bIsMultipleChars indicates that the bytes contain more than one sequence
  42  *                  according to the state table
  43  * f flag for roundtrip (0), fallback (1), sub mapping (2), reverse fallback (3)
  44  *   same values as in the source file after |
  45  */
  46 typedef struct UCMapping {
  47     UChar32 u;
  48     union {
  49         uint32_t index;
  50         uint8_t bytes[4];
  51     } b;
  52     int8_t uLen, bLen, f, moveFlag;
  53 } UCMapping;
  54
  55 enum {
  56     UCM_FLAGS_INITIAL,  /* no mappings parsed yet */
  57     UCM_FLAGS_EXPLICIT, /* .ucm file has mappings with | fallback indicators */
  58     UCM_FLAGS_IMPLICIT, /* .ucm file has mappings without | fallback indicators, later wins */
  59     UCM_FLAGS_MIXED     /* both implicit and explicit */
  60 };
  61
  62 typedef struct UCMTable {
  63     UCMapping *mappings;
  64     int32_t mappingsCapacity, mappingsLength;
  65
  66     UChar32 *codePoints;
  67     int32_t codePointsCapacity, codePointsLength;
  68
  69     uint8_t *bytes;
  70     int32_t bytesCapacity, bytesLength;
  71
  72     /* index map for mapping by bytes first */
  73     int32_t *reverseMap;
  74
  75     uint8_t unicodeMask;
  76     int8_t flagsType; /* UCM_FLAGS_INITIAL etc. */
  77     UBool isSorted;
  78 } UCMTable;
  79
  80 enum {
  81     MBCS_STATE_FLAG_DIRECT=1,
  82     MBCS_STATE_FLAG_SURROGATES,
  83
  84     MBCS_STATE_FLAG_READY=16
  85 };
  86
  87 typedef struct UCMStates {
  88     int32_t stateTable[MBCS_MAX_STATE_COUNT][256];
  89     uint32_t stateFlags[MBCS_MAX_STATE_COUNT],
  90              stateOffsetSum[MBCS_MAX_STATE_COUNT];
  91
  92     int32_t countStates, minCharLength, maxCharLength, countToUCodeUnits;
  93     int8_t conversionType, outputType;
  94 } UCMStates;
  95
  96 typedef struct UCMFile {
  97     UCMTable *base, *ext;
  98     UCMStates states;
  99
 100     char baseName[UCNV_MAX_CONVERTER_NAME_LENGTH];
 101 } UCMFile;
 102
 103 /* simple accesses ---------------------------------------------------------- */
 104
 105 #define UCM_GET_CODE_POINTS(t, m) \
 106     (((m)->uLen==1) ? &(m)->u : (t)->codePoints+(m)->u)
 107
 108 #define UCM_GET_BYTES(t, m) \
 109     (((m)->bLen<=4) ? (m)->b.bytes : (t)->bytes+(m)->b.index)
 110
 111 /* APIs --------------------------------------------------------------------- */
 112
 113 U_CAPI UCMFile * U_EXPORT2
 114 ucm_open(void);
 115
 116 U_CAPI void U_EXPORT2
 117 ucm_close(UCMFile *ucm);
 118
 119 U_CAPI UBool U_EXPORT2
 120 ucm_parseHeaderLine(UCMFile *ucm,
 121                     char *line, char **pKey, char **pValue);
 122
 123 /* @return -1 illegal bytes  0 suitable for base table  1 needs to go into extension table */
 124 U_CAPI int32_t U_EXPORT2
 125 ucm_mappingType(UCMStates *baseStates,
 126                 UCMapping *m,
 127                 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
 128                 uint8_t bytes[UCNV_EXT_MAX_BYTES]);
 129
 130 /* add a mapping to the base or extension table as appropriate */
 131 U_CAPI UBool U_EXPORT2
 132 ucm_addMappingAuto(UCMFile *ucm, UBool forBase, UCMStates *baseStates,
 133                    UCMapping *m,
 134                    UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
 135                    uint8_t bytes[UCNV_EXT_MAX_BYTES]);
 136
 137 U_CAPI UBool U_EXPORT2
 138 ucm_addMappingFromLine(UCMFile *ucm, const char *line, UBool forBase, UCMStates *baseStates);
 139
 140
 141 U_CAPI UCMTable * U_EXPORT2
 142 ucm_openTable(void);
 143
 144 U_CAPI void U_EXPORT2
 145 ucm_closeTable(UCMTable *table);
 146
 147 U_CAPI void U_EXPORT2
 148 ucm_resetTable(UCMTable *table);
 149
 150 U_CAPI void U_EXPORT2
 151 ucm_sortTable(UCMTable *t);
 152
 153 /**
 154  * Read a table from a .ucm file, from after the CHARMAP line to
 155  * including the END CHARMAP line.
 156  */
 157 U_CAPI void U_EXPORT2
 158 ucm_readTable(UCMFile *ucm, FileStream* convFile,
 159               UBool forBase, UCMStates *baseStates,
 160               UErrorCode *pErrorCode);
 161
 162 /**
 163  * Check the validity of mappings against a base table's states;
 164  * necessary for extension-only tables that were read before their base tables.
 165  */
 166 U_CAPI UBool U_EXPORT2
 167 ucm_checkValidity(UCMTable *ext, UCMStates *baseStates);
 168
 169 /**
 170  * Check a base table against an extension table.
 171  * Set the moveTarget!=NULL if it is possible to move mappings from the base.
 172  * This is the case where base and extension tables are parsed from a single file
 173  * (moveTarget==ext)
 174  * or when delta file mappings are subtracted from a base table.
 175  *
 176  * When a base table cannot be modified because a delta file is parsed in makeconv,
 177  * then set moveTarget=NULL.
 178  *
 179  * if(intersectBase) then mappings that exist in the base table but not in
 180  * the extension table are moved to moveTarget instead of showing an error.
 181  *
 182  * Special mode:
 183  * If intersectBase==2 for a DBCS extension table, then SBCS mappings are
 184  * not moved out of the base unless their Unicode input requires it.
 185  * This helps ucmkbase generate base tables for DBCS-only extension .cnv files.
 186  *
 187  * For both tables in the same file, the extension table is automatically
 188  * built.
 189  * For separate files, the extension file can use a complete mapping table,
 190  * so that common mappings need not be stripped out manually.
 191  *
 192  *
 193  * Sort both tables, and then for each mapping direction:
 194  *
 195  * If intersectBase is TRUE and the base table contains a mapping
 196  * that does not exist in the extension table, then this mapping is moved
 197  * to moveTarget.
 198  *
 199  * - otherwise -
 200  *
 201  * If the base table contains a mapping for which the input sequence is
 202  * the same as the extension input, then
 203  * - if the output is the same: remove the extension mapping
 204  * - else: error
 205  *
 206  * If the base table contains a mapping for which the input sequence is
 207  * a prefix of the extension input, then
 208  * - if moveTarget!=NULL: move the base mapping to the moveTarget table
 209  * - else: error
 210  *
 211  * @return FALSE in case of an irreparable error
 212  */
 213 U_CAPI UBool U_EXPORT2
 214 ucm_checkBaseExt(UCMStates *baseStates, UCMTable *base, UCMTable *ext,
 215                  UCMTable *moveTarget, UBool intersectBase);
 216
 217 U_CAPI void U_EXPORT2
 218 ucm_printTable(UCMTable *table, FILE *f, UBool byUnicode);
 219
 220 U_CAPI void U_EXPORT2
 221 ucm_printMapping(UCMTable *table, UCMapping *m, FILE *f);
 222
 223
 224 U_CAPI void U_EXPORT2
 225 ucm_addState(UCMStates *states, const char *s);
 226
 227 U_CAPI void U_EXPORT2
 228 ucm_processStates(UCMStates *states);
 229
 230 U_CAPI int32_t U_EXPORT2
 231 ucm_countChars(UCMStates *states,
 232                const uint8_t *bytes, int32_t length);
 233
 234
 235 U_CAPI int8_t U_EXPORT2
 236 ucm_parseBytes(uint8_t bytes[UCNV_EXT_MAX_BYTES], const char *line, const char **ps);
 237
 238 U_CAPI UBool U_EXPORT2
 239 ucm_parseMappingLine(UCMapping *m,
 240                      UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
 241                      uint8_t bytes[UCNV_EXT_MAX_BYTES],
 242                      const char *line);
 243
 244 U_CAPI void U_EXPORT2
 245 ucm_addMapping(UCMTable *table,
 246                UCMapping *m,
 247                UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
 248                uint8_t bytes[UCNV_EXT_MAX_BYTES]);
 249
 250 /* very makeconv-specific functions ----------------------------------------- */
 251
 252 /* finalize and optimize states after the toUnicode mappings are processed */
 253 U_CAPI void U_EXPORT2
 254 ucm_optimizeStates(UCMStates *states,
 255                    uint16_t **pUnicodeCodeUnits,
 256                    _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks,
 257                    UBool verbose);
 258
 259 /* moved here because it is used inside ucmstate.c */
 260 U_CAPI int32_t U_EXPORT2
 261 ucm_findFallback(_MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks,
 262                  uint32_t offset);
 263
 264 /* very rptp2ucm-specific functions ----------------------------------------- */
 265
 266 /*
 267  * Input: Separate tables with mappings from/to Unicode,
 268  * subchar and subchar1 (0 if none).
 269  * All mappings must have flag 0.
 270  *
 271  * Output: fromUTable will contain the union of mappings with the correct
 272  * precision flags, and be sorted.
 273  */
 274 U_CAPI void U_EXPORT2
 275 ucm_mergeTables(UCMTable *fromUTable, UCMTable *toUTable,
 276                 const uint8_t *subchar, int32_t subcharLength,
 277                 uint8_t subchar1);
 278
 279 U_CAPI UBool U_EXPORT2
 280 ucm_separateMappings(UCMFile *ucm, UBool isSISO);
 281
 282 U_CDECL_END
 283
 284 #endif
 285
 286 #endif
 287