icuSources/i18n/ucol_tok.h

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 2001-2010, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  ucol_tok.h
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created 02/22/2001
  14 *   created by: Vladimir Weinstein
  15 *
  16 * This module reads a tailoring rule string and produces a list of
  17 * tokens that will be turned into collation elements
  18 *
  19 */
  20
  21 #ifndef UCOL_TOKENS_H
  22 #define UCOL_TOKENS_H
  23
  24 #include "unicode/utypes.h"
  25 #include "unicode/uset.h"
  26
  27 #if !UCONFIG_NO_COLLATION
  28
  29 #include "ucol_imp.h"
  30 #include "uhash.h"
  31 #include "unicode/parseerr.h"
  32
  33 #define UCOL_TOK_UNSET 0xFFFFFFFF
  34 #define UCOL_TOK_RESET 0xDEADBEEF
  35
  36 #define UCOL_TOK_POLARITY_NEGATIVE 0
  37 #define UCOL_TOK_POLARITY_POSITIVE 1
  38
  39 #define UCOL_TOK_TOP 0x04
  40 #define UCOL_TOK_VARIABLE_TOP 0x08
  41 #define UCOL_TOK_BEFORE 0x03
  42 #define UCOL_TOK_SUCCESS 0x10
  43
  44 /* this is space for the extra strings that need to be unquoted */
  45 /* during the parsing of the rules */
  46 #define UCOL_TOK_EXTRA_RULE_SPACE_SIZE 4096
  47 typedef struct UColToken UColToken;
  48
  49 typedef struct  {
  50   UColToken* first;
  51   UColToken* last;
  52   UColToken* reset;
  53   UBool indirect;
  54   uint32_t baseCE;
  55   uint32_t baseContCE;
  56   uint32_t nextCE;
  57   uint32_t nextContCE;
  58   uint32_t previousCE;
  59   uint32_t previousContCE;
  60   int32_t pos[UCOL_STRENGTH_LIMIT];
  61   uint32_t gapsLo[3*UCOL_CE_STRENGTH_LIMIT];
  62   uint32_t gapsHi[3*UCOL_CE_STRENGTH_LIMIT];
  63   uint32_t numStr[UCOL_CE_STRENGTH_LIMIT];
  64   UColToken* fStrToken[UCOL_CE_STRENGTH_LIMIT];
  65   UColToken* lStrToken[UCOL_CE_STRENGTH_LIMIT];
  66 } UColTokListHeader;
  67
  68 struct UColToken {
  69   UChar debugSource;
  70   UChar debugExpansion;
  71   UChar debugPrefix;
  72   uint32_t CEs[128];
  73   uint32_t noOfCEs;
  74   uint32_t expCEs[128];
  75   uint32_t noOfExpCEs;
  76   uint32_t source;
  77   uint32_t expansion;
  78   uint32_t prefix;
  79   uint32_t strength;
  80   uint32_t toInsert;
  81   uint32_t polarity; /* 1 for <, <<, <<<, , ; and -1 for >, >>, >>> */
  82   UColTokListHeader *listHeader;
  83   UColToken* previous;
  84   UColToken* next;
  85   UChar **rulesToParseHdl;
  86   uint16_t flags;
  87 };
  88
  89 /*
  90  * This is a token that has been parsed
  91  * but not yet processed. Used to reduce
  92  * the number of arguments in the parser
  93  */
  94 typedef struct {
  95   uint32_t strength;
  96   uint32_t charsOffset;
  97   uint32_t charsLen;
  98   uint32_t extensionOffset;
  99   uint32_t extensionLen;
 100   uint32_t prefixOffset;
 101   uint32_t prefixLen;
 102   uint16_t flags;
 103   uint16_t indirectIndex;
 104 } UColParsedToken;
 105
 106
 107 typedef struct {
 108   UColParsedToken parsedToken;
 109   UChar *source;
 110   UChar *end;
 111   const UChar *current;
 112   UChar *sourceCurrent;
 113   UChar *extraCurrent;
 114   UChar *extraEnd;
 115   const InverseUCATableHeader *invUCA;
 116   const UCollator *UCA;
 117   UHashtable *tailored;
 118   UColOptionSet *opts;
 119   uint32_t resultLen;
 120   uint32_t listCapacity;
 121   UColTokListHeader *lh;
 122   UColToken *varTop;
 123   USet *copySet;
 124   USet *removeSet;
 125   UBool buildCCTabFlag;  /* Tailoring rule requirs building combining class table. */
 126
 127   UChar32 previousCp;               /* Previous code point. */
 128   /* For processing starred lists. */
 129   UBool isStarred;                   /* Are we processing a starred token? */
 130   UBool savedIsStarred;
 131   uint32_t currentStarredCharIndex;  /* Index of the current charrecter in the starred expression. */
 132   uint32_t lastStarredCharIndex;    /* Index to the last character in the starred expression. */
 133
 134   /* For processing ranges. */
 135   UBool inRange;                     /* Are we in a range? */
 136   UChar32 currentRangeCp;           /* Current code point in the range. */
 137   UChar32 lastRangeCp;              /* The last code point in the range. */
 138
 139   /* reorder codes for collation reordering */
 140   int32_t* reorderCodes;
 141   int32_t reorderCodesLength;
 142
 143 } UColTokenParser;
 144
 145 typedef struct {
 146   const UChar *subName;
 147   int32_t subLen;
 148   UColAttributeValue attrVal;
 149 } ucolTokSuboption;
 150
 151 typedef struct {
 152    const UChar *optionName;
 153    int32_t optionLen;
 154    const ucolTokSuboption *subopts;
 155    int32_t subSize;
 156    UColAttribute attr;
 157 } ucolTokOption;
 158
 159 #define ucol_tok_isSpecialChar(ch)              \
 160     (((((ch) <= 0x002F) && ((ch) >= 0x0020)) || \
 161       (((ch) <= 0x003F) && ((ch) >= 0x003A)) || \
 162       (((ch) <= 0x0060) && ((ch) >= 0x005B)) || \
 163       (((ch) <= 0x007E) && ((ch) >= 0x007D)) || \
 164       (ch) == 0x007B))
 165
 166
 167 U_CFUNC
 168 uint32_t ucol_tok_assembleTokenList(UColTokenParser *src,
 169                                     UParseError *parseError,
 170                                     UErrorCode *status);
 171
 172 U_CFUNC
 173 void ucol_tok_initTokenList(UColTokenParser *src,
 174                             const UChar *rules,
 175                             const uint32_t rulesLength,
 176                             const UCollator *UCA,
 177                             GetCollationRulesFunction importFunc,
 178                             void* context,
 179                             UErrorCode *status);
 180
 181 U_CFUNC void ucol_tok_closeTokenList(UColTokenParser *src);
 182
 183 U_CAPI const UChar* U_EXPORT2 ucol_tok_parseNextToken(UColTokenParser *src,
 184                         UBool startOfRules,
 185                         UParseError *parseError,
 186                         UErrorCode *status);
 187
 188
 189 U_CAPI const UChar * U_EXPORT2
 190 ucol_tok_getNextArgument(const UChar *start, const UChar *end,
 191                                UColAttribute *attrib, UColAttributeValue *value,
 192                                UErrorCode *status);
 193 U_CAPI int32_t U_EXPORT2 ucol_inv_getNextCE(const UColTokenParser *src,
 194                                             uint32_t CE, uint32_t contCE,
 195                                             uint32_t *nextCE, uint32_t *nextContCE,
 196                                             uint32_t strength);
 197 U_CFUNC int32_t U_EXPORT2 ucol_inv_getPrevCE(const UColTokenParser *src,
 198                                             uint32_t CE, uint32_t contCE,
 199                                             uint32_t *prevCE, uint32_t *prevContCE,
 200                                             uint32_t strength);
 201
 202 U_CFUNC const UChar* ucol_tok_getRulesFromBundle(
 203     void* context,
 204     const char* locale,
 205     const char* type,
 206     int32_t* pLength,
 207     UErrorCode* status);
 208
 209 #endif /* #if !UCONFIG_NO_COLLATION */
 210
 211 #endif