[apple/icu.git] / icuSources / i18n / ucol_tok.h

/*
*******************************************************************************
*
*   Copyright (C) 2001-2011, International Business Machines
*   Corporation and others.  All Rights Reserved.
*
*******************************************************************************
*   file name:  ucol_tok.h
*   encoding:   US-ASCII
*   tab size:   8 (not used)
*   indentation:4
*
*   created 02/22/2001
*   created by: Vladimir Weinstein
*
* This module reads a tailoring rule string and produces a list of 
* tokens that will be turned into collation elements
* 
*/

#ifndef UCOL_TOKENS_H
#define UCOL_TOKENS_H

#include "unicode/utypes.h"
#include "unicode/uset.h"

#if !UCONFIG_NO_COLLATION

#include "ucol_imp.h"
#include "uhash.h"
#include "unicode/parseerr.h"

#define UCOL_TOK_UNSET 0xFFFFFFFF
#define UCOL_TOK_RESET 0xDEADBEEF

#define UCOL_TOK_POLARITY_NEGATIVE 0
#define UCOL_TOK_POLARITY_POSITIVE 1

#define UCOL_TOK_TOP 0x04
#define UCOL_TOK_VARIABLE_TOP 0x08
#define UCOL_TOK_BEFORE 0x03
#define UCOL_TOK_SUCCESS 0x10

/* this is space for the extra strings that need to be unquoted */
/* during the parsing of the rules */
#define UCOL_TOK_EXTRA_RULE_SPACE_SIZE 4096
typedef struct UColToken UColToken;

typedef struct  {
  UColToken* first;
  UColToken* last;
  UColToken* reset;
  UBool indirect;
  uint32_t baseCE;
  uint32_t baseContCE;
  uint32_t nextCE;
  uint32_t nextContCE;
  uint32_t previousCE;
  uint32_t previousContCE;
  int32_t pos[UCOL_STRENGTH_LIMIT];
  uint32_t gapsLo[3*UCOL_CE_STRENGTH_LIMIT];
  uint32_t gapsHi[3*UCOL_CE_STRENGTH_LIMIT];
  uint32_t numStr[UCOL_CE_STRENGTH_LIMIT];
  UColToken* fStrToken[UCOL_CE_STRENGTH_LIMIT];
  UColToken* lStrToken[UCOL_CE_STRENGTH_LIMIT];
} UColTokListHeader;

struct UColToken {
  UChar debugSource;
  UChar debugExpansion;
  UChar debugPrefix;
  uint32_t CEs[128];
  uint32_t noOfCEs;
  uint32_t expCEs[128];
  uint32_t noOfExpCEs;
  uint32_t source;
  uint32_t expansion;
  uint32_t prefix;
  uint32_t strength;
  uint32_t toInsert;
  uint32_t polarity; /* 1 for <, <<, <<<, , ; and -1 for >, >>, >>> */
  UColTokListHeader *listHeader;
  UColToken* previous;
  UColToken* next;
  UChar **rulesToParseHdl;
  uint16_t flags;
};

/* 
 * This is a token that has been parsed
 * but not yet processed. Used to reduce
 * the number of arguments in the parser
 */
typedef struct {
  uint32_t strength;
  uint32_t charsOffset;
  uint32_t charsLen;
  uint32_t extensionOffset;
  uint32_t extensionLen;
  uint32_t prefixOffset;
  uint32_t prefixLen;
  uint16_t flags;
  uint16_t indirectIndex;
} UColParsedToken;


typedef struct {
  UColParsedToken parsedToken;
  UChar *source;
  UChar *end;
  const UChar *current;
  UChar *sourceCurrent;
  UChar *extraCurrent;
  UChar *extraEnd;
  const InverseUCATableHeader *invUCA;
  const UCollator *UCA;
  UHashtable *tailored;
  UColOptionSet *opts;
  uint32_t resultLen;
  uint32_t listCapacity;
  UColTokListHeader *lh;
  UColToken *varTop;
  USet *copySet;
  USet *removeSet;
  UBool buildCCTabFlag;  /* Tailoring rule requirs building combining class table. */

  UChar32 previousCp;               /* Previous code point. */
  /* For processing starred lists. */
  UBool isStarred;                   /* Are we processing a starred token? */
  UBool savedIsStarred;
  uint32_t currentStarredCharIndex;  /* Index of the current charrecter in the starred expression. */
  uint32_t lastStarredCharIndex;    /* Index to the last character in the starred expression. */

  /* For processing ranges. */
  UBool inRange;                     /* Are we in a range? */
  UChar32 currentRangeCp;           /* Current code point in the range. */
  UChar32 lastRangeCp;              /* The last code point in the range. */
  
  /* reorder codes for collation reordering */
  int32_t* reorderCodes;
  int32_t reorderCodesLength;

} UColTokenParser;

typedef struct {
  const UChar *subName;
  int32_t subLen;
  UColAttributeValue attrVal;
} ucolTokSuboption;

typedef struct {
   const UChar *optionName;
   int32_t optionLen;
   const ucolTokSuboption *subopts;
   int32_t subSize;
   UColAttribute attr;
} ucolTokOption;

#define ucol_tok_isSpecialChar(ch)              \
    (((((ch) <= 0x002F) && ((ch) >= 0x0020)) || \
      (((ch) <= 0x003F) && ((ch) >= 0x003A)) || \
      (((ch) <= 0x0060) && ((ch) >= 0x005B)) || \
      (((ch) <= 0x007E) && ((ch) >= 0x007D)) || \
      (ch) == 0x007B))


U_CFUNC 
uint32_t ucol_tok_assembleTokenList(UColTokenParser *src,
                                    UParseError *parseError, 
                                    UErrorCode *status);

U_CFUNC
void ucol_tok_initTokenList(UColTokenParser *src,
                            const UChar *rules,
                            const uint32_t rulesLength,
                            const UCollator *UCA,
                            GetCollationRulesFunction importFunc,
                            void* context,
                            UErrorCode *status);

U_CFUNC void ucol_tok_closeTokenList(UColTokenParser *src);

U_CAPI const UChar* U_EXPORT2 ucol_tok_parseNextToken(UColTokenParser *src, 
                        UBool startOfRules,
                        UParseError *parseError,
                        UErrorCode *status);


U_CAPI const UChar * U_EXPORT2
ucol_tok_getNextArgument(const UChar *start, const UChar *end, 
                               UColAttribute *attrib, UColAttributeValue *value, 
                               UErrorCode *status);
U_CAPI int32_t U_EXPORT2 ucol_inv_getNextCE(const UColTokenParser *src,
                                            uint32_t CE, uint32_t contCE,
                                            uint32_t *nextCE, uint32_t *nextContCE,
                                            uint32_t strength);
U_CFUNC int32_t U_EXPORT2 ucol_inv_getPrevCE(const UColTokenParser *src,
                                            uint32_t CE, uint32_t contCE,
                                            uint32_t *prevCE, uint32_t *prevContCE,
                                            uint32_t strength);

const UChar* U_CALLCONV ucol_tok_getRulesFromBundle(
    void* context,
    const char* locale,
    const char* type,
    int32_t* pLength,
    UErrorCode* status);

#endif /* #if !UCONFIG_NO_COLLATION */

#endif
Commit	Line	Data
b75a7d8f A	1	/*
	2	*******************************************************************************
	3	*
4388f060	4	* Copyright (C) 2001-2011, International Business Machines
b75a7d8f A	5	* Corporation and others. All Rights Reserved.
	6	*
	7	*******************************************************************************
	8	* file name: ucol_tok.h
	9	* encoding: US-ASCII
	10	* tab size: 8 (not used)
	11	* indentation:4
	12	*
	13	* created 02/22/2001
	14	* created by: Vladimir Weinstein
	15	*
	16	* This module reads a tailoring rule string and produces a list of
	17	* tokens that will be turned into collation elements
	18	*
	19	*/
	20
	21	#ifndef UCOL_TOKENS_H
	22	#define UCOL_TOKENS_H
	23
	24	#include "unicode/utypes.h"
	25	#include "unicode/uset.h"
	26
	27	#if !UCONFIG_NO_COLLATION
	28
	29	#include "ucol_imp.h"
	30	#include "uhash.h"
	31	#include "unicode/parseerr.h"
	32
	33	#define UCOL_TOK_UNSET 0xFFFFFFFF
	34	#define UCOL_TOK_RESET 0xDEADBEEF
	35
	36	#define UCOL_TOK_POLARITY_NEGATIVE 0
	37	#define UCOL_TOK_POLARITY_POSITIVE 1
	38
	39	#define UCOL_TOK_TOP 0x04
	40	#define UCOL_TOK_VARIABLE_TOP 0x08
	41	#define UCOL_TOK_BEFORE 0x03
	42	#define UCOL_TOK_SUCCESS 0x10
	43
	44	/* this is space for the extra strings that need to be unquoted */
	45	/* during the parsing of the rules */
73c04bcf	46	#define UCOL_TOK_EXTRA_RULE_SPACE_SIZE 4096
b75a7d8f A	47	typedef struct UColToken UColToken;
	48
	49	typedef struct {
	50	UColToken* first;
	51	UColToken* last;
	52	UColToken* reset;
	53	UBool indirect;
	54	uint32_t baseCE;
	55	uint32_t baseContCE;
	56	uint32_t nextCE;
	57	uint32_t nextContCE;
	58	uint32_t previousCE;
	59	uint32_t previousContCE;
	60	int32_t pos[UCOL_STRENGTH_LIMIT];
	61	uint32_t gapsLo[3*UCOL_CE_STRENGTH_LIMIT];
	62	uint32_t gapsHi[3*UCOL_CE_STRENGTH_LIMIT];
	63	uint32_t numStr[UCOL_CE_STRENGTH_LIMIT];
	64	UColToken* fStrToken[UCOL_CE_STRENGTH_LIMIT];
	65	UColToken* lStrToken[UCOL_CE_STRENGTH_LIMIT];
	66	} UColTokListHeader;
	67
	68	struct UColToken {
	69	UChar debugSource;
	70	UChar debugExpansion;
	71	UChar debugPrefix;
	72	uint32_t CEs[128];
	73	uint32_t noOfCEs;
	74	uint32_t expCEs[128];
	75	uint32_t noOfExpCEs;
	76	uint32_t source;
	77	uint32_t expansion;
	78	uint32_t prefix;
	79	uint32_t strength;
	80	uint32_t toInsert;
	81	uint32_t polarity; /* 1 for <, <<, <<<, , ; and -1 for >, >>, >>> */
	82	UColTokListHeader *listHeader;
	83	UColToken* previous;
	84	UColToken* next;
729e4ab9	85	UChar **rulesToParseHdl;
374ca955	86	uint16_t flags;
b75a7d8f A	87	};
	88
	89	/*
	90	* This is a token that has been parsed
	91	* but not yet processed. Used to reduce
	92	* the number of arguments in the parser
	93	*/
	94	typedef struct {
	95	uint32_t strength;
	96	uint32_t charsOffset;
	97	uint32_t charsLen;
	98	uint32_t extensionOffset;
	99	uint32_t extensionLen;
	100	uint32_t prefixOffset;
	101	uint32_t prefixLen;
	102	uint16_t flags;
	103	uint16_t indirectIndex;
	104	} UColParsedToken;
	105
	106
	107	typedef struct {
	108	UColParsedToken parsedToken;
	109	UChar *source;
	110	UChar *end;
	111	const UChar *current;
	112	UChar *sourceCurrent;
	113	UChar *extraCurrent;
	114	UChar *extraEnd;
	115	const InverseUCATableHeader *invUCA;
	116	const UCollator *UCA;
	117	UHashtable *tailored;
	118	UColOptionSet *opts;
	119	uint32_t resultLen;
	120	uint32_t listCapacity;
	121	UColTokListHeader *lh;
	122	UColToken *varTop;
	123	USet *copySet;
	124	USet *removeSet;
46f4442e	125	UBool buildCCTabFlag; /* Tailoring rule requirs building combining class table. */
729e4ab9 A	126
	127	UChar32 previousCp; /* Previous code point. */
	128	/* For processing starred lists. */
	129	UBool isStarred; /* Are we processing a starred token? */
	130	UBool savedIsStarred;
	131	uint32_t currentStarredCharIndex; /* Index of the current charrecter in the starred expression. */
	132	uint32_t lastStarredCharIndex; /* Index to the last character in the starred expression. */
	133
	134	/* For processing ranges. */
	135	UBool inRange; /* Are we in a range? */
	136	UChar32 currentRangeCp; /* Current code point in the range. */
	137	UChar32 lastRangeCp; /* The last code point in the range. */
	138
	139	/* reorder codes for collation reordering */
	140	int32_t* reorderCodes;
	141	int32_t reorderCodesLength;
	142
b75a7d8f A	143	} UColTokenParser;
	144
	145	typedef struct {
	146	const UChar *subName;
	147	int32_t subLen;
	148	UColAttributeValue attrVal;
	149	} ucolTokSuboption;
	150
	151	typedef struct {
	152	const UChar *optionName;
	153	int32_t optionLen;
	154	const ucolTokSuboption *subopts;
	155	int32_t subSize;
	156	UColAttribute attr;
	157	} ucolTokOption;
	158
	159	#define ucol_tok_isSpecialChar(ch) \
	160	(((((ch) <= 0x002F) && ((ch) >= 0x0020)) \|\| \
	161	(((ch) <= 0x003F) && ((ch) >= 0x003A)) \|\| \
	162	(((ch) <= 0x0060) && ((ch) >= 0x005B)) \|\| \
	163	(((ch) <= 0x007E) && ((ch) >= 0x007D)) \|\| \
	164	(ch) == 0x007B))
	165
	166
	167	U_CFUNC
	168	uint32_t ucol_tok_assembleTokenList(UColTokenParser *src,
	169	UParseError *parseError,
	170	UErrorCode *status);
	171
	172	U_CFUNC
729e4ab9 A	173	void ucol_tok_initTokenList(UColTokenParser *src,
	174	const UChar *rules,
	175	const uint32_t rulesLength,
	176	const UCollator *UCA,
	177	GetCollationRulesFunction importFunc,
	178	void* context,
	179	UErrorCode *status);
b75a7d8f A	180
	181	U_CFUNC void ucol_tok_closeTokenList(UColTokenParser *src);
	182
	183	U_CAPI const UChar* U_EXPORT2 ucol_tok_parseNextToken(UColTokenParser *src,
	184	UBool startOfRules,
	185	UParseError *parseError,
	186	UErrorCode *status);
	187
729e4ab9	188
b75a7d8f A	189	U_CAPI const UChar * U_EXPORT2
	190	ucol_tok_getNextArgument(const UChar start, const UChar end,
	191	UColAttribute attrib, UColAttributeValue value,
	192	UErrorCode *status);
374ca955 A	193	U_CAPI int32_t U_EXPORT2 ucol_inv_getNextCE(const UColTokenParser *src,
	194	uint32_t CE, uint32_t contCE,
	195	uint32_t nextCE, uint32_t nextContCE,
	196	uint32_t strength);
46f4442e	197	U_CFUNC int32_t U_EXPORT2 ucol_inv_getPrevCE(const UColTokenParser *src,
374ca955 A	198	uint32_t CE, uint32_t contCE,
	199	uint32_t prevCE, uint32_t prevContCE,
	200	uint32_t strength);
	201
4388f060	202	const UChar* U_CALLCONV ucol_tok_getRulesFromBundle(
729e4ab9 A	203	void* context,
	204	const char* locale,
	205	const char* type,
	206	int32_t* pLength,
	207	UErrorCode* status);
b75a7d8f A	208
	209	#endif /* #if !UCONFIG_NO_COLLATION */
	210
	211	#endif