[apple/icu.git] / icuSources / common / unicode / urbtok.h

/*
******************************************************************************
* Copyright (C) 2006-2008, 2017 Apple Inc. All Rights Reserved.
******************************************************************************
*/

#ifndef URBTOK_H
#define URBTOK_H

#include "unicode/utypes.h"

#if !UCONFIG_NO_BREAK_ITERATION

#include "unicode/utext.h"
#include "unicode/ubrk.h"
#include "unicode/parseerr.h"

/**
 * The interfaces here are meant to extend the functionality of the standard
 * ubrk_* interfaces in ubrk.h to allow for faster batch tokenization. This
 * was primarily intended for Spotlight and related processes. Note that
 * urbtok_tokenize here does not fully support all features of ICU break rules.
 */

typedef struct RuleBasedTokenRange {
    signed long location;
    signed long length;
} RuleBasedTokenRange;

/**
 * Open a new UBreakIterator for tokenizing text using specified breaking rules.
 * The rule syntax is ... (TBD)
 * @param rules A set of rules specifying the text breaking conventions.
 * @param rulesLength The number of characters in rules, or -1 if null-terminated.
 * @param parseErr   Receives position and context information for any syntax errors
 *                   detected while parsing the rules.
 * @param status A UErrorCode to receive any errors.
 * @return A UBreakIterator for the specified rules.
 * @see ubrk_open
 * @internal
 */
U_INTERNAL UBreakIterator* U_EXPORT2
urbtok_openRules(const UChar     *rules,
               int32_t         rulesLength,
               UParseError     *parseErr,
               UErrorCode      *status);

/**
 * Open a new UBreakIterator for tokenizing text using specified breaking rules.
 * @param rules A set of rules specifying the text breaking conventions. The binary rules
 *              must be at least 32-bit aligned. Note: This version makes a copy of the
 *				rules, so after calling this function the caller can close or release
 *				the rules that were passed to this function. The copy created by this
 *				call will be freed when ubrk_close() is called on the UBreakIterator*.
 * @param status A UErrorCode to receive any errors.
 * @return A UBreakIterator for the specified rules.
 * @see ubrk_open
 * @internal
 */
U_INTERNAL UBreakIterator* U_EXPORT2
urbtok_openBinaryRules(const uint8_t *rules,
               UErrorCode      *status);

/**
 * Open a new UBreakIterator for tokenizing text using specified breaking rules.
 * @param rules A set of rules specifying the text breaking conventions. The binary rules
 *              must be at least 32-bit aligned. Note: This version does NOT make a copy
 *				of the rules, so after calling this function the caller must not close or
 *				release the rules passed to this function until after they are finished
 *				with this UBreakIterator* (and any others created using the same rules)
  *				and have called ubrk_close() to close the UBreakIterator* (and any others
 *				using the same rules).
 * @param status A UErrorCode to receive any errors.
 * @return A UBreakIterator for the specified rules.
 * @see ubrk_open
 * @internal
 */
U_INTERNAL UBreakIterator* U_EXPORT2
urbtok_openBinaryRulesNoCopy(const uint8_t *rules,
               UErrorCode      *status);

/**
 * Get the (native-endian) binary break rules for this tokenizer.
 * @param bi The tokenizer to use.
 * @param buffer The output buffer for the rules. You can pass 0 to get the required size.
 * @param buffSize The size of the output buffer.
 * @param status A UErrorCode to receive any errors.
 * @return The actual size of the binary rules, whether they fit the buffer or not.
 * @internal
 */
U_INTERNAL uint32_t U_EXPORT2
urbtok_getBinaryRules(UBreakIterator      *bi,
                uint8_t             *buffer,
                uint32_t            buffSize,
                UErrorCode          *status);

/**
 * Tokenize text using a rule-based tokenizer.
 * This is primarily intended for speedy batch tokenization using very simple rules.
 * It does not currently implement support for all of the features of ICU break rules
 * (adding that would reduce performance). If you need support for all of the ICU rule
 * features, please use the standard ubrk_* interfaces; instead of urbtok_tokenize,
 * use a loop with ubrk_next and ubrk_getRuleStatus.
 *
 * @param bi The tokenizer to use.
 * @param maxTokens The maximum number of tokens to return.
 * @param outTokens An array of RuleBasedTokenRange to fill in with the tokens.
 * @param outTokenFlags An (optional) array of uint32_t to fill in with token flags.
 * @return The number of tokens returned, 0 if done.
 * @internal
 */
U_INTERNAL int32_t U_EXPORT2
urbtok_tokenize(UBreakIterator      *bi,
               int32_t              maxTokens,
               RuleBasedTokenRange  *outTokens,
               unsigned long        *outTokenFlags);

/**
 * Swap the endianness of a set of binary break rules.
 * @param rules A set of rules which need swapping.
 * @param buffer The output buffer for the swapped rules, which must be the same
 *               size as the input rules buffer.
 * @param inIsBigEndian UBool indicating whether the input is big-endian
 * @param outIsBigEndian UBool indicating whether the output should be big-endian
 * @param status A UErrorCode to receive any errors.
 * @internal
 */
U_INTERNAL void U_EXPORT2
urbtok_swapBinaryRules(const uint8_t *rules,
               uint8_t          *buffer,
               UBool            inIsBigEndian,
               UBool            outIsBigEndian,
               UErrorCode       *status);


#endif /* #if !UCONFIG_NO_BREAK_ITERATION */

#endif
Commit	Line	Data
73c04bcf A	1	/*
73c04bcf A	2	******************************************************************************
f3c0d7a5	3	* Copyright (C) 2006-2008, 2017 Apple Inc. All Rights Reserved.
73c04bcf A	4	******************************************************************************
	5	*/
	6
	7	#ifndef URBTOK_H
	8	#define URBTOK_H
	9
	10	#include "unicode/utypes.h"
	11
	12	#if !UCONFIG_NO_BREAK_ITERATION
	13
	14	#include "unicode/utext.h"
	15	#include "unicode/ubrk.h"
	16	#include "unicode/parseerr.h"
	17
f3c0d7a5 A	18	/**
	19	* The interfaces here are meant to extend the functionality of the standard
	20	* ubrk_* interfaces in ubrk.h to allow for faster batch tokenization. This
	21	* was primarily intended for Spotlight and related processes. Note that
	22	* urbtok_tokenize here does not fully support all features of ICU break rules.
	23	*/
73c04bcf A	24
	25	typedef struct RuleBasedTokenRange {
	26	signed long location;
	27	signed long length;
	28	} RuleBasedTokenRange;
	29
	30	/**
	31	* Open a new UBreakIterator for tokenizing text using specified breaking rules.
	32	* The rule syntax is ... (TBD)
	33	* @param rules A set of rules specifying the text breaking conventions.
	34	* @param rulesLength The number of characters in rules, or -1 if null-terminated.
	35	* @param parseErr Receives position and context information for any syntax errors
	36	* detected while parsing the rules.
	37	* @param status A UErrorCode to receive any errors.
	38	* @return A UBreakIterator for the specified rules.
	39	* @see ubrk_open
	40	* @internal
	41	*/
	42	U_INTERNAL UBreakIterator* U_EXPORT2
	43	urbtok_openRules(const UChar *rules,
	44	int32_t rulesLength,
	45	UParseError *parseErr,
	46	UErrorCode *status);
	47
	48	/**
	49	* Open a new UBreakIterator for tokenizing text using specified breaking rules.
	50	* @param rules A set of rules specifying the text breaking conventions. The binary rules
46f4442e A	51	* must be at least 32-bit aligned. Note: This version makes a copy of the
	52	* rules, so after calling this function the caller can close or release
	53	* the rules that were passed to this function. The copy created by this
	54	* call will be freed when ubrk_close() is called on the UBreakIterator*.
73c04bcf A	55	* @param status A UErrorCode to receive any errors.
	56	* @return A UBreakIterator for the specified rules.
	57	* @see ubrk_open
	58	* @internal
	59	*/
	60	U_INTERNAL UBreakIterator* U_EXPORT2
	61	urbtok_openBinaryRules(const uint8_t *rules,
46f4442e A	62	UErrorCode *status);
	63
	64	/**
	65	* Open a new UBreakIterator for tokenizing text using specified breaking rules.
	66	* @param rules A set of rules specifying the text breaking conventions. The binary rules
	67	* must be at least 32-bit aligned. Note: This version does NOT make a copy
	68	* of the rules, so after calling this function the caller must not close or
	69	* release the rules passed to this function until after they are finished
	70	* with this UBreakIterator* (and any others created using the same rules)
	71	* and have called ubrk_close() to close the UBreakIterator* (and any others
	72	* using the same rules).
	73	* @param status A UErrorCode to receive any errors.
	74	* @return A UBreakIterator for the specified rules.
	75	* @see ubrk_open
	76	* @internal
	77	*/
	78	U_INTERNAL UBreakIterator* U_EXPORT2
	79	urbtok_openBinaryRulesNoCopy(const uint8_t *rules,
73c04bcf A	80	UErrorCode *status);
	81
	82	/**
	83	* Get the (native-endian) binary break rules for this tokenizer.
	84	* @param bi The tokenizer to use.
	85	* @param buffer The output buffer for the rules. You can pass 0 to get the required size.
	86	* @param buffSize The size of the output buffer.
	87	* @param status A UErrorCode to receive any errors.
	88	* @return The actual size of the binary rules, whether they fit the buffer or not.
	89	* @internal
	90	*/
	91	U_INTERNAL uint32_t U_EXPORT2
	92	urbtok_getBinaryRules(UBreakIterator *bi,
	93	uint8_t *buffer,
	94	uint32_t buffSize,
	95	UErrorCode *status);
	96
	97	/**
	98	* Tokenize text using a rule-based tokenizer.
f3c0d7a5 A	99	* This is primarily intended for speedy batch tokenization using very simple rules.
	100	* It does not currently implement support for all of the features of ICU break rules
	101	* (adding that would reduce performance). If you need support for all of the ICU rule
	102	* features, please use the standard ubrk_* interfaces; instead of urbtok_tokenize,
	103	* use a loop with ubrk_next and ubrk_getRuleStatus.
	104	*
73c04bcf A	105	* @param bi The tokenizer to use.
	106	* @param maxTokens The maximum number of tokens to return.
	107	* @param outTokens An array of RuleBasedTokenRange to fill in with the tokens.
	108	* @param outTokenFlags An (optional) array of uint32_t to fill in with token flags.
	109	* @return The number of tokens returned, 0 if done.
	110	* @internal
	111	*/
	112	U_INTERNAL int32_t U_EXPORT2
	113	urbtok_tokenize(UBreakIterator *bi,
	114	int32_t maxTokens,
	115	RuleBasedTokenRange *outTokens,
	116	unsigned long *outTokenFlags);
	117
	118	/**
	119	* Swap the endianness of a set of binary break rules.
	120	* @param rules A set of rules which need swapping.
	121	* @param buffer The output buffer for the swapped rules, which must be the same
	122	* size as the input rules buffer.
	123	* @param inIsBigEndian UBool indicating whether the input is big-endian
	124	* @param outIsBigEndian UBool indicating whether the output should be big-endian
	125	* @param status A UErrorCode to receive any errors.
	126	* @internal
	127	*/
	128	U_INTERNAL void U_EXPORT2
	129	urbtok_swapBinaryRules(const uint8_t *rules,
	130	uint8_t *buffer,
	131	UBool inIsBigEndian,
	132	UBool outIsBigEndian,
	133	UErrorCode *status);
	134
	135
	136	#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
	137
	138	#endif