- const UChar *src, *start;
- uint8_t leadingCC;
- uint8_t trailingCC = 0;
- uint16_t fcd;
- UBool result = FALSE;
-
- start = data->string;
- src = data->pos + 1;
-
- /* Get the trailing combining class of the current character. */
- fcd = g_nfcImpl->previousFCD16(start, src);
-
- leadingCC = (uint8_t)(fcd >> SECOND_LAST_BYTE_SHIFT_);
-
- if (leadingCC != 0) {
- /*
- The current char has a non-zero leading combining class.
- Scan backward until we find a char with a trailing cc of zero.
- */
- for (;;)
- {
- if (start == src) {
- data->fcdPosition = NULL;
- return result;
- }
-
- fcd = g_nfcImpl->previousFCD16(start, src);
-
- trailingCC = (uint8_t)(fcd & LAST_BYTE_MASK_);
-
- if (trailingCC == 0) {
- break;
- }
-
- if (leadingCC < trailingCC) {
- result = TRUE;
- }
-
- leadingCC = (uint8_t)(fcd >> SECOND_LAST_BYTE_SHIFT_);
- }
- }
-
- data->fcdPosition = (UChar *)src;
-
- return result;
-}
-
-/** gets a code unit from the string at a given offset
- * Handles both normal and iterative cases.
- * No error checking - caller beware!
- */
-static inline
-UChar peekCodeUnit(collIterate *source, int32_t offset) {
- if(source->pos != NULL) {
- return *(source->pos + offset);
- } else if(source->iterator != NULL) {
- UChar32 c;
- if(offset != 0) {
- source->iterator->move(source->iterator, offset, UITER_CURRENT);
- c = source->iterator->next(source->iterator);
- source->iterator->move(source->iterator, -offset-1, UITER_CURRENT);
- } else {
- c = source->iterator->current(source->iterator);
- }
- return c >= 0 ? (UChar)c : 0xfffd; // If the caller works properly, we should never see c<0.
- } else {
- return 0xfffd;
- }
-}
-
-// Code point version. Treats the offset as a _code point_ delta.
-// We cannot use U16_FWD_1_UNSAFE and similar because we might not have well-formed UTF-16.
-// We cannot use U16_FWD_1 and similar because we do not know the start and limit of the buffer.
-static inline
-UChar32 peekCodePoint(collIterate *source, int32_t offset) {
- UChar32 c;
- if(source->pos != NULL) {
- const UChar *p = source->pos;
- if(offset >= 0) {
- // Skip forward over (offset-1) code points.
- while(--offset >= 0) {
- if(U16_IS_LEAD(*p++) && U16_IS_TRAIL(*p)) {
- ++p;
- }
- }
- // Read the code point there.
- c = *p++;
- UChar trail;
- if(U16_IS_LEAD(c) && U16_IS_TRAIL(trail = *p)) {
- c = U16_GET_SUPPLEMENTARY(c, trail);
- }
- } else /* offset<0 */ {
- // Skip backward over (offset-1) code points.
- while(++offset < 0) {
- if(U16_IS_TRAIL(*--p) && U16_IS_LEAD(*(p - 1))) {
- --p;
- }
- }
- // Read the code point before that.
- c = *--p;
- UChar lead;
- if(U16_IS_TRAIL(c) && U16_IS_LEAD(lead = *(p - 1))) {
- c = U16_GET_SUPPLEMENTARY(lead, c);
- }
- }
- } else if(source->iterator != NULL) {
- if(offset >= 0) {
- // Skip forward over (offset-1) code points.
- int32_t fwd = offset;
- while(fwd-- > 0) {
- uiter_next32(source->iterator);
- }
- // Read the code point there.
- c = uiter_current32(source->iterator);
- // Return to the starting point, skipping backward over (offset-1) code points.
- while(offset-- > 0) {
- uiter_previous32(source->iterator);
- }
- } else /* offset<0 */ {
- // Read backward, reading offset code points, remember only the last-read one.
- int32_t back = offset;
- do {
- c = uiter_previous32(source->iterator);
- } while(++back < 0);
- // Return to the starting position, skipping forward over offset code points.
- do {
- uiter_next32(source->iterator);
- } while(++offset < 0);
- }
- } else {
- c = U_SENTINEL;
- }
- return c;
-}
-
-/**
-* Determines if we are at the start of the data string in the backwards
-* collation iterator
-* @param data collation iterator
-* @return TRUE if we are at the start
-*/
-static
-inline UBool isAtStartPrevIterate(collIterate *data) {
- if(data->pos == NULL && data->iterator != NULL) {
- return !data->iterator->hasPrevious(data->iterator);
- }
- //return (collIter_bos(data)) ||
- return (data->pos == data->string) ||
- ((data->flags & UCOL_ITER_INNORMBUF) && (data->pos != NULL) &&
- *(data->pos - 1) == 0 && data->fcdPosition == NULL);
-}
-
-static
-inline void goBackOne(collIterate *data) {
-# if 0
- // somehow, it looks like we need to keep iterator synced up
- // at all times, as above.
- if(data->pos) {
- data->pos--;
- }
- if(data->iterator) {
- data->iterator->previous(data->iterator);
- }
-#endif
- if(data->iterator && (data->flags & UCOL_USE_ITERATOR)) {
- data->iterator->previous(data->iterator);
- }
- if(data->pos) {
- data->pos --;
- }
-}
-
-/**
-* Inline function that gets a simple CE.
-* So what it does is that it will first check the expansion buffer. If the
-* expansion buffer is not empty, ie the end pointer to the expansion buffer
-* is different from the string pointer, we return the collation element at the
-* return pointer and decrement it.
-* For more complicated CEs it resorts to getComplicatedCE.
-* @param coll collator data
-* @param data collation iterator struct
-* @param status error status
-*/
-static
-inline uint32_t ucol_IGetPrevCE(const UCollator *coll, collIterate *data,
- UErrorCode *status)
-{
- uint32_t result = (uint32_t)UCOL_NULLORDER;
-
- if (data->offsetReturn != NULL) {
- if (data->offsetRepeatCount > 0) {
- data->offsetRepeatCount -= 1;
- } else {
- if (data->offsetReturn == data->offsetBuffer) {
- data->offsetReturn = NULL;
- data->offsetStore = data->offsetBuffer;
- } else {
- data->offsetReturn -= 1;
- }
- }
- }
-
- if ((data->extendCEs && data->toReturn > data->extendCEs) ||
- (!data->extendCEs && data->toReturn > data->CEs))
- {
- data->toReturn -= 1;
- result = *(data->toReturn);
- if (data->CEs == data->toReturn || data->extendCEs == data->toReturn) {
- data->CEpos = data->toReturn;
- }
- }
- else {
- UChar ch = 0;
-
- do {
- /*
- Loop handles case when incremental normalize switches to or from the
- side buffer / original string, and we need to start again to get the
- next character.
- */
- for (;;) {
- if (data->flags & UCOL_ITER_HASLEN) {
- /*
- Normal path for strings when length is specified.
- Not in side buffer because it is always null terminated.
- */
- if (data->pos <= data->string) {
- /* End of the main source string */
- return UCOL_NO_MORE_CES;
- }
- data->pos --;
- ch = *data->pos;
- }
- // we are using an iterator to go back. Pray for us!
- else if (data->flags & UCOL_USE_ITERATOR) {
- UChar32 iterCh = data->iterator->previous(data->iterator);
- if(iterCh == U_SENTINEL) {
- return UCOL_NO_MORE_CES;
- } else {
- ch = (UChar)iterCh;
- }
- }
- else {
- data->pos --;
- ch = *data->pos;
- /* we are in the side buffer. */
- if (ch == 0) {
- /*
- At the start of the normalize side buffer.
- Go back to string.
- Because pointer points to the last accessed character,
- hence we have to increment it by one here.
- */
- data->flags = data->origFlags;
- data->offsetRepeatValue = 0;
-
- if (data->fcdPosition == NULL) {
- data->pos = data->string;
- return UCOL_NO_MORE_CES;
- }
- else {
- data->pos = data->fcdPosition + 1;
- }
-
- continue;
- }
- }
-
- if(data->flags&UCOL_HIRAGANA_Q) {
- if(ch>=0x3040 && ch<=0x309f) {
- data->flags |= UCOL_WAS_HIRAGANA;
- } else {
- data->flags &= ~UCOL_WAS_HIRAGANA;
- }
- }
-
- /*
- * got a character to determine if there's fcd and/or normalization
- * stuff to do.
- * if the current character is not fcd.
- * if current character is at the start of the string
- * Trailing combining class == 0.
- * Note if pos is in the writablebuffer, norm is always 0
- */
- if (ch < ZERO_CC_LIMIT_ ||
- // this should propel us out of the loop in the iterator case
- (data->flags & UCOL_ITER_NORM) == 0 ||
- (data->fcdPosition != NULL && data->fcdPosition <= data->pos)
- || data->string == data->pos) {
- break;
- }
-
- if (ch < NFC_ZERO_CC_BLOCK_LIMIT_) {
- /* if next character is FCD */
- if (data->pos == data->string) {
- /* First char of string is always OK for FCD check */
- break;
- }
-
- /* Not first char of string, do the FCD fast test */
- if (*(data->pos - 1) < NFC_ZERO_CC_BLOCK_LIMIT_) {
- break;
- }
- }
-
- /* Need a more complete FCD check and possible normalization. */
- if (collPrevIterFCD(data)) {
- collPrevIterNormalize(data);
- }
-
- if ((data->flags & UCOL_ITER_INNORMBUF) == 0) {
- /* No normalization. Go ahead and process the char. */
- break;
- }
-
- /*
- Some normalization happened.
- Next loop picks up a char from the normalization buffer.
- */
- }
-
- /* attempt to handle contractions, after removal of the backwards
- contraction
- */
- if (ucol_contractionEndCP(ch, coll) && !isAtStartPrevIterate(data)) {
- result = ucol_prv_getSpecialPrevCE(coll, ch, UCOL_CONTRACTION, data, status);
- } else {
- if (ch <= 0xFF) {
- result = coll->latinOneMapping[ch];
- }
- else {
- // Always use UCA for [3400..9FFF], [AC00..D7AF]
- // **** [FA0E..FA2F] ?? ****
- if ((data->flags & UCOL_FORCE_HAN_IMPLICIT) != 0 &&
- (ch >= 0x3400 && ch <= 0xD7AF)) {
- if (ch > 0x9FFF && ch < 0xAC00) {
- // between the two target ranges; do normal lookup
- // **** this range is YI, Modifier tone letters, ****
- // **** Latin-D, Syloti Nagari, Phagas-pa. ****
- // **** Latin-D might be tailored, so we need to ****
- // **** do the normal lookup for these guys. ****
- result = UTRIE_GET32_FROM_LEAD(&coll->mapping, ch);
- } else {
- result = UCOL_NOT_FOUND;
- }
- } else {
- result = UTRIE_GET32_FROM_LEAD(&coll->mapping, ch);
- }
- }
- if (result > UCOL_NOT_FOUND) {
- result = ucol_prv_getSpecialPrevCE(coll, ch, result, data, status);
- }
- if (result == UCOL_NOT_FOUND) { // Not found in master list
- if (!isAtStartPrevIterate(data) &&
- ucol_contractionEndCP(ch, data->coll))
- {
- result = UCOL_CONTRACTION;
- } else {
- if(coll->UCA) {
- result = UTRIE_GET32_FROM_LEAD(&coll->UCA->mapping, ch);
- }
- }
-
- if (result > UCOL_NOT_FOUND) {
- if(coll->UCA) {
- result = ucol_prv_getSpecialPrevCE(coll->UCA, ch, result, data, status);
- }
- }
- }
- }
- } while ( result == UCOL_IGNORABLE && ch >= UCOL_FIRST_HANGUL && ch <= UCOL_LAST_HANGUL );
-
- if(result == UCOL_NOT_FOUND) {
- result = getPrevImplicit(ch, data);
- }
- }
-
- return result;
-}
-
-
-/* ucol_getPrevCE, out-of-line version for use from other files. */
-U_CFUNC uint32_t U_EXPORT2
-ucol_getPrevCE(const UCollator *coll, collIterate *data,
- UErrorCode *status) {
- return ucol_IGetPrevCE(coll, data, status);
-}
-
-
-/* this should be connected to special Jamo handling */
-U_CFUNC uint32_t U_EXPORT2
-ucol_getFirstCE(const UCollator *coll, UChar u, UErrorCode *status) {
- collIterate colIt;
- IInit_collIterate(coll, &u, 1, &colIt, status);
- if(U_FAILURE(*status)) {
- return 0;
- }
- return ucol_IGetNextCE(coll, &colIt, status);
-}
-
-/**
-* Inserts the argument character into the end of the buffer pushing back the
-* null terminator.
-* @param data collIterate struct data
-* @param ch character to be appended
-* @return the position of the new addition
-*/
-static
-inline const UChar * insertBufferEnd(collIterate *data, UChar ch)
-{
- int32_t oldLength = data->writableBuffer.length();
- return data->writableBuffer.append(ch).getTerminatedBuffer() + oldLength;
-}
-
-/**
-* Inserts the argument string into the end of the buffer pushing back the
-* null terminator.
-* @param data collIterate struct data
-* @param string to be appended
-* @param length of the string to be appended
-* @return the position of the new addition
-*/
-static
-inline const UChar * insertBufferEnd(collIterate *data, const UChar *str, int32_t length)
-{
- int32_t oldLength = data->writableBuffer.length();
- return data->writableBuffer.append(str, length).getTerminatedBuffer() + oldLength;
-}
-
-/**
-* Special normalization function for contraction in the forwards iterator.
-* This normalization sequence will place the current character at source->pos
-* and its following normalized sequence into the buffer.
-* The fcd position, pos will be changed.
-* pos will now point to positions in the buffer.
-* Flags will be changed accordingly.
-* @param data collation iterator data
-*/
-static
-inline void normalizeNextContraction(collIterate *data)
-{
- int32_t strsize;
- UErrorCode status = U_ZERO_ERROR;
- /* because the pointer points to the next character */
- const UChar *pStart = data->pos - 1;
- const UChar *pEnd;
-
- if ((data->flags & UCOL_ITER_INNORMBUF) == 0) {
- data->writableBuffer.setTo(*(pStart - 1));
- strsize = 1;
- }
- else {
- strsize = data->writableBuffer.length();
- }
-
- pEnd = data->fcdPosition;
-
- data->writableBuffer.append(
- data->nfd->normalize(UnicodeString(FALSE, pStart, (int32_t)(pEnd - pStart)), status));
- if(U_FAILURE(status)) {
- return;
- }
-
- data->pos = data->writableBuffer.getTerminatedBuffer() + strsize;
- data->origFlags = data->flags;
- data->flags |= UCOL_ITER_INNORMBUF;
- data->flags &= ~(UCOL_ITER_NORM | UCOL_ITER_HASLEN);
-}
-
-/**
-* Contraction character management function that returns the next character
-* for the forwards iterator.
-* Does nothing if the next character is in buffer and not the first character
-* in it.
-* Else it checks next character in data string to see if it is normalizable.
-* If it is not, the character is simply copied into the buffer, else
-* the whole normalized substring is copied into the buffer, including the
-* current character.
-* @param data collation element iterator data
-* @return next character
-*/
-static
-inline UChar getNextNormalizedChar(collIterate *data)
-{
- UChar nextch;
- UChar ch;
- // Here we need to add the iterator code. One problem is the way
- // end of string is handled. If we just return next char, it could
- // be the sentinel. Most of the cases already check for this, but we
- // need to be sure.
- if ((data->flags & (UCOL_ITER_NORM | UCOL_ITER_INNORMBUF)) == 0 ) {
- /* if no normalization and not in buffer. */
- if(data->flags & UCOL_USE_ITERATOR) {
- return (UChar)data->iterator->next(data->iterator);
- } else {
- return *(data->pos ++);
- }
- }
-
- //if (data->flags & UCOL_ITER_NORM && data->flags & UCOL_USE_ITERATOR) {
- //normalizeIterator(data);
- //}
-
- UBool innormbuf = (UBool)(data->flags & UCOL_ITER_INNORMBUF);
- if ((innormbuf && *data->pos != 0) ||
- (data->fcdPosition != NULL && !innormbuf &&
- data->pos < data->fcdPosition)) {
- /*
- if next character is in normalized buffer, no further normalization
- is required
- */
- return *(data->pos ++);
- }
-
- if (data->flags & UCOL_ITER_HASLEN) {
- /* in data string */
- if (data->pos + 1 == data->endp) {
- return *(data->pos ++);
- }
- }
- else {
- if (innormbuf) {
- // inside the normalization buffer, but at the end
- // (since we encountered zero). This means, in the
- // case we're using char iterator, that we need to
- // do another round of normalization.
- //if(data->origFlags & UCOL_USE_ITERATOR) {
- // we need to restore original flags,
- // otherwise, we'll lose them
- //data->flags = data->origFlags;
- //normalizeIterator(data);
- //return *(data->pos++);
- //} else {
- /*
- in writable buffer, at this point fcdPosition can not be
- pointing to the end of the data string. see contracting tag.
- */
- if(data->fcdPosition) {
- if (*(data->fcdPosition + 1) == 0 ||
- data->fcdPosition + 1 == data->endp) {
- /* at the end of the string, dump it into the normalizer */
- data->pos = insertBufferEnd(data, *(data->fcdPosition)) + 1;
- // Check if data->pos received a null pointer
- if (data->pos == NULL) {
- return (UChar)-1; // Return to indicate error.
- }
- return *(data->fcdPosition ++);
- }
- data->pos = data->fcdPosition;
- } else if(data->origFlags & UCOL_USE_ITERATOR) {
- // if we are here, we're using a normalizing iterator.
- // we should just continue further.
- data->flags = data->origFlags;
- data->pos = NULL;
- return (UChar)data->iterator->next(data->iterator);
- }
- //}
- }
- else {
- if (*(data->pos + 1) == 0) {
- return *(data->pos ++);
- }
- }
- }
-
- ch = *data->pos ++;
- nextch = *data->pos;
-
- /*
- * if the current character is not fcd.
- * Trailing combining class == 0.
- */
- if ((data->fcdPosition == NULL || data->fcdPosition < data->pos) &&
- (nextch >= NFC_ZERO_CC_BLOCK_LIMIT_ ||
- ch >= NFC_ZERO_CC_BLOCK_LIMIT_)) {
- /*
- Need a more complete FCD check and possible normalization.
- normalize substring will be appended to buffer
- */
- if (collIterFCD(data)) {
- normalizeNextContraction(data);
- return *(data->pos ++);
- }
- else if (innormbuf) {
- /* fcdposition shifted even when there's no normalization, if we
- don't input the rest into this, we'll get the wrong position when
- we reach the end of the writableBuffer */
- int32_t length = (int32_t)(data->fcdPosition - data->pos + 1);
- data->pos = insertBufferEnd(data, data->pos - 1, length);
- // Check if data->pos received a null pointer
- if (data->pos == NULL) {
- return (UChar)-1; // Return to indicate error.
- }
- return *(data->pos ++);
- }
- }
-
- if (innormbuf) {
- /*
- no normalization is to be done hence only one character will be
- appended to the buffer.
- */
- data->pos = insertBufferEnd(data, ch) + 1;
- // Check if data->pos received a null pointer
- if (data->pos == NULL) {
- return (UChar)-1; // Return to indicate error.
- }
- }
-
- /* points back to the pos in string */
- return ch;
-}
-
-
-
-/**
-* Function to copy the buffer into writableBuffer and sets the fcd position to
-* the correct position
-* @param source data string source
-* @param buffer character buffer
-*/
-static
-inline void setDiscontiguosAttribute(collIterate *source, const UnicodeString &buffer)
-{
- /* okay confusing part here. to ensure that the skipped characters are
- considered later, we need to place it in the appropriate position in the
- normalization buffer and reassign the pos pointer. simple case if pos
- reside in string, simply copy to normalization buffer and
- fcdposition = pos, pos = start of normalization buffer. if pos in
- normalization buffer, we'll insert the copy infront of pos and point pos
- to the start of the normalization buffer. why am i doing these copies?
- well, so that the whole chunk of codes in the getNextCE, ucol_prv_getSpecialCE does
- not require any changes, which be really painful. */
- if (source->flags & UCOL_ITER_INNORMBUF) {
- int32_t replaceLength = source->pos - source->writableBuffer.getBuffer();
- source->writableBuffer.replace(0, replaceLength, buffer);
- }
- else {
- source->fcdPosition = source->pos;
- source->origFlags = source->flags;
- source->flags |= UCOL_ITER_INNORMBUF;
- source->flags &= ~(UCOL_ITER_NORM | UCOL_ITER_HASLEN | UCOL_USE_ITERATOR);
- source->writableBuffer = buffer;
- }
-
- source->pos = source->writableBuffer.getTerminatedBuffer();
-}
-
-/**
-* Function to get the discontiguos collation element within the source.
-* Note this function will set the position to the appropriate places.
-* @param coll current collator used
-* @param source data string source
-* @param constart index to the start character in the contraction table
-* @return discontiguos collation element offset
-*/
-static
-uint32_t getDiscontiguous(const UCollator *coll, collIterate *source,
- const UChar *constart)
-{
- /* source->pos currently points to the second combining character after
- the start character */
- const UChar *temppos = source->pos;
- UnicodeString buffer;
- const UChar *tempconstart = constart;
- uint8_t tempflags = source->flags;
- UBool multicontraction = FALSE;
- collIterateState discState;
-
- backupState(source, &discState);
-
- buffer.setTo(peekCodePoint(source, -1));
- for (;;) {
- UChar *UCharOffset;
- UChar schar,
- tchar;
- uint32_t result;
-
- if (((source->flags & UCOL_ITER_HASLEN) && source->pos >= source->endp)
- || (peekCodeUnit(source, 0) == 0 &&
- //|| (*source->pos == 0 &&
- ((source->flags & UCOL_ITER_INNORMBUF) == 0 ||
- source->fcdPosition == NULL ||
- source->fcdPosition == source->endp ||
- *(source->fcdPosition) == 0 ||
- u_getCombiningClass(*(source->fcdPosition)) == 0)) ||
- /* end of string in null terminated string or stopped by a
- null character, note fcd does not always point to a base
- character after the discontiguos change */
- u_getCombiningClass(peekCodePoint(source, 0)) == 0) {
- //u_getCombiningClass(*(source->pos)) == 0) {
- //constart = (UChar *)coll->image + getContractOffset(CE);
- if (multicontraction) {
- source->pos = temppos - 1;
- setDiscontiguosAttribute(source, buffer);
- return *(coll->contractionCEs +
- (tempconstart - coll->contractionIndex));
- }
- constart = tempconstart;
- break;
- }
-
- UCharOffset = (UChar *)(tempconstart + 1); /* skip the backward offset*/
- schar = getNextNormalizedChar(source);
-
- while (schar > (tchar = *UCharOffset)) {
- UCharOffset++;
- }
-
- if (schar != tchar) {
- /* not the correct codepoint. we stuff the current codepoint into
- the discontiguos buffer and try the next character */
- buffer.append(schar);
- continue;
- }
- else {
- if (u_getCombiningClass(schar) ==
- u_getCombiningClass(peekCodePoint(source, -2))) {
- buffer.append(schar);
- continue;
- }
- result = *(coll->contractionCEs +
- (UCharOffset - coll->contractionIndex));
- }
-
- if (result == UCOL_NOT_FOUND) {
- break;
- } else if (isContraction(result)) {
- /* this is a multi-contraction*/
- tempconstart = (UChar *)coll->image + getContractOffset(result);
- if (*(coll->contractionCEs + (constart - coll->contractionIndex))
- != UCOL_NOT_FOUND) {
- multicontraction = TRUE;
- temppos = source->pos + 1;
- }
- } else {
- setDiscontiguosAttribute(source, buffer);
- return result;
- }
- }
-
- /* no problems simply reverting just like that,
- if we are in string before getting into this function, points back to
- string hence no problem.
- if we are in normalization buffer before getting into this function,
- since we'll never use another normalization within this function, we
- know that fcdposition points to a base character. the normalization buffer
- never change, hence this revert works. */
- loadState(source, &discState, TRUE);
- goBackOne(source);
-
- //source->pos = temppos - 1;
- source->flags = tempflags;
- return *(coll->contractionCEs + (constart - coll->contractionIndex));
-}
-
-/* now uses Mark's getImplicitPrimary code */
-static
-inline uint32_t getImplicit(UChar32 cp, collIterate *collationSource) {
- uint32_t r = uprv_uca_getImplicitPrimary(cp);
- *(collationSource->CEpos++) = ((r & 0x0000FFFF)<<16) | 0x000000C0;
- collationSource->offsetRepeatCount += 1;
- return (r & UCOL_PRIMARYMASK) | 0x00000505; // This was 'order'
-}
-
-/**
-* Inserts the argument character into the front of the buffer replacing the
-* front null terminator.
-* @param data collation element iterator data
-* @param ch character to be appended
-*/
-static
-inline void insertBufferFront(collIterate *data, UChar ch)
-{
- data->pos = data->writableBuffer.setCharAt(0, ch).insert(0, (UChar)0).getTerminatedBuffer() + 2;
-}
-
-/**
-* Special normalization function for contraction in the previous iterator.
-* This normalization sequence will place the current character at source->pos
-* and its following normalized sequence into the buffer.
-* The fcd position, pos will be changed.
-* pos will now point to positions in the buffer.
-* Flags will be changed accordingly.
-* @param data collation iterator data
-*/
-static
-inline void normalizePrevContraction(collIterate *data, UErrorCode *status)
-{
- const UChar *pEnd = data->pos + 1; /* End normalize + 1 */
- const UChar *pStart;
-
- UnicodeString endOfBuffer;
- if (data->flags & UCOL_ITER_HASLEN) {
- /*
- normalization buffer not used yet, we'll pull down the next
- character into the end of the buffer
- */
- endOfBuffer.setTo(*pEnd);
- }
- else {
- endOfBuffer.setTo(data->writableBuffer, 1); // after the leading NUL
- }
-
- if (data->fcdPosition == NULL) {
- pStart = data->string;
- }
- else {
- pStart = data->fcdPosition + 1;
- }
- int32_t normLen =
- data->nfd->normalize(UnicodeString(FALSE, pStart, (int32_t)(pEnd - pStart)),
- data->writableBuffer,
- *status).
- length();
- if(U_FAILURE(*status)) {
- return;
- }
- /*
- this puts the null termination infront of the normalized string instead
- of the end
- */
- data->pos =
- data->writableBuffer.insert(0, (UChar)0).append(endOfBuffer).getTerminatedBuffer() +
- 1 + normLen;
- data->origFlags = data->flags;
- data->flags |= UCOL_ITER_INNORMBUF;
- data->flags &= ~(UCOL_ITER_NORM | UCOL_ITER_HASLEN);
-}
-
-/**
-* Contraction character management function that returns the previous character
-* for the backwards iterator.
-* Does nothing if the previous character is in buffer and not the first
-* character in it.
-* Else it checks previous character in data string to see if it is
-* normalizable.
-* If it is not, the character is simply copied into the buffer, else
-* the whole normalized substring is copied into the buffer, including the
-* current character.
-* @param data collation element iterator data
-* @return previous character
-*/
-static
-inline UChar getPrevNormalizedChar(collIterate *data, UErrorCode *status)
-{
- UChar prevch;
- UChar ch;
- const UChar *start;
- UBool innormbuf = (UBool)(data->flags & UCOL_ITER_INNORMBUF);
- if ((data->flags & (UCOL_ITER_NORM | UCOL_ITER_INNORMBUF)) == 0 ||
- (innormbuf && *(data->pos - 1) != 0)) {
- /*
- if no normalization.
- if previous character is in normalized buffer, no further normalization
- is required
- */
- if(data->flags & UCOL_USE_ITERATOR) {
- data->iterator->move(data->iterator, -1, UITER_CURRENT);
- return (UChar)data->iterator->next(data->iterator);
- } else {
- return *(data->pos - 1);
- }
- }
-
- start = data->pos;
- if ((data->fcdPosition==NULL)||(data->flags & UCOL_ITER_HASLEN)) {
- /* in data string */
- if ((start - 1) == data->string) {
- return *(start - 1);
- }
- start --;
- ch = *start;
- prevch = *(start - 1);
- }
- else {
- /*
- in writable buffer, at this point fcdPosition can not be NULL.
- see contracting tag.
- */
- if (data->fcdPosition == data->string) {
- /* at the start of the string, just dump it into the normalizer */
- insertBufferFront(data, *(data->fcdPosition));
- data->fcdPosition = NULL;
- return *(data->pos - 1);
- }
- start = data->fcdPosition;
- ch = *start;
- prevch = *(start - 1);
- }
- /*
- * if the current character is not fcd.
- * Trailing combining class == 0.
- */
- if (data->fcdPosition > start &&
- (ch >= NFC_ZERO_CC_BLOCK_LIMIT_ || prevch >= NFC_ZERO_CC_BLOCK_LIMIT_))
- {
- /*
- Need a more complete FCD check and possible normalization.
- normalize substring will be appended to buffer
- */
- const UChar *backuppos = data->pos;
- data->pos = start;
- if (collPrevIterFCD(data)) {
- normalizePrevContraction(data, status);
- return *(data->pos - 1);
- }
- data->pos = backuppos;
- data->fcdPosition ++;
- }
-
- if (innormbuf) {
- /*
- no normalization is to be done hence only one character will be
- appended to the buffer.
- */
- insertBufferFront(data, ch);
- data->fcdPosition --;
- }
-
- return ch;
-}
-
-/* This function handles the special CEs like contractions, expansions, surrogates, Thai */
-/* It is called by getNextCE */
-
-/* The following should be even */
-#define UCOL_MAX_DIGITS_FOR_NUMBER 254
-
-uint32_t ucol_prv_getSpecialCE(const UCollator *coll, UChar ch, uint32_t CE, collIterate *source, UErrorCode *status) {
- collIterateState entryState;
- backupState(source, &entryState);
- UChar32 cp = ch;
-
- for (;;) {
- // This loop will repeat only in the case of contractions, and only when a contraction
- // is found and the first CE resulting from that contraction is itself a special
- // (an expansion, for example.) All other special CE types are fully handled the
- // first time through, and the loop exits.
-
- const uint32_t *CEOffset = NULL;
- switch(getCETag(CE)) {
- case NOT_FOUND_TAG:
- /* This one is not found, and we'll let somebody else bother about it... no more games */
- return CE;
- case SPEC_PROC_TAG:
- {
- // Special processing is getting a CE that is preceded by a certain prefix
- // Currently this is only needed for optimizing Japanese length and iteration marks.
- // When we encouter a special processing tag, we go backwards and try to see if
- // we have a match.
- // Contraction tables are used - so the whole process is not unlike contraction.
- // prefix data is stored backwards in the table.
- const UChar *UCharOffset;
- UChar schar, tchar;
- collIterateState prefixState;
- backupState(source, &prefixState);
- loadState(source, &entryState, TRUE);
- goBackOne(source); // We want to look at the point where we entered - actually one
- // before that...
-
- for(;;) {
- // This loop will run once per source string character, for as long as we
- // are matching a potential contraction sequence
-
- // First we position ourselves at the begining of contraction sequence
- const UChar *ContractionStart = UCharOffset = (UChar *)coll->image+getContractOffset(CE);
- if (collIter_bos(source)) {
- CE = *(coll->contractionCEs + (UCharOffset - coll->contractionIndex));
- break;
- }
- schar = getPrevNormalizedChar(source, status);
- goBackOne(source);
-
- while(schar > (tchar = *UCharOffset)) { /* since the contraction codepoints should be ordered, we skip all that are smaller */
- UCharOffset++;
- }
-
- if (schar == tchar) {
- // Found the source string char in the table.
- // Pick up the corresponding CE from the table.
- CE = *(coll->contractionCEs +
- (UCharOffset - coll->contractionIndex));
- }
- else
- {
- // Source string char was not in the table.
- // We have not found the prefix.
- CE = *(coll->contractionCEs +
- (ContractionStart - coll->contractionIndex));
- }
-
- if(!isPrefix(CE)) {
- // The source string char was in the contraction table, and the corresponding
- // CE is not a prefix CE. We found the prefix, break
- // out of loop, this CE will end up being returned. This is the normal
- // way out of prefix handling when the source actually contained
- // the prefix.
- break;
- }
- }
- if(CE != UCOL_NOT_FOUND) { // we found something and we can merilly continue
- loadState(source, &prefixState, TRUE);
- if(source->origFlags & UCOL_USE_ITERATOR) {
- source->flags = source->origFlags;
- }
- } else { // prefix search was a failure, we have to backup all the way to the start
- loadState(source, &entryState, TRUE);
- }
- break;
- }
- case CONTRACTION_TAG:
- {
- /* This should handle contractions */
- collIterateState state;
- backupState(source, &state);
- uint32_t firstCE = *(coll->contractionCEs + ((UChar *)coll->image+getContractOffset(CE) - coll->contractionIndex)); //UCOL_NOT_FOUND;
- const UChar *UCharOffset;
- UChar schar, tchar;
-
- for (;;) {
- /* This loop will run once per source string character, for as long as we */
- /* are matching a potential contraction sequence */
-
- /* First we position ourselves at the begining of contraction sequence */
- const UChar *ContractionStart = UCharOffset = (UChar *)coll->image+getContractOffset(CE);
-
- if (collIter_eos(source)) {
- // Ran off the end of the source string.
- CE = *(coll->contractionCEs + (UCharOffset - coll->contractionIndex));
- // So we'll pick whatever we have at the point...
- if (CE == UCOL_NOT_FOUND) {
- // back up the source over all the chars we scanned going into this contraction.
- CE = firstCE;
- loadState(source, &state, TRUE);
- if(source->origFlags & UCOL_USE_ITERATOR) {
- source->flags = source->origFlags;
- }
- }
- break;
- }
-
- uint8_t maxCC = (uint8_t)(*(UCharOffset)&0xFF); /*get the discontiguos stuff */ /* skip the backward offset, see above */
- uint8_t allSame = (uint8_t)(*(UCharOffset++)>>8);
-
- schar = getNextNormalizedChar(source);
- while(schar > (tchar = *UCharOffset)) { /* since the contraction codepoints should be ordered, we skip all that are smaller */
- UCharOffset++;
- }
-
- if (schar == tchar) {
- // Found the source string char in the contraction table.
- // Pick up the corresponding CE from the table.
- CE = *(coll->contractionCEs +
- (UCharOffset - coll->contractionIndex));
- }
- else
- {
- // Source string char was not in contraction table.
- // Unless we have a discontiguous contraction, we have finished
- // with this contraction.
- // in order to do the proper detection, we
- // need to see if we're dealing with a supplementary
- /* We test whether the next two char are surrogate pairs.
- * This test is done if the iterator is not NULL.
- * If there is no surrogate pair, the iterator
- * goes back one if needed. */
- UChar32 miss = schar;
- if (source->iterator) {
- UChar32 surrNextChar; /* the next char in the iteration to test */
- int32_t prevPos; /* holds the previous position before move forward of the source iterator */
- if(U16_IS_LEAD(schar) && source->iterator->hasNext(source->iterator)) {
- prevPos = source->iterator->index;
- surrNextChar = getNextNormalizedChar(source);
- if (U16_IS_TRAIL(surrNextChar)) {
- miss = U16_GET_SUPPLEMENTARY(schar, surrNextChar);
- } else if (prevPos < source->iterator->index){
- goBackOne(source);
- }
- }
- } else if (U16_IS_LEAD(schar)) {
- miss = U16_GET_SUPPLEMENTARY(schar, getNextNormalizedChar(source));
- }
-
- uint8_t sCC;
- if (miss < 0x300 ||
- maxCC == 0 ||
- (sCC = i_getCombiningClass(miss, coll)) == 0 ||
- sCC>maxCC ||
- (allSame != 0 && sCC == maxCC) ||
- collIter_eos(source))
- {
- // Contraction can not be discontiguous.
- goBackOne(source); // back up the source string by one,
- // because the character we just looked at was
- // not part of the contraction. */
- if(U_IS_SUPPLEMENTARY(miss)) {
- goBackOne(source);
- }
- CE = *(coll->contractionCEs +
- (ContractionStart - coll->contractionIndex));
- } else {
- //
- // Contraction is possibly discontiguous.
- // Scan more of source string looking for a match
- //
- UChar tempchar;
- /* find the next character if schar is not a base character
- and we are not yet at the end of the string */
- tempchar = getNextNormalizedChar(source);
- // probably need another supplementary thingie here
- goBackOne(source);
- if (i_getCombiningClass(tempchar, coll) == 0) {
- goBackOne(source);
- if(U_IS_SUPPLEMENTARY(miss)) {
- goBackOne(source);
- }
- /* Spit out the last char of the string, wasn't tasty enough */
- CE = *(coll->contractionCEs +
- (ContractionStart - coll->contractionIndex));
- } else {
- CE = getDiscontiguous(coll, source, ContractionStart);
- }
- }
- } // else after if(schar == tchar)
-
- if(CE == UCOL_NOT_FOUND) {
- /* The Source string did not match the contraction that we were checking. */
- /* Back up the source position to undo the effects of having partially */
- /* scanned through what ultimately proved to not be a contraction. */
- loadState(source, &state, TRUE);
- CE = firstCE;
- break;
- }
-
- if(!isContraction(CE)) {
- // The source string char was in the contraction table, and the corresponding
- // CE is not a contraction CE. We completed the contraction, break
- // out of loop, this CE will end up being returned. This is the normal
- // way out of contraction handling when the source actually contained
- // the contraction.
- break;
- }
-
-
- // The source string char was in the contraction table, and the corresponding
- // CE is IS a contraction CE. We will continue looping to check the source
- // string for the remaining chars in the contraction.
- uint32_t tempCE = *(coll->contractionCEs + (ContractionStart - coll->contractionIndex));
- if(tempCE != UCOL_NOT_FOUND) {
- // We have scanned a a section of source string for which there is a
- // CE from the contraction table. Remember the CE and scan position, so
- // that we can return to this point if further scanning fails to
- // match a longer contraction sequence.
- firstCE = tempCE;
-
- goBackOne(source);
- backupState(source, &state);
- getNextNormalizedChar(source);
-
- // Another way to do this is:
- //collIterateState tempState;
- //backupState(source, &tempState);
- //goBackOne(source);
- //backupState(source, &state);
- //loadState(source, &tempState, TRUE);
-
- // The problem is that for incomplete contractions we have to remember the previous
- // position. Before, the only thing I needed to do was state.pos--;
- // After iterator introduction and especially after introduction of normalizing
- // iterators, it became much more difficult to decrease the saved state.
- // I'm not yet sure which of the two methods above is faster.
- }
- } // for(;;)
- break;
- } // case CONTRACTION_TAG:
- case LONG_PRIMARY_TAG:
- {
- *(source->CEpos++) = ((CE & 0xFF)<<24)|UCOL_CONTINUATION_MARKER;
- CE = ((CE & 0xFFFF00) << 8) | (UCOL_BYTE_COMMON << 8) | UCOL_BYTE_COMMON;
- source->offsetRepeatCount += 1;
- return CE;
- }
- case EXPANSION_TAG:
- {
- /* This should handle expansion. */
- /* NOTE: we can encounter both continuations and expansions in an expansion! */
- /* I have to decide where continuations are going to be dealt with */
- uint32_t size;
- uint32_t i; /* general counter */
-
- CEOffset = (uint32_t *)coll->image+getExpansionOffset(CE); /* find the offset to expansion table */
- size = getExpansionCount(CE);
- CE = *CEOffset++;
- //source->offsetRepeatCount = -1;
-
- if(size != 0) { /* if there are less than 16 elements in expansion, we don't terminate */
- for(i = 1; i<size; i++) {
- *(source->CEpos++) = *CEOffset++;
- source->offsetRepeatCount += 1;
- }
- } else { /* else, we do */
- while(*CEOffset != 0) {
- *(source->CEpos++) = *CEOffset++;
- source->offsetRepeatCount += 1;
- }
- }
-
- return CE;
- }
- case DIGIT_TAG:
- {
- /*
- We do a check to see if we want to collate digits as numbers; if so we generate
- a custom collation key. Otherwise we pull out the value stored in the expansion table.
- */
- //uint32_t size;
- uint32_t i; /* general counter */
-
- if (source->coll->numericCollation == UCOL_ON){
- collIterateState digitState = {0,0,0,0,0,0,0,0,0};
- UChar32 char32 = 0;
- int32_t digVal = 0;
-
- uint32_t digIndx = 0;
- uint32_t endIndex = 0;
- uint32_t trailingZeroIndex = 0;
-
- uint8_t collateVal = 0;
-
- UBool nonZeroValReached = FALSE;
-
- uint8_t numTempBuf[UCOL_MAX_DIGITS_FOR_NUMBER/2 + 3]; // I just need a temporary place to store my generated CEs.
- /*
- We parse the source string until we hit a char that's NOT a digit.
- Use this u_charDigitValue. This might be slow because we have to
- handle surrogates...
- */
- /*
- if (U16_IS_LEAD(ch)){
- if (!collIter_eos(source)) {
- backupState(source, &digitState);
- UChar trail = getNextNormalizedChar(source);
- if(U16_IS_TRAIL(trail)) {
- char32 = U16_GET_SUPPLEMENTARY(ch, trail);
- } else {
- loadState(source, &digitState, TRUE);
- char32 = ch;
- }
- } else {
- char32 = ch;
- }
- } else {
- char32 = ch;
- }
- digVal = u_charDigitValue(char32);
- */
- digVal = u_charDigitValue(cp); // if we have arrived here, we have
- // already processed possible supplementaries that trigered the digit tag -
- // all supplementaries are marked in the UCA.
- /*
- We pad a zero in front of the first element anyways. This takes
- care of the (probably) most common case where people are sorting things followed
- by a single digit
- */
- digIndx++;
- for(;;){
- // Make sure we have enough space. No longer needed;
- // at this point digIndx now has a max value of UCOL_MAX_DIGITS_FOR_NUMBER
- // (it has been pre-incremented) so we just ensure that numTempBuf is big enough
- // (UCOL_MAX_DIGITS_FOR_NUMBER/2 + 3).
-
- // Skipping over leading zeroes.
- if (digVal != 0) {
- nonZeroValReached = TRUE;
- }
- if (nonZeroValReached) {
- /*
- We parse the digit string into base 100 numbers (this fits into a byte).
- We only add to the buffer in twos, thus if we are parsing an odd character,
- that serves as the 'tens' digit while the if we are parsing an even one, that
- is the 'ones' digit. We dumped the parsed base 100 value (collateVal) into
- a buffer. We multiply each collateVal by 2 (to give us room) and add 5 (to avoid
- overlapping magic CE byte values). The last byte we subtract 1 to ensure it is less
- than all the other bytes.
- */
-
- if (digIndx % 2 == 1){
- collateVal += (uint8_t)digVal;
-
- // We don't enter the low-order-digit case unless we've already seen
- // the high order, or for the first digit, which is always non-zero.
- if (collateVal != 0)
- trailingZeroIndex = 0;
-
- numTempBuf[(digIndx/2) + 2] = collateVal*2 + 6;
- collateVal = 0;
- }
- else{
- // We drop the collation value into the buffer so if we need to do
- // a "front patch" we don't have to check to see if we're hitting the
- // last element.
- collateVal = (uint8_t)(digVal * 10);
-
- // Check for trailing zeroes.
- if (collateVal == 0)
- {
- if (!trailingZeroIndex)
- trailingZeroIndex = (digIndx/2) + 2;
- }
- else
- trailingZeroIndex = 0;
-
- numTempBuf[(digIndx/2) + 2] = collateVal*2 + 6;
- }
- digIndx++;
- }
-
- // Get next character.
- if (!collIter_eos(source)){
- ch = getNextNormalizedChar(source);
- if (U16_IS_LEAD(ch)){
- if (!collIter_eos(source)) {
- backupState(source, &digitState);
- UChar trail = getNextNormalizedChar(source);
- if(U16_IS_TRAIL(trail)) {
- char32 = U16_GET_SUPPLEMENTARY(ch, trail);
- } else {
- loadState(source, &digitState, TRUE);
- char32 = ch;
- }
- }
- } else {
- char32 = ch;
- }
-
- if ((digVal = u_charDigitValue(char32)) == -1 || digIndx > UCOL_MAX_DIGITS_FOR_NUMBER){
- // Resetting position to point to the next unprocessed char. We
- // overshot it when doing our test/set for numbers.
- if (char32 > 0xFFFF) { // For surrogates.
- loadState(source, &digitState, TRUE);
- //goBackOne(source);
- }
- goBackOne(source);
- break;
- }
- } else {
- break;
- }
- }
-
- if (nonZeroValReached == FALSE){
- digIndx = 2;
- numTempBuf[2] = 6;
- }
-
- endIndex = trailingZeroIndex ? trailingZeroIndex : ((digIndx/2) + 2) ;
- if (digIndx % 2 != 0){
- /*
- We missed a value. Since digIndx isn't even, stuck too many values into the buffer (this is what
- we get for padding the first byte with a zero). "Front-patch" now by pushing all nybbles forward.
- Doing it this way ensures that at least 50% of the time (statistically speaking) we'll only be doing a
- single pass and optimizes for strings with single digits. I'm just assuming that's the more common case.
- */
-
- for(i = 2; i < endIndex; i++){
- numTempBuf[i] = (((((numTempBuf[i] - 6)/2) % 10) * 10) +
- (((numTempBuf[i+1])-6)/2) / 10) * 2 + 6;
- }
- --digIndx;
- }
-
- // Subtract one off of the last byte.
- numTempBuf[endIndex-1] -= 1;
-
- /*
- We want to skip over the first two slots in the buffer. The first slot
- is reserved for the header byte UCOL_CODAN_PLACEHOLDER. The second slot is for the
- sign/exponent byte: 0x80 + (decimalPos/2) & 7f.
- */
- numTempBuf[0] = UCOL_CODAN_PLACEHOLDER;
- numTempBuf[1] = (uint8_t)(0x80 + ((digIndx/2) & 0x7F));
-
- // Now transfer the collation key to our collIterate struct.
- // The total size for our collation key is endIndx bumped up to the next largest even value divided by two.
- //size = ((endIndex+1) & ~1)/2;
- CE = (((numTempBuf[0] << 8) | numTempBuf[1]) << UCOL_PRIMARYORDERSHIFT) | //Primary weight
- (UCOL_BYTE_COMMON << UCOL_SECONDARYORDERSHIFT) | // Secondary weight
- UCOL_BYTE_COMMON; // Tertiary weight.
- i = 2; // Reset the index into the buffer.
- while(i < endIndex)
- {
- uint32_t primWeight = numTempBuf[i++] << 8;
- if ( i < endIndex)
- primWeight |= numTempBuf[i++];
- *(source->CEpos++) = (primWeight << UCOL_PRIMARYORDERSHIFT) | UCOL_CONTINUATION_MARKER;
- }
-
- } else {
- // no numeric mode, we'll just switch to whatever we stashed and continue
- CEOffset = (uint32_t *)coll->image+getExpansionOffset(CE); /* find the offset to expansion table */
- CE = *CEOffset++;
- break;
- }
- return CE;
- }
- /* various implicits optimization */
- case IMPLICIT_TAG: /* everything that is not defined otherwise */
- /* UCA is filled with these. Tailorings are NOT_FOUND */
- return getImplicit(cp, source);
- case CJK_IMPLICIT_TAG: /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/
- // TODO: remove CJK_IMPLICIT_TAG completely - handled by the getImplicit
- return getImplicit(cp, source);
- case HANGUL_SYLLABLE_TAG: /* AC00-D7AF*/
- {
- static const uint32_t
- SBase = 0xAC00, LBase = 0x1100, VBase = 0x1161, TBase = 0x11A7;
- //const uint32_t LCount = 19;
- static const uint32_t VCount = 21;
- static const uint32_t TCount = 28;
- //const uint32_t NCount = VCount * TCount; // 588
- //const uint32_t SCount = LCount * NCount; // 11172
- uint32_t L = ch - SBase;
-
- // divide into pieces
-
- uint32_t T = L % TCount; // we do it in this order since some compilers can do % and / in one operation
- L /= TCount;
- uint32_t V = L % VCount;
- L /= VCount;
-
- // offset them
-
- L += LBase;
- V += VBase;
- T += TBase;
-
- // return the first CE, but first put the rest into the expansion buffer
- if (!source->coll->image->jamoSpecial) { // FAST PATH
-
- *(source->CEpos++) = UTRIE_GET32_FROM_LEAD(&coll->mapping, V);
- if (T != TBase) {
- *(source->CEpos++) = UTRIE_GET32_FROM_LEAD(&coll->mapping, T);
- }
-
- return UTRIE_GET32_FROM_LEAD(&coll->mapping, L);
-
- } else { // Jamo is Special
- // Since Hanguls pass the FCD check, it is
- // guaranteed that we won't be in
- // the normalization buffer if something like this happens
-
- // However, if we are using a uchar iterator and normalization
- // is ON, the Hangul that lead us here is going to be in that
- // normalization buffer. Here we want to restore the uchar
- // iterator state and pull out of the normalization buffer
- if(source->iterator != NULL && source->flags & UCOL_ITER_INNORMBUF) {
- source->flags = source->origFlags; // restore the iterator
- source->pos = NULL;
- }
-
- // Move Jamos into normalization buffer
- UChar *buffer = source->writableBuffer.getBuffer(4);
- int32_t bufferLength;
- buffer[0] = (UChar)L;
- buffer[1] = (UChar)V;
- if (T != TBase) {
- buffer[2] = (UChar)T;
- bufferLength = 3;
- } else {
- bufferLength = 2;
- }
- source->writableBuffer.releaseBuffer(bufferLength);
-
- // Indicate where to continue in main input string after exhausting the writableBuffer
- source->fcdPosition = source->pos;
-
- source->pos = source->writableBuffer.getTerminatedBuffer();
- source->origFlags = source->flags;
- source->flags |= UCOL_ITER_INNORMBUF;
- source->flags &= ~(UCOL_ITER_NORM | UCOL_ITER_HASLEN);
-
- return(UCOL_IGNORABLE);
- }
- }
- case SURROGATE_TAG:
- /* we encountered a leading surrogate. We shall get the CE by using the following code unit */
- /* two things can happen here: next code point can be a trailing surrogate - we will use it */
- /* to retrieve the CE, or it is not a trailing surrogate (or the string is done). In that case */
- /* we treat it like an unassigned code point. */
- {
- UChar trail;
- collIterateState state;
- backupState(source, &state);
- if (collIter_eos(source) || !(U16_IS_TRAIL((trail = getNextNormalizedChar(source))))) {
- // we chould have stepped one char forward and it might have turned that it
- // was not a trail surrogate. In that case, we have to backup.
- loadState(source, &state, TRUE);
- return UCOL_NOT_FOUND;
- } else {
- /* TODO: CE contain the data from the previous CE + the mask. It should at least be unmasked */
- CE = UTRIE_GET32_FROM_OFFSET_TRAIL(&coll->mapping, CE&0xFFFFFF, trail);
- if(CE == UCOL_NOT_FOUND) { // there are tailored surrogates in this block, but not this one.
- // We need to backup
- loadState(source, &state, TRUE);
- return CE;
- }
- // calculate the supplementary code point value, if surrogate was not tailored
- cp = ((((uint32_t)ch)<<10UL)+(trail)-(((uint32_t)0xd800<<10UL)+0xdc00-0x10000));
- }
- }
- break;
- case LEAD_SURROGATE_TAG: /* D800-DBFF*/
- UChar nextChar;
- if( source->flags & UCOL_USE_ITERATOR) {
- if(U_IS_TRAIL(nextChar = (UChar)source->iterator->current(source->iterator))) {
- cp = U16_GET_SUPPLEMENTARY(ch, nextChar);
- source->iterator->next(source->iterator);
- return getImplicit(cp, source);
- }
- } else if((((source->flags & UCOL_ITER_HASLEN) == 0 ) || (source->pos<source->endp)) &&
- U_IS_TRAIL((nextChar=*source->pos))) {
- cp = U16_GET_SUPPLEMENTARY(ch, nextChar);
- source->pos++;
- return getImplicit(cp, source);
- }
- return UCOL_NOT_FOUND;
- case TRAIL_SURROGATE_TAG: /* DC00-DFFF*/
- return UCOL_NOT_FOUND; /* broken surrogate sequence */
- case CHARSET_TAG:
- /* not yet implemented */
- /* probably after 1.8 */
- return UCOL_NOT_FOUND;
- default:
- *status = U_INTERNAL_PROGRAM_ERROR;
- CE=0;
- break;
- }
- if (CE <= UCOL_NOT_FOUND) break;
- }
- return CE;
-}
-
-
-/* now uses Mark's getImplicitPrimary code */
-static
-inline uint32_t getPrevImplicit(UChar32 cp, collIterate *collationSource) {
- uint32_t r = uprv_uca_getImplicitPrimary(cp);
-
- *(collationSource->CEpos++) = (r & UCOL_PRIMARYMASK) | 0x00000505;
- collationSource->toReturn = collationSource->CEpos;
-
- // **** doesn't work if using iterator ****
- if (collationSource->flags & UCOL_ITER_INNORMBUF) {
- collationSource->offsetRepeatCount = 1;
- } else {
- int32_t firstOffset = (int32_t)(collationSource->pos - collationSource->string);
-
- UErrorCode errorCode = U_ZERO_ERROR;
- collationSource->appendOffset(firstOffset, errorCode);
- collationSource->appendOffset(firstOffset + 1, errorCode);
-
- collationSource->offsetReturn = collationSource->offsetStore - 1;
- *(collationSource->offsetBuffer) = firstOffset;
- if (collationSource->offsetReturn == collationSource->offsetBuffer) {
- collationSource->offsetStore = collationSource->offsetBuffer;
- }
- }
-
- return ((r & 0x0000FFFF)<<16) | 0x000000C0;
-}
-
-/**
- * This function handles the special CEs like contractions, expansions,
- * surrogates, Thai.
- * It is called by both getPrevCE
- */
-uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE,
- collIterate *source,
- UErrorCode *status)
-{
- const uint32_t *CEOffset = NULL;
- UChar *UCharOffset = NULL;
- UChar schar;
- const UChar *constart = NULL;
- uint32_t size;
- UChar buffer[UCOL_MAX_BUFFER];
- uint32_t *endCEBuffer;
- UChar *strbuffer;
- int32_t noChars = 0;
- int32_t CECount = 0;
-
- for(;;)
- {
- /* the only ces that loops are thai and contractions */
- switch (getCETag(CE))
- {
- case NOT_FOUND_TAG: /* this tag always returns */
- return CE;
-
- case SPEC_PROC_TAG:
- {
- // Special processing is getting a CE that is preceded by a certain prefix
- // Currently this is only needed for optimizing Japanese length and iteration marks.
- // When we encouter a special processing tag, we go backwards and try to see if
- // we have a match.
- // Contraction tables are used - so the whole process is not unlike contraction.
- // prefix data is stored backwards in the table.
- const UChar *UCharOffset;
- UChar schar, tchar;
- collIterateState prefixState;
- backupState(source, &prefixState);
- for(;;) {
- // This loop will run once per source string character, for as long as we
- // are matching a potential contraction sequence
-
- // First we position ourselves at the begining of contraction sequence
- const UChar *ContractionStart = UCharOffset = (UChar *)coll->image+getContractOffset(CE);
-
- if (collIter_bos(source)) {
- CE = *(coll->contractionCEs + (UCharOffset - coll->contractionIndex));
- break;
- }
- schar = getPrevNormalizedChar(source, status);
- goBackOne(source);
-
- while(schar > (tchar = *UCharOffset)) { /* since the contraction codepoints should be ordered, we skip all that are smaller */
- UCharOffset++;
- }
-
- if (schar == tchar) {
- // Found the source string char in the table.
- // Pick up the corresponding CE from the table.
- CE = *(coll->contractionCEs +
- (UCharOffset - coll->contractionIndex));
- }
- else
- {
- // if there is a completely ignorable code point in the middle of
- // a prefix, we need to act as if it's not there
- // assumption: 'real' noncharacters (*fffe, *ffff, fdd0-fdef are set to zero)
- // lone surrogates cannot be set to zero as it would break other processing
- uint32_t isZeroCE = UTRIE_GET32_FROM_LEAD(&coll->mapping, schar);
- // it's easy for BMP code points
- if(isZeroCE == 0) {
- continue;
- } else if(U16_IS_SURROGATE(schar)) {
- // for supplementary code points, we have to check the next one
- // situations where we are going to ignore
- // 1. beginning of the string: schar is a lone surrogate
- // 2. schar is a lone surrogate
- // 3. schar is a trail surrogate in a valid surrogate sequence
- // that is explicitly set to zero.
- if (!collIter_bos(source)) {
- UChar lead;
- if(!U16_IS_SURROGATE_LEAD(schar) && U16_IS_LEAD(lead = getPrevNormalizedChar(source, status))) {
- isZeroCE = UTRIE_GET32_FROM_LEAD(&coll->mapping, lead);
- if(isSpecial(isZeroCE) && getCETag(isZeroCE) == SURROGATE_TAG) {
- uint32_t finalCE = UTRIE_GET32_FROM_OFFSET_TRAIL(&coll->mapping, isZeroCE&0xFFFFFF, schar);
- if(finalCE == 0) {
- // this is a real, assigned completely ignorable code point
- goBackOne(source);
- continue;
- }
- }
- } else {
- // lone surrogate, treat like unassigned
- return UCOL_NOT_FOUND;
- }
- } else {
- // lone surrogate at the beggining, treat like unassigned
- return UCOL_NOT_FOUND;
- }
- }
- // Source string char was not in the table.
- // We have not found the prefix.
- CE = *(coll->contractionCEs +
- (ContractionStart - coll->contractionIndex));
- }
-
- if(!isPrefix(CE)) {
- // The source string char was in the contraction table, and the corresponding
- // CE is not a prefix CE. We found the prefix, break
- // out of loop, this CE will end up being returned. This is the normal
- // way out of prefix handling when the source actually contained
- // the prefix.
- break;
- }
- }
- loadState(source, &prefixState, TRUE);
- break;
- }
-
- case CONTRACTION_TAG: {
- /* to ensure that the backwards and forwards iteration matches, we
- take the current region of most possible match and pass it through
- the forward iteration. this will ensure that the obstinate problem of
- overlapping contractions will not occur.
- */
- schar = peekCodeUnit(source, 0);
- constart = (UChar *)coll->image + getContractOffset(CE);
- if (isAtStartPrevIterate(source)
- /* commented away contraction end checks after adding the checks
- in getPrevCE */) {
- /* start of string or this is not the end of any contraction */
- CE = *(coll->contractionCEs +
- (constart - coll->contractionIndex));
- break;
- }
- strbuffer = buffer;
- UCharOffset = strbuffer + (UCOL_MAX_BUFFER - 1);
- *(UCharOffset --) = 0;
- noChars = 0;
- // have to swap thai characters
- while (ucol_unsafeCP(schar, coll)) {
- *(UCharOffset) = schar;
- noChars++;
- UCharOffset --;
- schar = getPrevNormalizedChar(source, status);
- goBackOne(source);
- // TODO: when we exhaust the contraction buffer,
- // it needs to get reallocated. The problem is
- // that the size depends on the string which is
- // not iterated over. However, since we're travelling
- // backwards, we already had to set the iterator at
- // the end - so we might as well know where we are?
- if (UCharOffset + 1 == buffer) {
- /* we have exhausted the buffer */
- int32_t newsize = 0;
- if(source->pos) { // actually dealing with a position
- newsize = (int32_t)(source->pos - source->string + 1);
- } else { // iterator
- newsize = 4 * UCOL_MAX_BUFFER;
- }
- strbuffer = (UChar *)uprv_malloc(sizeof(UChar) *
- (newsize + UCOL_MAX_BUFFER));
- /* test for NULL */
- if (strbuffer == NULL) {
- *status = U_MEMORY_ALLOCATION_ERROR;
- return UCOL_NO_MORE_CES;
- }
- UCharOffset = strbuffer + newsize;
- uprv_memcpy(UCharOffset, buffer,
- UCOL_MAX_BUFFER * sizeof(UChar));
- UCharOffset --;
- }
- if ((source->pos && (source->pos == source->string ||
- ((source->flags & UCOL_ITER_INNORMBUF) &&
- *(source->pos - 1) == 0 && source->fcdPosition == NULL)))
- || (source->iterator && !source->iterator->hasPrevious(source->iterator))) {
- break;
- }
- }
- /* adds the initial base character to the string */
- *(UCharOffset) = schar;
- noChars++;
-
- int32_t offsetBias;
-
- // **** doesn't work if using iterator ****
- if (source->flags & UCOL_ITER_INNORMBUF) {
- offsetBias = -1;
- } else {
- offsetBias = (int32_t)(source->pos - source->string);
- }
-
- /* a new collIterate is used to simplify things, since using the current
- collIterate will mean that the forward and backwards iteration will
- share and change the same buffers. we don't want to get into that. */
- collIterate temp;
- int32_t rawOffset;
-
- IInit_collIterate(coll, UCharOffset, noChars, &temp, status);
- if(U_FAILURE(*status)) {
- return UCOL_NULLORDER;
- }
- temp.flags &= ~UCOL_ITER_NORM;
- temp.flags |= source->flags & UCOL_FORCE_HAN_IMPLICIT;
-
- rawOffset = (int32_t)(temp.pos - temp.string); // should always be zero?
- CE = ucol_IGetNextCE(coll, &temp, status);
-
- if (source->extendCEs) {
- endCEBuffer = source->extendCEs + source->extendCEsSize;
- CECount = (int32_t)((source->CEpos - source->extendCEs)/sizeof(uint32_t));
- } else {
- endCEBuffer = source->CEs + UCOL_EXPAND_CE_BUFFER_SIZE;
- CECount = (int32_t)((source->CEpos - source->CEs)/sizeof(uint32_t));
- }
-
- while (CE != UCOL_NO_MORE_CES) {
- *(source->CEpos ++) = CE;
-
- if (offsetBias >= 0) {
- source->appendOffset(rawOffset + offsetBias, *status);
- }
-
- CECount++;
- if (source->CEpos == endCEBuffer) {
- /* ran out of CE space, reallocate to new buffer.
- If reallocation fails, reset pointers and bail out,
- there's no guarantee of the right character position after
- this bail*/
- if (!increaseCEsCapacity(source)) {
- *status = U_MEMORY_ALLOCATION_ERROR;
- break;
- }
-
- endCEBuffer = source->extendCEs + source->extendCEsSize;
- }
-
- if ((temp.flags & UCOL_ITER_INNORMBUF) != 0) {
- rawOffset = (int32_t)(temp.fcdPosition - temp.string);
- } else {
- rawOffset = (int32_t)(temp.pos - temp.string);
- }
-
- CE = ucol_IGetNextCE(coll, &temp, status);
- }
-
- if (strbuffer != buffer) {
- uprv_free(strbuffer);
- }
- if (U_FAILURE(*status)) {
- return (uint32_t)UCOL_NULLORDER;
- }
-
- if (source->offsetRepeatValue != 0) {
- if (CECount > noChars) {
- source->offsetRepeatCount += temp.offsetRepeatCount;
- } else {
- // **** does this really skip the right offsets? ****
- source->offsetReturn -= (noChars - CECount);
- }
- }
-
- if (offsetBias >= 0) {
- source->offsetReturn = source->offsetStore - 1;
- if (source->offsetReturn == source->offsetBuffer) {
- source->offsetStore = source->offsetBuffer;
- }
- }
-
- source->toReturn = source->CEpos - 1;
- if (source->toReturn == source->CEs) {
- source->CEpos = source->CEs;
- }
-
- return *(source->toReturn);
- }
- case LONG_PRIMARY_TAG:
- {
- *(source->CEpos++) = ((CE & 0xFFFF00) << 8) | (UCOL_BYTE_COMMON << 8) | UCOL_BYTE_COMMON;
- *(source->CEpos++) = ((CE & 0xFF)<<24)|UCOL_CONTINUATION_MARKER;
- source->toReturn = source->CEpos - 1;
-
- if (source->flags & UCOL_ITER_INNORMBUF) {
- source->offsetRepeatCount = 1;
- } else {
- int32_t firstOffset = (int32_t)(source->pos - source->string);
-
- source->appendOffset(firstOffset, *status);
- source->appendOffset(firstOffset + 1, *status);
-
- source->offsetReturn = source->offsetStore - 1;
- *(source->offsetBuffer) = firstOffset;
- if (source->offsetReturn == source->offsetBuffer) {
- source->offsetStore = source->offsetBuffer;
- }
- }
-
-
- return *(source->toReturn);
- }
-
- case EXPANSION_TAG: /* this tag always returns */
- {
- /*
- This should handle expansion.
- NOTE: we can encounter both continuations and expansions in an expansion!
- I have to decide where continuations are going to be dealt with
- */
- int32_t firstOffset = (int32_t)(source->pos - source->string);
-
- // **** doesn't work if using iterator ****
- if (source->offsetReturn != NULL) {
- if (! (source->flags & UCOL_ITER_INNORMBUF) && source->offsetReturn == source->offsetBuffer) {
- source->offsetStore = source->offsetBuffer;
- }else {
- firstOffset = -1;
- }
- }
-
- /* find the offset to expansion table */
- CEOffset = (uint32_t *)coll->image + getExpansionOffset(CE);
- size = getExpansionCount(CE);
- if (size != 0) {
- /*
- if there are less than 16 elements in expansion, we don't terminate
- */
- uint32_t count;
-
- for (count = 0; count < size; count++) {
- *(source->CEpos ++) = *CEOffset++;
-
- if (firstOffset >= 0) {
- source->appendOffset(firstOffset + 1, *status);
- }
- }
- } else {
- /* else, we do */
- while (*CEOffset != 0) {
- *(source->CEpos ++) = *CEOffset ++;
-
- if (firstOffset >= 0) {
- source->appendOffset(firstOffset + 1, *status);
- }
- }
- }
-
- if (firstOffset >= 0) {
- source->offsetReturn = source->offsetStore - 1;
- *(source->offsetBuffer) = firstOffset;
- if (source->offsetReturn == source->offsetBuffer) {
- source->offsetStore = source->offsetBuffer;
- }
- } else {
- source->offsetRepeatCount += size - 1;
- }
-
- source->toReturn = source->CEpos - 1;
- // in case of one element expansion, we
- // want to immediately return CEpos
- if(source->toReturn == source->CEs) {
- source->CEpos = source->CEs;
- }
-
- return *(source->toReturn);
- }
-
- case DIGIT_TAG:
- {
- /*
- We do a check to see if we want to collate digits as numbers; if so we generate
- a custom collation key. Otherwise we pull out the value stored in the expansion table.
- */
- uint32_t i; /* general counter */
-
- if (source->coll->numericCollation == UCOL_ON){
- uint32_t digIndx = 0;
- uint32_t endIndex = 0;
- uint32_t leadingZeroIndex = 0;
- uint32_t trailingZeroCount = 0;
-
- uint8_t collateVal = 0;
-
- UBool nonZeroValReached = FALSE;
-
- uint8_t numTempBuf[UCOL_MAX_DIGITS_FOR_NUMBER/2 + 2]; // I just need a temporary place to store my generated CEs.
- /*
- We parse the source string until we hit a char that's NOT a digit.
- Use this u_charDigitValue. This might be slow because we have to
- handle surrogates...
- */
- /*
- We need to break up the digit string into collection elements of UCOL_MAX_DIGITS_FOR_NUMBER or less,
- with any chunks smaller than that being on the right end of the digit string - i.e. the first collation
- element we process when going backward. To determine how long that chunk might be, we may need to make
- two passes through the loop that collects digits - one to see how long the string is (and how much is
- leading zeros) to determine the length of that right-hand chunk, and a second (if the whole string has
- more than UCOL_MAX_DIGITS_FOR_NUMBER non-leading-zero digits) to actually process that collation
- element chunk after resetting the state to the initialState at the right side of the digit string.
- */
- uint32_t ceLimit = 0;
- UChar initial_ch = ch;
- collIterateState initialState = {0,0,0,0,0,0,0,0,0};
- backupState(source, &initialState);
-
- for(;;) {
- collIterateState state = {0,0,0,0,0,0,0,0,0};
- UChar32 char32 = 0;
- int32_t digVal = 0;
-
- if (U16_IS_TRAIL (ch)) {
- if (!collIter_bos(source)){
- UChar lead = getPrevNormalizedChar(source, status);
- if(U16_IS_LEAD(lead)) {
- char32 = U16_GET_SUPPLEMENTARY(lead,ch);
- goBackOne(source);
- } else {
- char32 = ch;
- }
- } else {
- char32 = ch;
- }
- } else {
- char32 = ch;
- }
- digVal = u_charDigitValue(char32);
-
- for(;;) {
- // Make sure we have enough space. No longer needed;
- // at this point the largest value of digIndx when we need to save data in numTempBuf
- // is UCOL_MAX_DIGITS_FOR_NUMBER-1 (digIndx is post-incremented) so we just ensure
- // that numTempBuf is big enough (UCOL_MAX_DIGITS_FOR_NUMBER/2 + 2).
-
- // Skip over trailing zeroes, and keep a count of them.
- if (digVal != 0)
- nonZeroValReached = TRUE;
-
- if (nonZeroValReached) {
- /*
- We parse the digit string into base 100 numbers (this fits into a byte).
- We only add to the buffer in twos, thus if we are parsing an odd character,
- that serves as the 'tens' digit while the if we are parsing an even one, that
- is the 'ones' digit. We dumped the parsed base 100 value (collateVal) into
- a buffer. We multiply each collateVal by 2 (to give us room) and add 5 (to avoid
- overlapping magic CE byte values). The last byte we subtract 1 to ensure it is less
- than all the other bytes.
-
- Since we're doing in this reverse we want to put the first digit encountered into the
- ones place and the second digit encountered into the tens place.
- */
-
- if ((digIndx + trailingZeroCount) % 2 == 1) {
- // High-order digit case (tens place)
- collateVal += (uint8_t)(digVal * 10);
-
- // We cannot set leadingZeroIndex unless it has been set for the
- // low-order digit. Therefore, all we can do for the high-order
- // digit is turn it off, never on.
- // The only time we will have a high digit without a low is for
- // the very first non-zero digit, so no zero check is necessary.
- if (collateVal != 0)
- leadingZeroIndex = 0;
-
- // The first pass through, digIndx may exceed the limit, but in that case
- // we no longer care about numTempBuf contents since they will be discarded
- if ( digIndx < UCOL_MAX_DIGITS_FOR_NUMBER ) {
- numTempBuf[(digIndx/2) + 2] = collateVal*2 + 6;
- }
- collateVal = 0;
- } else {
- // Low-order digit case (ones place)
- collateVal = (uint8_t)digVal;
-
- // Check for leading zeroes.
- if (collateVal == 0) {
- if (!leadingZeroIndex)
- leadingZeroIndex = (digIndx/2) + 2;
- } else
- leadingZeroIndex = 0;
-
- // No need to write to buffer; the case of a last odd digit
- // is handled below.
- }
- ++digIndx;
- } else
- ++trailingZeroCount;
-
- if (!collIter_bos(source)) {
- ch = getPrevNormalizedChar(source, status);
- //goBackOne(source);
- if (U16_IS_TRAIL(ch)) {
- backupState(source, &state);
- if (!collIter_bos(source)) {
- goBackOne(source);
- UChar lead = getPrevNormalizedChar(source, status);
-
- if(U16_IS_LEAD(lead)) {
- char32 = U16_GET_SUPPLEMENTARY(lead,ch);
- } else {
- loadState(source, &state, FALSE);
- char32 = ch;
- }
- }
- } else
- char32 = ch;
-
- if ((digVal = u_charDigitValue(char32)) == -1 || (ceLimit > 0 && (digIndx + trailingZeroCount) >= ceLimit)) {
- if (char32 > 0xFFFF) {// For surrogates.
- loadState(source, &state, FALSE);
- }
- // Don't need to "reverse" the goBackOne call,
- // as this points to the next position to process..
- //if (char32 > 0xFFFF) // For surrogates.
- //getNextNormalizedChar(source);
- break;
- }
-
- goBackOne(source);
- }else
- break;
- }
-
- if (digIndx + trailingZeroCount <= UCOL_MAX_DIGITS_FOR_NUMBER) {
- // our collation element is not too big, go ahead and finish with it
- break;
- }
- // our digit string is too long for a collation element;
- // set the limit for it, reset the state and begin again
- ceLimit = (digIndx + trailingZeroCount) % UCOL_MAX_DIGITS_FOR_NUMBER;
- if ( ceLimit == 0 ) {
- ceLimit = UCOL_MAX_DIGITS_FOR_NUMBER;
- }
- ch = initial_ch;
- loadState(source, &initialState, FALSE);
- digIndx = endIndex = leadingZeroIndex = trailingZeroCount = 0;
- collateVal = 0;
- nonZeroValReached = FALSE;
- }
-
- if (! nonZeroValReached) {
- digIndx = 2;
- trailingZeroCount = 0;
- numTempBuf[2] = 6;
- }
-
- if ((digIndx + trailingZeroCount) % 2 != 0) {
- numTempBuf[((digIndx)/2) + 2] = collateVal*2 + 6;
- digIndx += 1; // The implicit leading zero
- }
- if (trailingZeroCount % 2 != 0) {
- // We had to consume one trailing zero for the low digit
- // of the least significant byte
- digIndx += 1; // The trailing zero not in the exponent
- trailingZeroCount -= 1;
- }
-
- endIndex = leadingZeroIndex ? leadingZeroIndex : ((digIndx/2) + 2) ;
-
- // Subtract one off of the last byte. Really the first byte here, but it's reversed...
- numTempBuf[2] -= 1;
-
- /*
- We want to skip over the first two slots in the buffer. The first slot
- is reserved for the header byte UCOL_CODAN_PLACEHOLDER. The second slot is for the
- sign/exponent byte: 0x80 + (decimalPos/2) & 7f.
- The exponent must be adjusted by the number of leading zeroes, and the number of
- trailing zeroes.
- */
- numTempBuf[0] = UCOL_CODAN_PLACEHOLDER;
- uint32_t exponent = (digIndx+trailingZeroCount)/2;
- if (leadingZeroIndex)
- exponent -= ((digIndx/2) + 2 - leadingZeroIndex);
- numTempBuf[1] = (uint8_t)(0x80 + (exponent & 0x7F));
-
- // Now transfer the collation key to our collIterate struct.
- // The total size for our collation key is half of endIndex, rounded up.
- int32_t size = (endIndex+1)/2;
- if(!ensureCEsCapacity(source, size)) {
- return UCOL_NULLORDER;
- }
- *(source->CEpos++) = (((numTempBuf[0] << 8) | numTempBuf[1]) << UCOL_PRIMARYORDERSHIFT) | //Primary weight
- (UCOL_BYTE_COMMON << UCOL_SECONDARYORDERSHIFT) | // Secondary weight
- UCOL_BYTE_COMMON; // Tertiary weight.
- i = endIndex - 1; // Reset the index into the buffer.
- while(i >= 2) {
- uint32_t primWeight = numTempBuf[i--] << 8;
- if ( i >= 2)
- primWeight |= numTempBuf[i--];
- *(source->CEpos++) = (primWeight << UCOL_PRIMARYORDERSHIFT) | UCOL_CONTINUATION_MARKER;
- }
-
- source->toReturn = source->CEpos -1;
- return *(source->toReturn);
- } else {
- CEOffset = (uint32_t *)coll->image + getExpansionOffset(CE);
- CE = *(CEOffset++);
- break;
- }
- }
-
- case HANGUL_SYLLABLE_TAG: /* AC00-D7AF*/
- {
- static const uint32_t
- SBase = 0xAC00, LBase = 0x1100, VBase = 0x1161, TBase = 0x11A7;
- //const uint32_t LCount = 19;
- static const uint32_t VCount = 21;
- static const uint32_t TCount = 28;
- //const uint32_t NCount = VCount * TCount; /* 588 */
- //const uint32_t SCount = LCount * NCount; /* 11172 */
-
- uint32_t L = ch - SBase;
- /*
- divide into pieces.
- we do it in this order since some compilers can do % and / in one
- operation
- */
- uint32_t T = L % TCount;
- L /= TCount;
- uint32_t V = L % VCount;
- L /= VCount;
-
- /* offset them */
- L += LBase;
- V += VBase;
- T += TBase;
-
- int32_t firstOffset = (int32_t)(source->pos - source->string);
- source->appendOffset(firstOffset, *status);
-
- /*
- * return the first CE, but first put the rest into the expansion buffer
- */
- if (!source->coll->image->jamoSpecial) {
- *(source->CEpos++) = UTRIE_GET32_FROM_LEAD(&coll->mapping, L);
- *(source->CEpos++) = UTRIE_GET32_FROM_LEAD(&coll->mapping, V);
- source->appendOffset(firstOffset + 1, *status);
-
- if (T != TBase) {
- *(source->CEpos++) = UTRIE_GET32_FROM_LEAD(&coll->mapping, T);
- source->appendOffset(firstOffset + 1, *status);
- }
-
- source->toReturn = source->CEpos - 1;
-
- source->offsetReturn = source->offsetStore - 1;
- if (source->offsetReturn == source->offsetBuffer) {
- source->offsetStore = source->offsetBuffer;
- }
-
- return *(source->toReturn);
- } else {
- // Since Hanguls pass the FCD check, it is
- // guaranteed that we won't be in
- // the normalization buffer if something like this happens
-
- // Move Jamos into normalization buffer
- UChar *tempbuffer = source->writableBuffer.getBuffer(5);
- int32_t tempbufferLength, jamoOffset;
- tempbuffer[0] = 0;
- tempbuffer[1] = (UChar)L;
- tempbuffer[2] = (UChar)V;
- if (T != TBase) {
- tempbuffer[3] = (UChar)T;
- tempbufferLength = 4;
- } else {
- tempbufferLength = 3;
- }
- source->writableBuffer.releaseBuffer(tempbufferLength);
-
- // Indicate where to continue in main input string after exhausting the writableBuffer
- if (source->pos == source->string) {
- jamoOffset = 0;
- source->fcdPosition = NULL;
- } else {
- jamoOffset = source->pos - source->string;
- source->fcdPosition = source->pos-1;
- }
-
- // Append offsets for the additional chars
- // (not the 0, and not the L whose offsets match the original Hangul)
- int32_t jamoRemaining = tempbufferLength - 2;
- jamoOffset++; // appended offsets should match end of original Hangul
- while (jamoRemaining-- > 0) {
- source->appendOffset(jamoOffset, *status);
- }
-
- source->offsetRepeatValue = jamoOffset;
-
- source->offsetReturn = source->offsetStore - 1;
- if (source->offsetReturn == source->offsetBuffer) {
- source->offsetStore = source->offsetBuffer;
- }
-
- source->pos = source->writableBuffer.getTerminatedBuffer() + tempbufferLength;
- source->origFlags = source->flags;
- source->flags |= UCOL_ITER_INNORMBUF;
- source->flags &= ~(UCOL_ITER_NORM | UCOL_ITER_HASLEN);
-
- return(UCOL_IGNORABLE);
- }
- }
-
- case IMPLICIT_TAG: /* everything that is not defined otherwise */
- return getPrevImplicit(ch, source);
-
- // TODO: Remove CJK implicits as they are handled by the getImplicitPrimary function
- case CJK_IMPLICIT_TAG: /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/
- return getPrevImplicit(ch, source);
-
- case SURROGATE_TAG: /* This is a surrogate pair */
- /* essentially an engaged lead surrogate. */
- /* if you have encountered it here, it means that a */
- /* broken sequence was encountered and this is an error */
- return UCOL_NOT_FOUND;
-
- case LEAD_SURROGATE_TAG: /* D800-DBFF*/
- return UCOL_NOT_FOUND; /* broken surrogate sequence */
-
- case TRAIL_SURROGATE_TAG: /* DC00-DFFF*/
- {
- UChar32 cp = 0;
- UChar prevChar;
- const UChar *prev;
- if (isAtStartPrevIterate(source)) {
- /* we are at the start of the string, wrong place to be at */
- return UCOL_NOT_FOUND;
- }
- if (source->pos != source->writableBuffer.getBuffer()) {
- prev = source->pos - 1;
- } else {
- prev = source->fcdPosition;
- }
- prevChar = *prev;
-
- /* Handles Han and Supplementary characters here.*/
- if (U16_IS_LEAD(prevChar)) {
- cp = ((((uint32_t)prevChar)<<10UL)+(ch)-(((uint32_t)0xd800<<10UL)+0xdc00-0x10000));
- source->pos = prev;
- } else {
- return UCOL_NOT_FOUND; /* like unassigned */
- }
-
- return getPrevImplicit(cp, source);
- }
-
- /* UCA is filled with these. Tailorings are NOT_FOUND */
- /* not yet implemented */
- case CHARSET_TAG: /* this tag always returns */
- /* probably after 1.8 */
- return UCOL_NOT_FOUND;
-
- default: /* this tag always returns */
- *status = U_INTERNAL_PROGRAM_ERROR;
- CE=0;
- break;
- }
-
- if (CE <= UCOL_NOT_FOUND) {
- break;
- }
- }
-
- return CE;
-}
-
-/* This should really be a macro */
-/* This function is used to reverse parts of a buffer. We need this operation when doing continuation */
-/* secondaries in French */
-/*
-void uprv_ucol_reverse_buffer(uint8_t *start, uint8_t *end) {
- uint8_t temp;
- while(start<end) {
- temp = *start;
- *start++ = *end;
- *end-- = temp;
- }
-}
-*/
-
-#define uprv_ucol_reverse_buffer(TYPE, start, end) { \
- TYPE tempA; \
-while((start)<(end)) { \
- tempA = *(start); \
- *(start)++ = *(end); \
- *(end)-- = tempA; \
-} \
-}
-
-/****************************************************************************/
-/* Following are the sortkey generation functions */
-/* */
-/****************************************************************************/
-
-/**
- * Merge two sort keys.
- * This is useful, for example, to combine sort keys from first and last names
- * to sort such pairs.
- * Merged sort keys consider on each collation level the first part first entirely,
- * then the second one.
- * It is possible to merge multiple sort keys by consecutively merging
- * another one with the intermediate result.
- *
- * The length of the merge result is the sum of the lengths of the input sort keys
- * minus 1.
- *
- * @param src1 the first sort key
- * @param src1Length the length of the first sort key, including the zero byte at the end;
- * can be -1 if the function is to find the length
- * @param src2 the second sort key
- * @param src2Length the length of the second sort key, including the zero byte at the end;
- * can be -1 if the function is to find the length
- * @param dest the buffer where the merged sort key is written,
- * can be NULL if destCapacity==0
- * @param destCapacity the number of bytes in the dest buffer
- * @return the length of the merged sort key, src1Length+src2Length-1;
- * can be larger than destCapacity, or 0 if an error occurs (only for illegal arguments),
- * in which cases the contents of dest is undefined
- *
- * @draft
- */
-U_CAPI int32_t U_EXPORT2
-ucol_mergeSortkeys(const uint8_t *src1, int32_t src1Length,
- const uint8_t *src2, int32_t src2Length,
- uint8_t *dest, int32_t destCapacity) {
- int32_t destLength;
- uint8_t b;
-
- /* check arguments */
- if( src1==NULL || src1Length<-2 || src1Length==0 || (src1Length>0 && src1[src1Length-1]!=0) ||
- src2==NULL || src2Length<-2 || src2Length==0 || (src2Length>0 && src2[src2Length-1]!=0) ||
- destCapacity<0 || (destCapacity>0 && dest==NULL)
- ) {
- /* error, attempt to write a zero byte and return 0 */
- if(dest!=NULL && destCapacity>0) {
- *dest=0;
- }
- return 0;
- }
-
- /* check lengths and capacity */
- if(src1Length<0) {
- src1Length=(int32_t)uprv_strlen((const char *)src1)+1;
- }
- if(src2Length<0) {
- src2Length=(int32_t)uprv_strlen((const char *)src2)+1;
- }
-
- destLength=src1Length+src2Length-1;
- if(destLength>destCapacity) {
- /* the merged sort key does not fit into the destination */
- return destLength;
- }
-
- /* merge the sort keys with the same number of levels */
- while(*src1!=0 && *src2!=0) { /* while both have another level */
- /* copy level from src1 not including 00 or 01 */
- while((b=*src1)>=2) {
- ++src1;
- *dest++=b;
- }
-
- /* add a 02 merge separator */
- *dest++=2;
-
- /* copy level from src2 not including 00 or 01 */
- while((b=*src2)>=2) {
- ++src2;
- *dest++=b;
- }
-
- /* if both sort keys have another level, then add a 01 level separator and continue */
- if(*src1==1 && *src2==1) {
- ++src1;
- ++src2;
- *dest++=1;
- }
- }
-
- /*
- * here, at least one sort key is finished now, but the other one
- * might have some contents left from containing more levels;
- * that contents is just appended to the result
- */
- if(*src1!=0) {
- /* src1 is not finished, therefore *src2==0, and src1 is appended */
- src2=src1;
- }
- /* append src2, "the other, unfinished sort key" */
- uprv_strcpy((char *)dest, (const char *)src2);
-
- /* trust that neither sort key contained illegally embedded zero bytes */
- return destLength;
-}
-
-U_NAMESPACE_BEGIN
-
-class SortKeyByteSink : public ByteSink {
-public:
- static const uint32_t FILL_ORIGINAL_BUFFER = 1;
- static const uint32_t DONT_GROW = 2;
- SortKeyByteSink(char *dest, int32_t destCapacity, uint32_t flags=0)
- : ownedBuffer_(NULL), buffer_(dest), capacity_(destCapacity),
- appended_(0),
- fill_(flags & FILL_ORIGINAL_BUFFER),
- grow_((flags & DONT_GROW) == 0) {
- if (buffer_ == NULL || capacity_ < 0) {
- buffer_ = reinterpret_cast<char *>(&lastResortByte_);
- capacity_ = 0;
- }
- }
- virtual ~SortKeyByteSink();
-
- virtual void Append(const char *bytes, int32_t n);
- void Append(const uint8_t *bytes, int32_t n) { Append(reinterpret_cast<const char *>(bytes), n); }
- void Append(uint8_t b) {
- if (appended_ < capacity_) {
- buffer_[appended_++] = (char)b;
- } else {
- Append(&b, 1);
- }
- }
- void Append(uint8_t b1, uint8_t b2) {
- int32_t a2 = appended_ + 2;
- if (a2 <= capacity_) {
- buffer_[appended_] = (char)b1;
- buffer_[appended_ + 1] = (char)b2;
- appended_ = a2;
- } else {
- char bytes[2] = { (char)b1, (char)b2 };
- Append(bytes, 2);
- }
- }
- void Append(const SortKeyByteSink &other) { Append(other.buffer_, other.appended_); }
- virtual char *GetAppendBuffer(int32_t min_capacity,
- int32_t desired_capacity_hint,
- char *scratch, int32_t scratch_capacity,
- int32_t *result_capacity);
- int32_t NumberOfBytesAppended() const { return appended_; }
- uint8_t &LastByte() {
- if (buffer_ != NULL && appended_ > 0) {
- return reinterpret_cast<uint8_t *>(buffer_)[appended_ - 1];
- } else {
- return lastResortByte_;
- }
- }
- uint8_t *GetLastFewBytes(int32_t n) {
- if (buffer_ != NULL && appended_ >= n) {
- return reinterpret_cast<uint8_t *>(buffer_) + appended_ - n;
- } else {
- return NULL;
- }
- }
- char *GetBuffer() { return buffer_; }
- uint8_t *GetUnsignedBuffer() { return reinterpret_cast<uint8_t *>(buffer_); }
- uint8_t *OrphanUnsignedBuffer(int32_t &orphanedCapacity);
- UBool IsOk() const { return buffer_ != NULL; } // otherwise out-of-memory
-
-private:
- SortKeyByteSink(const SortKeyByteSink &); // copy constructor not implemented
- SortKeyByteSink &operator=(const SortKeyByteSink &); // assignment operator not implemented
-
- UBool Resize(int32_t appendCapacity, int32_t length);
- void SetNotOk() {
- buffer_ = NULL;
- capacity_ = 0;
- }
-
- static uint8_t lastResortByte_; // last-resort return value from LastByte()
-
- char *ownedBuffer_;
- char *buffer_;
- int32_t capacity_;
- int32_t appended_;
- UBool fill_;
- UBool grow_;
-};
-
-uint8_t SortKeyByteSink::lastResortByte_ = 0;
-
-SortKeyByteSink::~SortKeyByteSink() {
- uprv_free(ownedBuffer_);
-}
-
-void
-SortKeyByteSink::Append(const char *bytes, int32_t n) {
- if (n <= 0) {
- return;
- }
- int32_t length = appended_;
- appended_ += n;
- if ((buffer_ + length) == bytes) {
- return; // the caller used GetAppendBuffer() and wrote the bytes already
- }
- if (buffer_ == NULL) {
- return; // allocation failed before already
- }
- int32_t available = capacity_ - length;
- if (bytes == NULL) {
- // assume that the caller failed to allocate memory
- if (fill_) {
- if (n > available) {
- n = available;
- }
- uprv_memset(buffer_, 0, n);
- }
- SetNotOk(); // propagate the out-of-memory error
- return;
- }
- if (n > available) {
- if (fill_ && available > 0) {
- // Fill the original buffer completely.
- uprv_memcpy(buffer_ + length, bytes, available);
- bytes += available;
- length += available;
- n -= available;
- available = 0;
- }
- fill_ = FALSE;
- if (!Resize(n, length)) {
- SetNotOk();
- return;
- }
- }
- uprv_memcpy(buffer_ + length, bytes, n);
-}
-
-char *
-SortKeyByteSink::GetAppendBuffer(int32_t min_capacity,
- int32_t desired_capacity_hint,
- char *scratch,
- int32_t scratch_capacity,
- int32_t *result_capacity) {
- if (min_capacity < 1 || scratch_capacity < min_capacity) {
- *result_capacity = 0;
- return NULL;
- }
- int32_t available = capacity_ - appended_;
- if (available >= min_capacity) {
- *result_capacity = available;
- return buffer_ + appended_;
- } else if (Resize(desired_capacity_hint, appended_)) {
- *result_capacity = capacity_ - appended_;
- return buffer_ + appended_;
- } else {
- *result_capacity = scratch_capacity;
- return scratch;
- }
-}
-
-UBool
-SortKeyByteSink::Resize(int32_t appendCapacity, int32_t length) {
- if (!grow_) {
- return FALSE;
- }
- int32_t newCapacity = 2 * capacity_;
- int32_t altCapacity = length + 2 * appendCapacity;
- if (newCapacity < altCapacity) {
- newCapacity = altCapacity;
- }
- if (newCapacity < 1024) {
- newCapacity = 1024;
- }
- char *newBuffer = (char *)uprv_malloc(newCapacity);
- if (newBuffer == NULL) {
- return FALSE;
- }
- uprv_memcpy(newBuffer, buffer_, length);
- uprv_free(ownedBuffer_);
- ownedBuffer_ = buffer_ = newBuffer;
- capacity_ = newCapacity;
- return TRUE;
-}
-
-uint8_t *
-SortKeyByteSink::OrphanUnsignedBuffer(int32_t &orphanedCapacity) {
- if (buffer_ == NULL || appended_ == 0) {
- orphanedCapacity = 0;
- return NULL;
- }
- if (ownedBuffer_ != NULL) {
- // orphan & forget the ownedBuffer_
- uint8_t *returnBuffer = reinterpret_cast<uint8_t *>(ownedBuffer_);
- ownedBuffer_ = buffer_ = NULL;
- orphanedCapacity = capacity_;
- capacity_ = appended_ = 0;
- return returnBuffer;
- }
- // clone the buffer_
- uint8_t *newBuffer = (uint8_t *)uprv_malloc(appended_);
- if (newBuffer == NULL) {
- orphanedCapacity = 0;
- return NULL;
- }
- uprv_memcpy(newBuffer, buffer_, appended_);
- orphanedCapacity = appended_;
- return newBuffer;
-}
-
-U_NAMESPACE_END
-
-/* sortkey API */
-U_CAPI int32_t U_EXPORT2
-ucol_getSortKey(const UCollator *coll,
- const UChar *source,
- int32_t sourceLength,
- uint8_t *result,
- int32_t resultLength)
-{
- UTRACE_ENTRY(UTRACE_UCOL_GET_SORTKEY);
- if (UTRACE_LEVEL(UTRACE_VERBOSE)) {
- UTRACE_DATA3(UTRACE_VERBOSE, "coll=%p, source string = %vh ", coll, source,
- ((sourceLength==-1 && source!=NULL) ? u_strlen(source) : sourceLength));
- }
-
- if(coll->delegate != NULL) {
- return ((const Collator*)coll->delegate)->getSortKey(source, sourceLength, result, resultLength);
- }
-
- UErrorCode status = U_ZERO_ERROR;
- int32_t keySize = 0;
-
- if(source != NULL) {
- // source == NULL is actually an error situation, but we would need to
- // have an error code to return it. Until we introduce a new
- // API, it stays like this
-
- /* this uses the function pointer that is set in updateinternalstate */
- /* currently, there are two funcs: */
- /*ucol_calcSortKey(...);*/
- /*ucol_calcSortKeySimpleTertiary(...);*/
-
- SortKeyByteSink sink(reinterpret_cast<char *>(result), resultLength,
- SortKeyByteSink::FILL_ORIGINAL_BUFFER | SortKeyByteSink::DONT_GROW);
- coll->sortKeyGen(coll, source, sourceLength, sink, &status);
- keySize = sink.NumberOfBytesAppended();
- }
- UTRACE_DATA2(UTRACE_VERBOSE, "Sort Key = %vb", result, keySize);
- UTRACE_EXIT_STATUS(status);
- return keySize;
-}
-
-/* this function is called by the C++ API for sortkey generation */
-U_CFUNC int32_t
-ucol_getSortKeyWithAllocation(const UCollator *coll,
- const UChar *source, int32_t sourceLength,
- uint8_t *&result, int32_t &resultCapacity,
- UErrorCode *pErrorCode) {
- SortKeyByteSink sink(reinterpret_cast<char *>(result), resultCapacity);
- coll->sortKeyGen(coll, source, sourceLength, sink, pErrorCode);
- int32_t resultLen = sink.NumberOfBytesAppended();
- if (U_SUCCESS(*pErrorCode)) {
- if (!sink.IsOk()) {
- *pErrorCode = U_MEMORY_ALLOCATION_ERROR;
- } else if (result != sink.GetUnsignedBuffer()) {
- result = sink.OrphanUnsignedBuffer(resultCapacity);
- }
- }
- return resultLen;
-}
-
-// Is this primary weight compressible?
-// Returns false for multi-lead-byte scripts (digits, Latin, Han, implicit).
-// TODO: This should use per-lead-byte flags from FractionalUCA.txt.
-static inline UBool
-isCompressible(const UCollator * /*coll*/, uint8_t primary1) {
- return UCOL_BYTE_FIRST_NON_LATIN_PRIMARY <= primary1 && primary1 <= maxRegularPrimary;
-}
-
-static
-inline void doCaseShift(SortKeyByteSink &cases, uint32_t &caseShift) {
- if (caseShift == 0) {
- cases.Append(UCOL_CASE_BYTE_START);
- caseShift = UCOL_CASE_SHIFT_START;
- }
-}
-
-// Packs the secondary buffer when processing French locale.
-static void
-packFrench(uint8_t *secondaries, int32_t secsize, SortKeyByteSink &result) {
- secondaries += secsize; // We read the secondary-level bytes back to front.
- uint8_t secondary;
- int32_t count2 = 0;
- int32_t i = 0;
- // we use i here since the key size already accounts for terminators, so we'll discard the increment
- for(i = 0; i<secsize; i++) {
- secondary = *(secondaries-i-1);
- /* This is compression code. */
- if (secondary == UCOL_COMMON2) {
- ++count2;
- } else {
- if (count2 > 0) {
- if (secondary > UCOL_COMMON2) { // not necessary for 4th level.
- while (count2 > UCOL_TOP_COUNT2) {
- result.Append((uint8_t)(UCOL_COMMON_TOP2 - UCOL_TOP_COUNT2));
- count2 -= (uint32_t)UCOL_TOP_COUNT2;
- }
- result.Append((uint8_t)(UCOL_COMMON_TOP2 - (count2-1)));
- } else {
- while (count2 > UCOL_BOT_COUNT2) {
- result.Append((uint8_t)(UCOL_COMMON_BOT2 + UCOL_BOT_COUNT2));
- count2 -= (uint32_t)UCOL_BOT_COUNT2;
- }
- result.Append((uint8_t)(UCOL_COMMON_BOT2 + (count2-1)));
- }
- count2 = 0;
- }
- result.Append(secondary);
- }
- }
- if (count2 > 0) {
- while (count2 > UCOL_BOT_COUNT2) {
- result.Append((uint8_t)(UCOL_COMMON_BOT2 + UCOL_BOT_COUNT2));
- count2 -= (uint32_t)UCOL_BOT_COUNT2;
- }
- result.Append((uint8_t)(UCOL_COMMON_BOT2 + (count2-1)));
- }
-}
-
-#define DEFAULT_ERROR_SIZE_FOR_CALCSORTKEY 0
-
-/* This is the sortkey work horse function */
-U_CFUNC void U_CALLCONV
-ucol_calcSortKey(const UCollator *coll,
- const UChar *source,
- int32_t sourceLength,
- SortKeyByteSink &result,
- UErrorCode *status)
-{
- if(U_FAILURE(*status)) {
- return;
- }
-
- /* Stack allocated buffers for buffers we use */
- char second[UCOL_SECONDARY_MAX_BUFFER], tert[UCOL_TERTIARY_MAX_BUFFER];
- char caseB[UCOL_CASE_MAX_BUFFER], quad[UCOL_QUAD_MAX_BUFFER];
-
- SortKeyByteSink &primaries = result;
- SortKeyByteSink secondaries(second, LENGTHOF(second));
- SortKeyByteSink tertiaries(tert, LENGTHOF(tert));
- SortKeyByteSink cases(caseB, LENGTHOF(caseB));
- SortKeyByteSink quads(quad, LENGTHOF(quad));
-
- UnicodeString normSource;
-
- int32_t len = (sourceLength == -1 ? u_strlen(source) : sourceLength);
-
- UColAttributeValue strength = coll->strength;
-
- uint8_t compareSec = (uint8_t)((strength >= UCOL_SECONDARY)?0:0xFF);
- uint8_t compareTer = (uint8_t)((strength >= UCOL_TERTIARY)?0:0xFF);
- uint8_t compareQuad = (uint8_t)((strength >= UCOL_QUATERNARY)?0:0xFF);
- UBool compareIdent = (strength == UCOL_IDENTICAL);
- UBool doCase = (coll->caseLevel == UCOL_ON);
- UBool isFrenchSec = (coll->frenchCollation == UCOL_ON) && (compareSec == 0);
- UBool shifted = (coll->alternateHandling == UCOL_SHIFTED);
- //UBool qShifted = shifted && (compareQuad == 0);
- UBool doHiragana = (coll->hiraganaQ == UCOL_ON) && (compareQuad == 0);
-
- uint32_t variableTopValue = coll->variableTopValue;
- // TODO: UCOL_COMMON_BOT4 should be a function of qShifted. If we have no
- // qShifted, we don't need to set UCOL_COMMON_BOT4 so high.
- uint8_t UCOL_COMMON_BOT4 = (uint8_t)((coll->variableTopValue>>8)+1);
- uint8_t UCOL_HIRAGANA_QUAD = 0;
- if(doHiragana) {
- UCOL_HIRAGANA_QUAD=UCOL_COMMON_BOT4++;
- /* allocate one more space for hiragana, value for hiragana */
- }
- uint8_t UCOL_BOT_COUNT4 = (uint8_t)(0xFF - UCOL_COMMON_BOT4);
-
- /* support for special features like caselevel and funky secondaries */
- int32_t lastSecondaryLength = 0;
- uint32_t caseShift = 0;
-
- /* If we need to normalize, we'll do it all at once at the beginning! */
- const Normalizer2 *norm2;
- if(compareIdent) {
- norm2 = Normalizer2Factory::getNFDInstance(*status);
- } else if(coll->normalizationMode != UCOL_OFF) {
- norm2 = Normalizer2Factory::getFCDInstance(*status);
- } else {
- norm2 = NULL;
- }
- if(norm2 != NULL) {
- normSource.setTo(FALSE, source, len);
- int32_t qcYesLength = norm2->spanQuickCheckYes(normSource, *status);
- if(qcYesLength != len) {
- UnicodeString unnormalized = normSource.tempSubString(qcYesLength);
- normSource.truncate(qcYesLength);
- norm2->normalizeSecondAndAppend(normSource, unnormalized, *status);
- source = normSource.getBuffer();
- len = normSource.length();
- }
- }
- collIterate s;
- IInit_collIterate(coll, source, len, &s, status);
- if(U_FAILURE(*status)) {
- return;
- }
- s.flags &= ~UCOL_ITER_NORM; // source passed the FCD test or else was normalized.
-
- uint32_t order = 0;
-
- uint8_t primary1 = 0;
- uint8_t primary2 = 0;
- uint8_t secondary = 0;
- uint8_t tertiary = 0;
- uint8_t caseSwitch = coll->caseSwitch;
- uint8_t tertiaryMask = coll->tertiaryMask;
- int8_t tertiaryAddition = coll->tertiaryAddition;
- uint8_t tertiaryTop = coll->tertiaryTop;
- uint8_t tertiaryBottom = coll->tertiaryBottom;
- uint8_t tertiaryCommon = coll->tertiaryCommon;
- uint8_t caseBits = 0;
-
- UBool wasShifted = FALSE;
- UBool notIsContinuation = FALSE;
-
- uint32_t count2 = 0, count3 = 0, count4 = 0;
- uint8_t leadPrimary = 0;
-
- for(;;) {
- order = ucol_IGetNextCE(coll, &s, status);
- if(order == UCOL_NO_MORE_CES) {
- break;
- }
-
- if(order == 0) {
- continue;
- }
-
- notIsContinuation = !isContinuation(order);
-
- if(notIsContinuation) {
- tertiary = (uint8_t)(order & UCOL_BYTE_SIZE_MASK);
- } else {
- tertiary = (uint8_t)((order & UCOL_REMOVE_CONTINUATION));
- }
-
- secondary = (uint8_t)((order >>= 8) & UCOL_BYTE_SIZE_MASK);
- primary2 = (uint8_t)((order >>= 8) & UCOL_BYTE_SIZE_MASK);
- primary1 = (uint8_t)(order >> 8);
-
- uint8_t originalPrimary1 = primary1;
- if(notIsContinuation && coll->leadBytePermutationTable != NULL) {
- primary1 = coll->leadBytePermutationTable[primary1];
- }
-
- if((shifted && ((notIsContinuation && order <= variableTopValue && primary1 > 0)
- || (!notIsContinuation && wasShifted)))
- || (wasShifted && primary1 == 0)) /* amendment to the UCA says that primary ignorables */
- {
- /* and other ignorables should be removed if following a shifted code point */
- if(primary1 == 0) { /* if we were shifted and we got an ignorable code point */
- /* we should just completely ignore it */
- continue;
- }
- if(compareQuad == 0) {
- if(count4 > 0) {
- while (count4 > UCOL_BOT_COUNT4) {
- quads.Append((uint8_t)(UCOL_COMMON_BOT4 + UCOL_BOT_COUNT4));
- count4 -= UCOL_BOT_COUNT4;
- }
- quads.Append((uint8_t)(UCOL_COMMON_BOT4 + (count4-1)));
- count4 = 0;
- }
- /* We are dealing with a variable and we're treating them as shifted */
- /* This is a shifted ignorable */
- if(primary1 != 0) { /* we need to check this since we could be in continuation */
- quads.Append(primary1);
- }
- if(primary2 != 0) {
- quads.Append(primary2);
- }
- }
- wasShifted = TRUE;
- } else {
- wasShifted = FALSE;
- /* Note: This code assumes that the table is well built i.e. not having 0 bytes where they are not supposed to be. */
- /* Usually, we'll have non-zero primary1 & primary2, except in cases of a-z and friends, when primary2 will */
- /* regular and simple sortkey calc */
- if(primary1 != UCOL_IGNORABLE) {
- if(notIsContinuation) {
- if(leadPrimary == primary1) {
- primaries.Append(primary2);
- } else {
- if(leadPrimary != 0) {
- primaries.Append((uint8_t)((primary1 > leadPrimary) ? UCOL_BYTE_UNSHIFTED_MAX : UCOL_BYTE_UNSHIFTED_MIN));
- }
- if(primary2 == UCOL_IGNORABLE) {
- /* one byter, not compressed */
- primaries.Append(primary1);
- leadPrimary = 0;
- } else if(isCompressible(coll, originalPrimary1)) {
- /* compress */
- primaries.Append(leadPrimary = primary1, primary2);
- } else {
- leadPrimary = 0;
- primaries.Append(primary1, primary2);
- }
- }
- } else { /* we are in continuation, so we're gonna add primary to the key don't care about compression */
- if(primary2 == UCOL_IGNORABLE) {
- primaries.Append(primary1);
- } else {
- primaries.Append(primary1, primary2);
- }
- }
- }
-
- if(secondary > compareSec) {
- if(!isFrenchSec) {
- /* This is compression code. */
- if (secondary == UCOL_COMMON2 && notIsContinuation) {
- ++count2;
- } else {
- if (count2 > 0) {
- if (secondary > UCOL_COMMON2) { // not necessary for 4th level.
- while (count2 > UCOL_TOP_COUNT2) {
- secondaries.Append((uint8_t)(UCOL_COMMON_TOP2 - UCOL_TOP_COUNT2));
- count2 -= (uint32_t)UCOL_TOP_COUNT2;
- }
- secondaries.Append((uint8_t)(UCOL_COMMON_TOP2 - (count2-1)));
- } else {
- while (count2 > UCOL_BOT_COUNT2) {
- secondaries.Append((uint8_t)(UCOL_COMMON_BOT2 + UCOL_BOT_COUNT2));
- count2 -= (uint32_t)UCOL_BOT_COUNT2;
- }
- secondaries.Append((uint8_t)(UCOL_COMMON_BOT2 + (count2-1)));
- }
- count2 = 0;
- }
- secondaries.Append(secondary);
- }
- } else {
- /* Do the special handling for French secondaries */
- /* We need to get continuation elements and do intermediate restore */
- /* abc1c2c3de with french secondaries need to be edc1c2c3ba NOT edc3c2c1ba */
- if(notIsContinuation) {
- if (lastSecondaryLength > 1) {
- uint8_t *frenchStartPtr = secondaries.GetLastFewBytes(lastSecondaryLength);
- if (frenchStartPtr != NULL) {
- /* reverse secondaries from frenchStartPtr up to frenchEndPtr */
- uint8_t *frenchEndPtr = frenchStartPtr + lastSecondaryLength - 1;
- uprv_ucol_reverse_buffer(uint8_t, frenchStartPtr, frenchEndPtr);
- }
- }
- lastSecondaryLength = 1;
- } else {
- ++lastSecondaryLength;
- }
- secondaries.Append(secondary);
- }
- }
-
- if(doCase && (primary1 > 0 || strength >= UCOL_SECONDARY)) {
- // do the case level if we need to do it. We don't want to calculate
- // case level for primary ignorables if we have only primary strength and case level
- // otherwise we would break well formedness of CEs
- doCaseShift(cases, caseShift);
- if(notIsContinuation) {
- caseBits = (uint8_t)(tertiary & 0xC0);
-
- if(tertiary != 0) {
- if(coll->caseFirst == UCOL_UPPER_FIRST) {
- if((caseBits & 0xC0) == 0) {
- cases.LastByte() |= 1 << (--caseShift);
- } else {
- cases.LastByte() |= 0 << (--caseShift);
- /* second bit */
- doCaseShift(cases, caseShift);
- cases.LastByte() |= ((caseBits>>6)&1) << (--caseShift);
- }
- } else {
- if((caseBits & 0xC0) == 0) {
- cases.LastByte() |= 0 << (--caseShift);
- } else {
- cases.LastByte() |= 1 << (--caseShift);
- /* second bit */
- doCaseShift(cases, caseShift);
- cases.LastByte() |= ((caseBits>>7)&1) << (--caseShift);
- }
- }
- }
- }
- } else {
- if(notIsContinuation) {
- tertiary ^= caseSwitch;
- }
- }
-
- tertiary &= tertiaryMask;
- if(tertiary > compareTer) {
- /* This is compression code. */
- /* sequence size check is included in the if clause */
- if (tertiary == tertiaryCommon && notIsContinuation) {
- ++count3;
- } else {
- if(tertiary > tertiaryCommon && tertiaryCommon == UCOL_COMMON3_NORMAL) {
- tertiary += tertiaryAddition;
- } else if(tertiary <= tertiaryCommon && tertiaryCommon == UCOL_COMMON3_UPPERFIRST) {
- tertiary -= tertiaryAddition;
- }
- if (count3 > 0) {
- if ((tertiary > tertiaryCommon)) {
- while (count3 > coll->tertiaryTopCount) {
- tertiaries.Append((uint8_t)(tertiaryTop - coll->tertiaryTopCount));
- count3 -= (uint32_t)coll->tertiaryTopCount;
- }
- tertiaries.Append((uint8_t)(tertiaryTop - (count3-1)));
- } else {
- while (count3 > coll->tertiaryBottomCount) {
- tertiaries.Append((uint8_t)(tertiaryBottom + coll->tertiaryBottomCount));
- count3 -= (uint32_t)coll->tertiaryBottomCount;
- }
- tertiaries.Append((uint8_t)(tertiaryBottom + (count3-1)));
- }
- count3 = 0;
- }
- tertiaries.Append(tertiary);
- }
- }
-
- if(/*qShifted*/(compareQuad==0) && notIsContinuation) {
- if(s.flags & UCOL_WAS_HIRAGANA) { // This was Hiragana and we need to note it
- if(count4>0) { // Close this part
- while (count4 > UCOL_BOT_COUNT4) {
- quads.Append((uint8_t)(UCOL_COMMON_BOT4 + UCOL_BOT_COUNT4));
- count4 -= UCOL_BOT_COUNT4;
- }
- quads.Append((uint8_t)(UCOL_COMMON_BOT4 + (count4-1)));
- count4 = 0;
- }
- quads.Append(UCOL_HIRAGANA_QUAD); // Add the Hiragana
- } else { // This wasn't Hiragana, so we can continue adding stuff
- count4++;
- }
- }
- }
- }
-
- /* Here, we are generally done with processing */
- /* bailing out would not be too productive */
-
- if(U_SUCCESS(*status)) {
- /* we have done all the CE's, now let's put them together to form a key */
- if(compareSec == 0) {
- if (count2 > 0) {
- while (count2 > UCOL_BOT_COUNT2) {
- secondaries.Append((uint8_t)(UCOL_COMMON_BOT2 + UCOL_BOT_COUNT2));
- count2 -= (uint32_t)UCOL_BOT_COUNT2;
- }
- secondaries.Append((uint8_t)(UCOL_COMMON_BOT2 + (count2-1)));
- }
- result.Append(UCOL_LEVELTERMINATOR);
- if(!isFrenchSec || !secondaries.IsOk()) {
- result.Append(secondaries);
- } else {
- // If there are any unresolved continuation secondaries,
- // reverse them here so that we can reverse the whole secondary thing.
- if (lastSecondaryLength > 1) {
- uint8_t *frenchStartPtr = secondaries.GetLastFewBytes(lastSecondaryLength);
- if (frenchStartPtr != NULL) {
- /* reverse secondaries from frenchStartPtr up to frenchEndPtr */
- uint8_t *frenchEndPtr = frenchStartPtr + lastSecondaryLength - 1;
- uprv_ucol_reverse_buffer(uint8_t, frenchStartPtr, frenchEndPtr);
- }
- }
- packFrench(secondaries.GetUnsignedBuffer(), secondaries.NumberOfBytesAppended(), result);
- }
- }
-
- if(doCase) {
- result.Append(UCOL_LEVELTERMINATOR);
- result.Append(cases);
- }
-
- if(compareTer == 0) {
- if (count3 > 0) {
- if (coll->tertiaryCommon != UCOL_COMMON_BOT3) {
- while (count3 >= coll->tertiaryTopCount) {
- tertiaries.Append((uint8_t)(tertiaryTop - coll->tertiaryTopCount));
- count3 -= (uint32_t)coll->tertiaryTopCount;
- }
- tertiaries.Append((uint8_t)(tertiaryTop - count3));
- } else {
- while (count3 > coll->tertiaryBottomCount) {
- tertiaries.Append((uint8_t)(tertiaryBottom + coll->tertiaryBottomCount));
- count3 -= (uint32_t)coll->tertiaryBottomCount;
- }
- tertiaries.Append((uint8_t)(tertiaryBottom + (count3-1)));
- }
- }
- result.Append(UCOL_LEVELTERMINATOR);
- result.Append(tertiaries);
-
- if(compareQuad == 0/*qShifted == TRUE*/) {
- if(count4 > 0) {
- while (count4 > UCOL_BOT_COUNT4) {
- quads.Append((uint8_t)(UCOL_COMMON_BOT4 + UCOL_BOT_COUNT4));
- count4 -= UCOL_BOT_COUNT4;
- }
- quads.Append((uint8_t)(UCOL_COMMON_BOT4 + (count4-1)));
- }
- result.Append(UCOL_LEVELTERMINATOR);
- result.Append(quads);
- }
-
- if(compareIdent) {
- result.Append(UCOL_LEVELTERMINATOR);
- u_writeIdenticalLevelRun(s.string, len, result);
- }
- }
- result.Append(0);
- }
-
- /* To avoid memory leak, free the offset buffer if necessary. */
- ucol_freeOffsetBuffer(&s);
-}
-
-
-U_CFUNC void U_CALLCONV
-ucol_calcSortKeySimpleTertiary(const UCollator *coll,
- const UChar *source,
- int32_t sourceLength,
- SortKeyByteSink &result,
- UErrorCode *status)
-{
- U_ALIGN_CODE(16);
-
- if(U_FAILURE(*status)) {
- return;
- }
-
- /* Stack allocated buffers for buffers we use */
- char second[UCOL_SECONDARY_MAX_BUFFER], tert[UCOL_TERTIARY_MAX_BUFFER];
-
- SortKeyByteSink &primaries = result;
- SortKeyByteSink secondaries(second, LENGTHOF(second));
- SortKeyByteSink tertiaries(tert, LENGTHOF(tert));
-
- UnicodeString normSource;
-
- int32_t len = sourceLength;
-
- /* If we need to normalize, we'll do it all at once at the beginning! */
- if(coll->normalizationMode != UCOL_OFF) {
- normSource.setTo(len < 0, source, len);
- const Normalizer2 *norm2 = Normalizer2Factory::getFCDInstance(*status);
- int32_t qcYesLength = norm2->spanQuickCheckYes(normSource, *status);
- if(qcYesLength != normSource.length()) {
- UnicodeString unnormalized = normSource.tempSubString(qcYesLength);
- normSource.truncate(qcYesLength);
- norm2->normalizeSecondAndAppend(normSource, unnormalized, *status);
- source = normSource.getBuffer();
- len = normSource.length();
- }
- }
- collIterate s;
- IInit_collIterate(coll, (UChar *)source, len, &s, status);
- if(U_FAILURE(*status)) {
- return;
- }
- s.flags &= ~UCOL_ITER_NORM; // source passed the FCD test or else was normalized.
-
- uint32_t order = 0;
-
- uint8_t primary1 = 0;
- uint8_t primary2 = 0;
- uint8_t secondary = 0;
- uint8_t tertiary = 0;
- uint8_t caseSwitch = coll->caseSwitch;
- uint8_t tertiaryMask = coll->tertiaryMask;
- int8_t tertiaryAddition = coll->tertiaryAddition;
- uint8_t tertiaryTop = coll->tertiaryTop;
- uint8_t tertiaryBottom = coll->tertiaryBottom;
- uint8_t tertiaryCommon = coll->tertiaryCommon;
-
- UBool notIsContinuation = FALSE;
-
- uint32_t count2 = 0, count3 = 0;
- uint8_t leadPrimary = 0;
-
- for(;;) {
- order = ucol_IGetNextCE(coll, &s, status);
-
- if(order == 0) {
- continue;
- }
-
- if(order == UCOL_NO_MORE_CES) {
- break;
- }
-
- notIsContinuation = !isContinuation(order);
-
- if(notIsContinuation) {
- tertiary = (uint8_t)((order & tertiaryMask));
- } else {
- tertiary = (uint8_t)((order & UCOL_REMOVE_CONTINUATION));
- }
-
- secondary = (uint8_t)((order >>= 8) & UCOL_BYTE_SIZE_MASK);
- primary2 = (uint8_t)((order >>= 8) & UCOL_BYTE_SIZE_MASK);
- primary1 = (uint8_t)(order >> 8);
-
- uint8_t originalPrimary1 = primary1;
- if (coll->leadBytePermutationTable != NULL && notIsContinuation) {
- primary1 = coll->leadBytePermutationTable[primary1];
- }
-
- /* Note: This code assumes that the table is well built i.e. not having 0 bytes where they are not supposed to be. */
- /* Usually, we'll have non-zero primary1 & primary2, except in cases of a-z and friends, when primary2 will */
- /* be zero with non zero primary1. primary3 is different than 0 only for long primaries - see above. */
- /* regular and simple sortkey calc */
- if(primary1 != UCOL_IGNORABLE) {
- if(notIsContinuation) {
- if(leadPrimary == primary1) {
- primaries.Append(primary2);
- } else {
- if(leadPrimary != 0) {
- primaries.Append((uint8_t)((primary1 > leadPrimary) ? UCOL_BYTE_UNSHIFTED_MAX : UCOL_BYTE_UNSHIFTED_MIN));
- }
- if(primary2 == UCOL_IGNORABLE) {
- /* one byter, not compressed */
- primaries.Append(primary1);
- leadPrimary = 0;
- } else if(isCompressible(coll, originalPrimary1)) {
- /* compress */
- primaries.Append(leadPrimary = primary1, primary2);
- } else {
- leadPrimary = 0;
- primaries.Append(primary1, primary2);
- }
- }
- } else { /* we are in continuation, so we're gonna add primary to the key don't care about compression */
- if(primary2 == UCOL_IGNORABLE) {
- primaries.Append(primary1);
- } else {
- primaries.Append(primary1, primary2);
- }
- }
- }
-
- if(secondary > 0) { /* I think that != 0 test should be != IGNORABLE */
- /* This is compression code. */
- if (secondary == UCOL_COMMON2 && notIsContinuation) {
- ++count2;
- } else {
- if (count2 > 0) {
- if (secondary > UCOL_COMMON2) { // not necessary for 4th level.
- while (count2 > UCOL_TOP_COUNT2) {
- secondaries.Append((uint8_t)(UCOL_COMMON_TOP2 - UCOL_TOP_COUNT2));
- count2 -= (uint32_t)UCOL_TOP_COUNT2;
- }
- secondaries.Append((uint8_t)(UCOL_COMMON_TOP2 - (count2-1)));
- } else {
- while (count2 > UCOL_BOT_COUNT2) {
- secondaries.Append((uint8_t)(UCOL_COMMON_BOT2 + UCOL_BOT_COUNT2));
- count2 -= (uint32_t)UCOL_BOT_COUNT2;
- }
- secondaries.Append((uint8_t)(UCOL_COMMON_BOT2 + (count2-1)));
- }
- count2 = 0;
- }
- secondaries.Append(secondary);
- }
- }
-
- if(notIsContinuation) {
- tertiary ^= caseSwitch;
- }
-
- if(tertiary > 0) {
- /* This is compression code. */
- /* sequence size check is included in the if clause */
- if (tertiary == tertiaryCommon && notIsContinuation) {
- ++count3;
- } else {
- if(tertiary > tertiaryCommon && tertiaryCommon == UCOL_COMMON3_NORMAL) {
- tertiary += tertiaryAddition;
- } else if (tertiary <= tertiaryCommon && tertiaryCommon == UCOL_COMMON3_UPPERFIRST) {
- tertiary -= tertiaryAddition;
- }
- if (count3 > 0) {
- if ((tertiary > tertiaryCommon)) {
- while (count3 > coll->tertiaryTopCount) {
- tertiaries.Append((uint8_t)(tertiaryTop - coll->tertiaryTopCount));
- count3 -= (uint32_t)coll->tertiaryTopCount;
- }
- tertiaries.Append((uint8_t)(tertiaryTop - (count3-1)));
- } else {
- while (count3 > coll->tertiaryBottomCount) {
- tertiaries.Append((uint8_t)(tertiaryBottom + coll->tertiaryBottomCount));
- count3 -= (uint32_t)coll->tertiaryBottomCount;
- }
- tertiaries.Append((uint8_t)(tertiaryBottom + (count3-1)));
- }
- count3 = 0;
- }
- tertiaries.Append(tertiary);
- }
- }
- }
-
- if(U_SUCCESS(*status)) {
- /* we have done all the CE's, now let's put them together to form a key */
- if (count2 > 0) {
- while (count2 > UCOL_BOT_COUNT2) {
- secondaries.Append((uint8_t)(UCOL_COMMON_BOT2 + UCOL_BOT_COUNT2));
- count2 -= (uint32_t)UCOL_BOT_COUNT2;
- }
- secondaries.Append((uint8_t)(UCOL_COMMON_BOT2 + (count2-1)));
- }
- result.Append(UCOL_LEVELTERMINATOR);
- result.Append(secondaries);
-
- if (count3 > 0) {
- if (coll->tertiaryCommon != UCOL_COMMON3_NORMAL) {
- while (count3 >= coll->tertiaryTopCount) {
- tertiaries.Append((uint8_t)(tertiaryTop - coll->tertiaryTopCount));
- count3 -= (uint32_t)coll->tertiaryTopCount;
- }
- tertiaries.Append((uint8_t)(tertiaryTop - count3));
- } else {
- while (count3 > coll->tertiaryBottomCount) {
- tertiaries.Append((uint8_t)(tertiaryBottom + coll->tertiaryBottomCount));
- count3 -= (uint32_t)coll->tertiaryBottomCount;
- }
- tertiaries.Append((uint8_t)(tertiaryBottom + (count3-1)));
- }
- }
- result.Append(UCOL_LEVELTERMINATOR);
- result.Append(tertiaries);
-
- result.Append(0);
- }
-
- /* To avoid memory leak, free the offset buffer if necessary. */
- ucol_freeOffsetBuffer(&s);
-
- if (U_SUCCESS(*status) && !result.IsOk()) {
- *status = U_BUFFER_OVERFLOW_ERROR;
- }
-}
-
-static inline
-UBool isShiftedCE(uint32_t CE, uint32_t LVT, UBool *wasShifted) {
- UBool notIsContinuation = !isContinuation(CE);
- uint8_t primary1 = (uint8_t)((CE >> 24) & 0xFF);
- if((LVT && ((notIsContinuation && (CE & 0xFFFF0000)<= LVT && primary1 > 0)
- || (!notIsContinuation && *wasShifted)))
- || (*wasShifted && primary1 == 0)) /* amendment to the UCA says that primary ignorables */
- {
- // The stuff below should probably be in the sortkey code... maybe not...
- if(primary1 != 0) { /* if we were shifted and we got an ignorable code point */
- /* we should just completely ignore it */
- *wasShifted = TRUE;
- //continue;
- }
- //*wasShifted = TRUE;
- return TRUE;
- } else {
- *wasShifted = FALSE;
- return FALSE;
- }
-}
-static inline
-void terminatePSKLevel(int32_t level, int32_t maxLevel, int32_t &i, uint8_t *dest) {
- if(level < maxLevel) {
- dest[i++] = UCOL_LEVELTERMINATOR;
- } else {
- dest[i++] = 0;
- }
-}
-
-/** enumeration of level identifiers for partial sort key generation */
-enum {
- UCOL_PSK_PRIMARY = 0,
- UCOL_PSK_SECONDARY = 1,
- UCOL_PSK_CASE = 2,
- UCOL_PSK_TERTIARY = 3,
- UCOL_PSK_QUATERNARY = 4,
- UCOL_PSK_QUIN = 5, /** This is an extra level, not used - but we have three bits to blow */
- UCOL_PSK_IDENTICAL = 6,
- UCOL_PSK_NULL = 7, /** level for the end of sort key. Will just produce zeros */
- UCOL_PSK_LIMIT
-};
-
-/** collation state enum. *_SHIFT value is how much to shift right
- * to get the state piece to the right. *_MASK value should be
- * ANDed with the shifted state. This data is stored in state[1]
- * field.
- */
-enum {
- UCOL_PSK_LEVEL_SHIFT = 0, /** level identificator. stores an enum value from above */
- UCOL_PSK_LEVEL_MASK = 7, /** three bits */
- UCOL_PSK_BYTE_COUNT_OR_FRENCH_DONE_SHIFT = 3, /** number of bytes of primary or quaternary already written */
- UCOL_PSK_BYTE_COUNT_OR_FRENCH_DONE_MASK = 1,
- /** can be only 0 or 1, since we get up to two bytes from primary or quaternary
- * This field is also used to denote that the French secondary level is finished
- */
- UCOL_PSK_WAS_SHIFTED_SHIFT = 4,/** was the last value shifted */
- UCOL_PSK_WAS_SHIFTED_MASK = 1, /** can be 0 or 1 (Boolean) */
- UCOL_PSK_USED_FRENCH_SHIFT = 5,/** how many French bytes have we already written */
- UCOL_PSK_USED_FRENCH_MASK = 3, /** up to 4 bytes. See comment just below */
- /** When we do French we need to reverse secondary values. However, continuations
- * need to stay the same. So if you had abc1c2c3de, you need to have edc1c2c3ba
- */
- UCOL_PSK_BOCSU_BYTES_SHIFT = 7,
- UCOL_PSK_BOCSU_BYTES_MASK = 3,
- UCOL_PSK_CONSUMED_CES_SHIFT = 9,
- UCOL_PSK_CONSUMED_CES_MASK = 0x7FFFF
-};
-
-// macro calculating the number of expansion CEs available
-#define uprv_numAvailableExpCEs(s) (s).CEpos - (s).toReturn
-
-
-/** main sortkey part procedure. On the first call,
- * you should pass in a collator, an iterator, empty state
- * state[0] == state[1] == 0, a buffer to hold results
- * number of bytes you need and an error code pointer.
- * Make sure your buffer is big enough to hold the wanted
- * number of sortkey bytes. I don't check.
- * The only meaningful status you can get back is
- * U_BUFFER_OVERFLOW_ERROR, which basically means that you
- * have been dealt a raw deal and that you probably won't
- * be able to use partial sortkey generation for this
- * particular combination of string and collator. This
- * is highly unlikely, but you should still check the error code.
- * Any other status means that you're not in a sane situation
- * anymore. After the first call, preserve state values and
- * use them on subsequent calls to obtain more bytes of a sortkey.
- * Use until the number of bytes written is smaller than the requested
- * number of bytes. Generated sortkey is not compatible with the
- * one generated by ucol_getSortKey, as we don't do any compression.
- * However, levels are still terminated by a 1 (one) and the sortkey
- * is terminated by a 0 (zero). Identical level is the same as in the
- * regular sortkey - internal bocu-1 implementation is used.
- * For curious, although you cannot do much about this, here is
- * the structure of state words.
- * state[0] - iterator state. Depends on the iterator implementation,
- * but allows the iterator to continue where it stopped in
- * the last iteration.
- * state[1] - collation processing state. Here is the distribution
- * of the bits:
- * 0, 1, 2 - level of the sortkey - primary, secondary, case, tertiary
- * quaternary, quin (we don't use this one), identical and
- * null (producing only zeroes - first one to terminate the
- * sortkey and subsequent to fill the buffer).
- * 3 - byte count. Number of bytes written on the primary level.
- * 4 - was shifted. Whether the previous iteration finished in the
- * shifted state.
- * 5, 6 - French continuation bytes written. See the comment in the enum
- * 7,8 - Bocsu bytes used. Number of bytes from a bocu sequence on
- * the identical level.
- * 9..31 - CEs consumed. Number of getCE or next32 operations performed
- * since thes last successful update of the iterator state.
- */
-U_CAPI int32_t U_EXPORT2
-ucol_nextSortKeyPart(const UCollator *coll,
- UCharIterator *iter,
- uint32_t state[2],
- uint8_t *dest, int32_t count,
- UErrorCode *status)
-{
- /* error checking */
- if(status==NULL || U_FAILURE(*status)) {
- return 0;
- }
- UTRACE_ENTRY(UTRACE_UCOL_NEXTSORTKEYPART);
- if( coll==NULL || iter==NULL ||
- state==NULL ||
- count<0 || (count>0 && dest==NULL)
- ) {
- *status=U_ILLEGAL_ARGUMENT_ERROR;
- UTRACE_EXIT_STATUS(status);
- return 0;
- }
-
- UTRACE_DATA6(UTRACE_VERBOSE, "coll=%p, iter=%p, state=%d %d, dest=%p, count=%d",
- coll, iter, state[0], state[1], dest, count);
-
- if(count==0) {
- /* nothing to do */
- UTRACE_EXIT_VALUE(0);
- return 0;
- }
- /** Setting up situation according to the state we got from the previous iteration */
- // The state of the iterator from the previous invocation
- uint32_t iterState = state[0];
- // Has the last iteration ended in the shifted state
- UBool wasShifted = ((state[1] >> UCOL_PSK_WAS_SHIFTED_SHIFT) & UCOL_PSK_WAS_SHIFTED_MASK)?TRUE:FALSE;
- // What is the current level of the sortkey?
- int32_t level= (state[1] >> UCOL_PSK_LEVEL_SHIFT) & UCOL_PSK_LEVEL_MASK;
- // Have we written only one byte from a two byte primary in the previous iteration?
- // Also on secondary level - have we finished with the French secondary?
- int32_t byteCountOrFrenchDone = (state[1] >> UCOL_PSK_BYTE_COUNT_OR_FRENCH_DONE_SHIFT) & UCOL_PSK_BYTE_COUNT_OR_FRENCH_DONE_MASK;
- // number of bytes in the continuation buffer for French
- int32_t usedFrench = (state[1] >> UCOL_PSK_USED_FRENCH_SHIFT) & UCOL_PSK_USED_FRENCH_MASK;
- // Number of bytes already written from a bocsu sequence. Since
- // the longes bocsu sequence is 4 long, this can be up to 3.
- int32_t bocsuBytesUsed = (state[1] >> UCOL_PSK_BOCSU_BYTES_SHIFT) & UCOL_PSK_BOCSU_BYTES_MASK;
- // Number of elements that need to be consumed in this iteration because
- // the iterator returned UITER_NO_STATE at the end of the last iteration,
- // so we had to save the last valid state.
- int32_t cces = (state[1] >> UCOL_PSK_CONSUMED_CES_SHIFT) & UCOL_PSK_CONSUMED_CES_MASK;
-
- /** values that depend on the collator attributes */
- // strength of the collator.
- int32_t strength = ucol_getAttribute(coll, UCOL_STRENGTH, status);
- // maximal level of the partial sortkey. Need to take whether case level is done
- int32_t maxLevel = 0;
- if(strength < UCOL_TERTIARY) {
- if(ucol_getAttribute(coll, UCOL_CASE_LEVEL, status) == UCOL_ON) {
- maxLevel = UCOL_PSK_CASE;
- } else {
- maxLevel = strength;
- }
- } else {
- if(strength == UCOL_TERTIARY) {
- maxLevel = UCOL_PSK_TERTIARY;
- } else if(strength == UCOL_QUATERNARY) {
- maxLevel = UCOL_PSK_QUATERNARY;
- } else { // identical
- maxLevel = UCOL_IDENTICAL;
- }
- }
- // value for the quaternary level if Hiragana is encountered. Used for JIS X 4061 collation
- uint8_t UCOL_HIRAGANA_QUAD =
- (ucol_getAttribute(coll, UCOL_HIRAGANA_QUATERNARY_MODE, status) == UCOL_ON)?0xFE:0xFF;
- // Boundary value that decides whether a CE is shifted or not
- uint32_t LVT = (coll->alternateHandling == UCOL_SHIFTED)?(coll->variableTopValue<<16):0;
- // Are we doing French collation?
- UBool doingFrench = (ucol_getAttribute(coll, UCOL_FRENCH_COLLATION, status) == UCOL_ON);
-
- /** initializing the collation state */
- UBool notIsContinuation = FALSE;
- uint32_t CE = UCOL_NO_MORE_CES;
-
- collIterate s;
- IInit_collIterate(coll, NULL, -1, &s, status);
- if(U_FAILURE(*status)) {
- UTRACE_EXIT_STATUS(*status);
- return 0;
- }
- s.iterator = iter;
- s.flags |= UCOL_USE_ITERATOR;
- // This variable tells us whether we have produced some other levels in this iteration
- // before we moved to the identical level. In that case, we need to switch the
- // type of the iterator.
- UBool doingIdenticalFromStart = FALSE;
- // Normalizing iterator
- // The division for the array length may truncate the array size to
- // a little less than UNORM_ITER_SIZE, but that size is dimensioned too high
- // for all platforms anyway.
- UAlignedMemory stackNormIter[UNORM_ITER_SIZE/sizeof(UAlignedMemory)];
- UNormIterator *normIter = NULL;
- // If the normalization is turned on for the collator and we are below identical level
- // we will use a FCD normalizing iterator
- if(ucol_getAttribute(coll, UCOL_NORMALIZATION_MODE, status) == UCOL_ON && level < UCOL_PSK_IDENTICAL) {
- normIter = unorm_openIter(stackNormIter, sizeof(stackNormIter), status);
- s.iterator = unorm_setIter(normIter, iter, UNORM_FCD, status);
- s.flags &= ~UCOL_ITER_NORM;
- if(U_FAILURE(*status)) {
- UTRACE_EXIT_STATUS(*status);
- return 0;
- }
- } else if(level == UCOL_PSK_IDENTICAL) {
- // for identical level, we need a NFD iterator. We need to instantiate it here, since we
- // will be updating the state - and this cannot be done on an ordinary iterator.
- normIter = unorm_openIter(stackNormIter, sizeof(stackNormIter), status);
- s.iterator = unorm_setIter(normIter, iter, UNORM_NFD, status);
- s.flags &= ~UCOL_ITER_NORM;
- if(U_FAILURE(*status)) {
- UTRACE_EXIT_STATUS(*status);
- return 0;
- }
- doingIdenticalFromStart = TRUE;
- }
-
- // This is the tentative new state of the iterator. The problem
- // is that the iterator might return an undefined state, in
- // which case we should save the last valid state and increase
- // the iterator skip value.
- uint32_t newState = 0;
-
- // First, we set the iterator to the last valid position
- // from the last iteration. This was saved in state[0].
- if(iterState == 0) {
- /* initial state */
- if(level == UCOL_PSK_SECONDARY && doingFrench && !byteCountOrFrenchDone) {
- s.iterator->move(s.iterator, 0, UITER_LIMIT);
- } else {
- s.iterator->move(s.iterator, 0, UITER_START);
- }
- } else {
- /* reset to previous state */
- s.iterator->setState(s.iterator, iterState, status);
- if(U_FAILURE(*status)) {
- UTRACE_EXIT_STATUS(*status);
- return 0;
- }
- }
-
-
-
- // This variable tells us whether we can attempt to update the state
- // of iterator. Situations where we don't want to update iterator state
- // are the existence of expansion CEs that are not yet processed, and
- // finishing the case level without enough space in the buffer to insert
- // a level terminator.
- UBool canUpdateState = TRUE;
-
- // Consume all the CEs that were consumed at the end of the previous
- // iteration without updating the iterator state. On identical level,
- // consume the code points.
- int32_t counter = cces;
- if(level < UCOL_PSK_IDENTICAL) {
- while(counter-->0) {
- // If we're doing French and we are on the secondary level,
- // we go backwards.
- if(level == UCOL_PSK_SECONDARY && doingFrench) {
- CE = ucol_IGetPrevCE(coll, &s, status);
- } else {
- CE = ucol_IGetNextCE(coll, &s, status);
- }
- if(CE==UCOL_NO_MORE_CES) {
- /* should not happen */
- *status=U_INTERNAL_PROGRAM_ERROR;
- UTRACE_EXIT_STATUS(*status);
- return 0;
- }
- if(uprv_numAvailableExpCEs(s)) {
- canUpdateState = FALSE;
- }
- }
- } else {
- while(counter-->0) {
- uiter_next32(s.iterator);
- }
- }
-
- // French secondary needs to know whether the iterator state of zero came from previous level OR
- // from a new invocation...
- UBool wasDoingPrimary = FALSE;
- // destination buffer byte counter. When this guy
- // gets to count, we're done with the iteration
- int32_t i = 0;
- // used to count the zero bytes written after we
- // have finished with the sort key
- int32_t j = 0;
-
-
- // Hm.... I think we're ready to plunge in. Basic story is as following:
- // we have a fall through case based on level. This is used for initial
- // positioning on iteration start. Every level processor contains a
- // for(;;) which will be broken when we exhaust all the CEs. Other
- // way to exit is a goto saveState, which happens when we have filled
- // out our buffer.
- switch(level) {
- case UCOL_PSK_PRIMARY:
- wasDoingPrimary = TRUE;
- for(;;) {
- if(i==count) {
- goto saveState;
- }
- // We should save the state only if we
- // are sure that we are done with the
- // previous iterator state
- if(canUpdateState && byteCountOrFrenchDone == 0) {
- newState = s.iterator->getState(s.iterator);
- if(newState != UITER_NO_STATE) {
- iterState = newState;
- cces = 0;
- }
- }
- CE = ucol_IGetNextCE(coll, &s, status);
- cces++;
- if(CE==UCOL_NO_MORE_CES) {
- // Add the level separator
- terminatePSKLevel(level, maxLevel, i, dest);
- byteCountOrFrenchDone=0;
- // Restart the iteration an move to the
- // second level
- s.iterator->move(s.iterator, 0, UITER_START);
- cces = 0;
- level = UCOL_PSK_SECONDARY;
- break;
- }
- if(!isContinuation(CE)){
- if(coll->leadBytePermutationTable != NULL){
- CE = (coll->leadBytePermutationTable[CE>>24] << 24) | (CE & 0x00FFFFFF);
- }
- }
- if(!isShiftedCE(CE, LVT, &wasShifted)) {
- CE >>= UCOL_PRIMARYORDERSHIFT; /* get primary */
- if(CE != 0) {
- if(byteCountOrFrenchDone == 0) {
- // get the second byte of primary
- dest[i++]=(uint8_t)(CE >> 8);
- } else {
- byteCountOrFrenchDone = 0;
- }
- if((CE &=0xff)!=0) {
- if(i==count) {
- /* overflow */
- byteCountOrFrenchDone = 1;
- cces--;
- goto saveState;
- }
- dest[i++]=(uint8_t)CE;
- }
- }
- }
- if(uprv_numAvailableExpCEs(s)) {
- canUpdateState = FALSE;
- } else {
- canUpdateState = TRUE;
- }
- }
- /* fall through to next level */
- case UCOL_PSK_SECONDARY:
- if(strength >= UCOL_SECONDARY) {
- if(!doingFrench) {
- for(;;) {
- if(i == count) {
- goto saveState;
- }
- // We should save the state only if we
- // are sure that we are done with the
- // previous iterator state
- if(canUpdateState) {
- newState = s.iterator->getState(s.iterator);
- if(newState != UITER_NO_STATE) {
- iterState = newState;
- cces = 0;
- }
- }
- CE = ucol_IGetNextCE(coll, &s, status);
- cces++;
- if(CE==UCOL_NO_MORE_CES) {
- // Add the level separator
- terminatePSKLevel(level, maxLevel, i, dest);
- byteCountOrFrenchDone = 0;
- // Restart the iteration an move to the
- // second level
- s.iterator->move(s.iterator, 0, UITER_START);
- cces = 0;
- level = UCOL_PSK_CASE;
- break;
- }
- if(!isShiftedCE(CE, LVT, &wasShifted)) {
- CE >>= 8; /* get secondary */
- if(CE != 0) {
- dest[i++]=(uint8_t)CE;
- }
- }
- if(uprv_numAvailableExpCEs(s)) {
- canUpdateState = FALSE;
- } else {
- canUpdateState = TRUE;
- }
- }
- } else { // French secondary processing
- uint8_t frenchBuff[UCOL_MAX_BUFFER];
- int32_t frenchIndex = 0;
- // Here we are going backwards.
- // If the iterator is at the beggining, it should be
- // moved to end.
- if(wasDoingPrimary) {
- s.iterator->move(s.iterator, 0, UITER_LIMIT);
- cces = 0;
- }
- for(;;) {
- if(i == count) {
- goto saveState;
- }
- if(canUpdateState) {
- newState = s.iterator->getState(s.iterator);
- if(newState != UITER_NO_STATE) {
- iterState = newState;
- cces = 0;
- }
- }
- CE = ucol_IGetPrevCE(coll, &s, status);
- cces++;
- if(CE==UCOL_NO_MORE_CES) {
- // Add the level separator
- terminatePSKLevel(level, maxLevel, i, dest);
- byteCountOrFrenchDone = 0;
- // Restart the iteration an move to the next level
- s.iterator->move(s.iterator, 0, UITER_START);
- level = UCOL_PSK_CASE;
- break;
- }
- if(isContinuation(CE)) { // if it's a continuation, we want to save it and
- // reverse when we get a first non-continuation CE.
- CE >>= 8;
- frenchBuff[frenchIndex++] = (uint8_t)CE;
- } else if(!isShiftedCE(CE, LVT, &wasShifted)) {
- CE >>= 8; /* get secondary */
- if(!frenchIndex) {
- if(CE != 0) {
- dest[i++]=(uint8_t)CE;
- }
- } else {
- frenchBuff[frenchIndex++] = (uint8_t)CE;
- frenchIndex -= usedFrench;
- usedFrench = 0;
- while(i < count && frenchIndex) {
- dest[i++] = frenchBuff[--frenchIndex];
- usedFrench++;
- }
- }
- }
- if(uprv_numAvailableExpCEs(s)) {
- canUpdateState = FALSE;
- } else {
- canUpdateState = TRUE;
- }
- }
- }
- } else {
- level = UCOL_PSK_CASE;
- }
- /* fall through to next level */
- case UCOL_PSK_CASE:
- if(ucol_getAttribute(coll, UCOL_CASE_LEVEL, status) == UCOL_ON) {
- uint32_t caseShift = UCOL_CASE_SHIFT_START;
- uint8_t caseByte = UCOL_CASE_BYTE_START;
- uint8_t caseBits = 0;
-
- for(;;) {
- U_ASSERT(caseShift <= UCOL_CASE_SHIFT_START);
- if(i == count) {
- goto saveState;
- }
- // We should save the state only if we
- // are sure that we are done with the
- // previous iterator state
- if(canUpdateState) {
- newState = s.iterator->getState(s.iterator);
- if(newState != UITER_NO_STATE) {
- iterState = newState;
- cces = 0;
- }
- }
- CE = ucol_IGetNextCE(coll, &s, status);
- cces++;
- if(CE==UCOL_NO_MORE_CES) {
- // On the case level we might have an unfinished
- // case byte. Add one if it's started.
- if(caseShift != UCOL_CASE_SHIFT_START) {
- dest[i++] = caseByte;
- }
- cces = 0;
- // We have finished processing CEs on this level.
- // However, we don't know if we have enough space
- // to add a case level terminator.
- if(i < count) {
- // Add the level separator
- terminatePSKLevel(level, maxLevel, i, dest);
- // Restart the iteration and move to the
- // next level
- s.iterator->move(s.iterator, 0, UITER_START);
- level = UCOL_PSK_TERTIARY;
- } else {
- canUpdateState = FALSE;
- }
- break;
- }
-
- if(!isShiftedCE(CE, LVT, &wasShifted)) {
- if(!isContinuation(CE) && ((CE & UCOL_PRIMARYMASK) != 0 || strength > UCOL_PRIMARY)) {
- // do the case level if we need to do it. We don't want to calculate
- // case level for primary ignorables if we have only primary strength and case level
- // otherwise we would break well formedness of CEs
- CE = (uint8_t)(CE & UCOL_BYTE_SIZE_MASK);
- caseBits = (uint8_t)(CE & 0xC0);
- // this copies the case level logic from the
- // sort key generation code
- if(CE != 0) {
- if (caseShift == 0) {
- dest[i++] = caseByte;
- caseShift = UCOL_CASE_SHIFT_START;
- caseByte = UCOL_CASE_BYTE_START;
- }
- if(coll->caseFirst == UCOL_UPPER_FIRST) {
- if((caseBits & 0xC0) == 0) {
- caseByte |= 1 << (--caseShift);
- } else {
- caseByte |= 0 << (--caseShift);
- /* second bit */
- if(caseShift == 0) {
- dest[i++] = caseByte;
- caseShift = UCOL_CASE_SHIFT_START;
- caseByte = UCOL_CASE_BYTE_START;
- }
- caseByte |= ((caseBits>>6)&1) << (--caseShift);
- }
- } else {
- if((caseBits & 0xC0) == 0) {
- caseByte |= 0 << (--caseShift);
- } else {
- caseByte |= 1 << (--caseShift);
- /* second bit */
- if(caseShift == 0) {
- dest[i++] = caseByte;
- caseShift = UCOL_CASE_SHIFT_START;
- caseByte = UCOL_CASE_BYTE_START;
- }
- caseByte |= ((caseBits>>7)&1) << (--caseShift);
- }
- }
- }
-
- }
- }
- // Not sure this is correct for the case level - revisit
- if(uprv_numAvailableExpCEs(s)) {
- canUpdateState = FALSE;
- } else {
- canUpdateState = TRUE;
- }
- }
- } else {
- level = UCOL_PSK_TERTIARY;
- }
- /* fall through to next level */
- case UCOL_PSK_TERTIARY:
- if(strength >= UCOL_TERTIARY) {
- for(;;) {
- if(i == count) {
- goto saveState;
- }
- // We should save the state only if we
- // are sure that we are done with the
- // previous iterator state
- if(canUpdateState) {
- newState = s.iterator->getState(s.iterator);
- if(newState != UITER_NO_STATE) {
- iterState = newState;
- cces = 0;
- }
- }
- CE = ucol_IGetNextCE(coll, &s, status);
- cces++;
- if(CE==UCOL_NO_MORE_CES) {
- // Add the level separator
- terminatePSKLevel(level, maxLevel, i, dest);
- byteCountOrFrenchDone = 0;
- // Restart the iteration an move to the
- // second level
- s.iterator->move(s.iterator, 0, UITER_START);
- cces = 0;
- level = UCOL_PSK_QUATERNARY;
- break;
- }
- if(!isShiftedCE(CE, LVT, &wasShifted)) {
- notIsContinuation = !isContinuation(CE);
-
- if(notIsContinuation) {
- CE = (uint8_t)(CE & UCOL_BYTE_SIZE_MASK);
- CE ^= coll->caseSwitch;
- CE &= coll->tertiaryMask;
- } else {
- CE = (uint8_t)((CE & UCOL_REMOVE_CONTINUATION));
- }
-
- if(CE != 0) {
- dest[i++]=(uint8_t)CE;
- }
- }
- if(uprv_numAvailableExpCEs(s)) {
- canUpdateState = FALSE;
- } else {
- canUpdateState = TRUE;
- }
- }
- } else {
- // if we're not doing tertiary
- // skip to the end
- level = UCOL_PSK_NULL;
- }
- /* fall through to next level */
- case UCOL_PSK_QUATERNARY:
- if(strength >= UCOL_QUATERNARY) {
- for(;;) {
- if(i == count) {
- goto saveState;
- }
- // We should save the state only if we
- // are sure that we are done with the
- // previous iterator state
- if(canUpdateState) {
- newState = s.iterator->getState(s.iterator);
- if(newState != UITER_NO_STATE) {
- iterState = newState;
- cces = 0;
- }
- }
- CE = ucol_IGetNextCE(coll, &s, status);
- cces++;
- if(CE==UCOL_NO_MORE_CES) {
- // Add the level separator
- terminatePSKLevel(level, maxLevel, i, dest);
- //dest[i++] = UCOL_LEVELTERMINATOR;
- byteCountOrFrenchDone = 0;
- // Restart the iteration an move to the
- // second level
- s.iterator->move(s.iterator, 0, UITER_START);
- cces = 0;
- level = UCOL_PSK_QUIN;
- break;
- }
- if(CE==0)
- continue;
- if(isShiftedCE(CE, LVT, &wasShifted)) {
- CE >>= 16; /* get primary */
- if(CE != 0) {
- if(byteCountOrFrenchDone == 0) {
- dest[i++]=(uint8_t)(CE >> 8);
- } else {
- byteCountOrFrenchDone = 0;
- }
- if((CE &=0xff)!=0) {
- if(i==count) {
- /* overflow */
- byteCountOrFrenchDone = 1;
- goto saveState;
- }
- dest[i++]=(uint8_t)CE;
- }
- }
- } else {
- notIsContinuation = !isContinuation(CE);
- if(notIsContinuation) {
- if(s.flags & UCOL_WAS_HIRAGANA) { // This was Hiragana and we need to note it
- dest[i++] = UCOL_HIRAGANA_QUAD;
- } else {
- dest[i++] = 0xFF;
- }
- }
- }
- if(uprv_numAvailableExpCEs(s)) {
- canUpdateState = FALSE;
- } else {
- canUpdateState = TRUE;
- }
- }
- } else {
- // if we're not doing quaternary
- // skip to the end
- level = UCOL_PSK_NULL;
- }
- /* fall through to next level */
- case UCOL_PSK_QUIN:
- level = UCOL_PSK_IDENTICAL;
- /* fall through to next level */
- case UCOL_PSK_IDENTICAL:
- if(strength >= UCOL_IDENTICAL) {
- UChar32 first, second;
- int32_t bocsuBytesWritten = 0;
- // We always need to do identical on
- // the NFD form of the string.
- if(normIter == NULL) {
- // we arrived from the level below and
- // normalization was not turned on.
- // therefore, we need to make a fresh NFD iterator
- normIter = unorm_openIter(stackNormIter, sizeof(stackNormIter), status);
- s.iterator = unorm_setIter(normIter, iter, UNORM_NFD, status);
- } else if(!doingIdenticalFromStart) {
- // there is an iterator, but we did some other levels.
- // therefore, we have a FCD iterator - need to make
- // a NFD one.
- // normIter being at the beginning does not guarantee
- // that the underlying iterator is at the beginning
- iter->move(iter, 0, UITER_START);
- s.iterator = unorm_setIter(normIter, iter, UNORM_NFD, status);
- }
- // At this point we have a NFD iterator that is positioned
- // in the right place
- if(U_FAILURE(*status)) {
- UTRACE_EXIT_STATUS(*status);
- return 0;
- }
- first = uiter_previous32(s.iterator);
- // maybe we're at the start of the string
- if(first == U_SENTINEL) {
- first = 0;
- } else {
- uiter_next32(s.iterator);
- }
-
- j = 0;
- for(;;) {
- if(i == count) {
- if(j+1 < bocsuBytesWritten) {
- bocsuBytesUsed = j+1;
- }
- goto saveState;
- }
-
- // On identical level, we will always save
- // the state if we reach this point, since
- // we don't depend on getNextCE for content
- // all the content is in our buffer and we
- // already either stored the full buffer OR
- // otherwise we won't arrive here.
- newState = s.iterator->getState(s.iterator);
- if(newState != UITER_NO_STATE) {
- iterState = newState;
- cces = 0;
- }
-
- uint8_t buff[4];
- second = uiter_next32(s.iterator);
- cces++;
-
- // end condition for identical level
- if(second == U_SENTINEL) {
- terminatePSKLevel(level, maxLevel, i, dest);
- level = UCOL_PSK_NULL;
- break;
- }
- bocsuBytesWritten = u_writeIdenticalLevelRunTwoChars(first, second, buff);
- first = second;
-
- j = 0;
- if(bocsuBytesUsed != 0) {
- while(bocsuBytesUsed-->0) {
- j++;
- }
- }
-
- while(i < count && j < bocsuBytesWritten) {
- dest[i++] = buff[j++];
- }
- }
-
- } else {
- level = UCOL_PSK_NULL;
- }
- /* fall through to next level */
- case UCOL_PSK_NULL:
- j = i;
- while(j<count) {
- dest[j++]=0;
- }
- break;
- default:
- *status = U_INTERNAL_PROGRAM_ERROR;
- UTRACE_EXIT_STATUS(*status);
- return 0;
- }
-
-saveState:
- // Now we need to return stuff. First we want to see whether we have
- // done everything for the current state of iterator.
- if(byteCountOrFrenchDone
- || canUpdateState == FALSE
- || (newState = s.iterator->getState(s.iterator)) == UITER_NO_STATE)
- {
- // Any of above mean that the previous transaction
- // wasn't finished and that we should store the
- // previous iterator state.
- state[0] = iterState;
- } else {
- // The transaction is complete. We will continue in the next iteration.
- state[0] = s.iterator->getState(s.iterator);
- cces = 0;
- }
- // Store the number of bocsu bytes written.
- if((bocsuBytesUsed & UCOL_PSK_BOCSU_BYTES_MASK) != bocsuBytesUsed) {
- *status = U_INDEX_OUTOFBOUNDS_ERROR;
- }
- state[1] = (bocsuBytesUsed & UCOL_PSK_BOCSU_BYTES_MASK) << UCOL_PSK_BOCSU_BYTES_SHIFT;
-
- // Next we put in the level of comparison
- state[1] |= ((level & UCOL_PSK_LEVEL_MASK) << UCOL_PSK_LEVEL_SHIFT);
-
- // If we are doing French, we need to store whether we have just finished the French level
- if(level == UCOL_PSK_SECONDARY && doingFrench) {
- state[1] |= (((state[0] == 0) & UCOL_PSK_BYTE_COUNT_OR_FRENCH_DONE_MASK) << UCOL_PSK_BYTE_COUNT_OR_FRENCH_DONE_SHIFT);
- } else {
- state[1] |= ((byteCountOrFrenchDone & UCOL_PSK_BYTE_COUNT_OR_FRENCH_DONE_MASK) << UCOL_PSK_BYTE_COUNT_OR_FRENCH_DONE_SHIFT);
- }
-
- // Was the latest CE shifted
- if(wasShifted) {
- state[1] |= 1 << UCOL_PSK_WAS_SHIFTED_SHIFT;
- }
- // Check for cces overflow
- if((cces & UCOL_PSK_CONSUMED_CES_MASK) != cces) {
- *status = U_INDEX_OUTOFBOUNDS_ERROR;
- }
- // Store cces
- state[1] |= ((cces & UCOL_PSK_CONSUMED_CES_MASK) << UCOL_PSK_CONSUMED_CES_SHIFT);
-
- // Check for French overflow
- if((usedFrench & UCOL_PSK_USED_FRENCH_MASK) != usedFrench) {
- *status = U_INDEX_OUTOFBOUNDS_ERROR;
- }
- // Store number of bytes written in the French secondary continuation sequence
- state[1] |= ((usedFrench & UCOL_PSK_USED_FRENCH_MASK) << UCOL_PSK_USED_FRENCH_SHIFT);
-
-
- // If we have used normalizing iterator, get rid of it
- if(normIter != NULL) {
- unorm_closeIter(normIter);
- }
-
- /* To avoid memory leak, free the offset buffer if necessary. */
- ucol_freeOffsetBuffer(&s);
-
- // Return number of meaningful sortkey bytes.
- UTRACE_DATA4(UTRACE_VERBOSE, "dest = %vb, state=%d %d",
- dest,i, state[0], state[1]);
- UTRACE_EXIT_VALUE(i);
- return i;
-}
-
-/**
- * Produce a bound for a given sortkey and a number of levels.
- */
-U_CAPI int32_t U_EXPORT2
-ucol_getBound(const uint8_t *source,
- int32_t sourceLength,
- UColBoundMode boundType,
- uint32_t noOfLevels,
- uint8_t *result,
- int32_t resultLength,
- UErrorCode *status)
-{
- // consistency checks
- if(status == NULL || U_FAILURE(*status)) {
- return 0;
- }
- if(source == NULL) {
- *status = U_ILLEGAL_ARGUMENT_ERROR;
- return 0;
- }
-
- int32_t sourceIndex = 0;
- // Scan the string until we skip enough of the key OR reach the end of the key
- do {
- sourceIndex++;
- if(source[sourceIndex] == UCOL_LEVELTERMINATOR) {
- noOfLevels--;
- }
- } while (noOfLevels > 0
- && (source[sourceIndex] != 0 || sourceIndex < sourceLength));
-
- if((source[sourceIndex] == 0 || sourceIndex == sourceLength)
- && noOfLevels > 0) {
- *status = U_SORT_KEY_TOO_SHORT_WARNING;
- }
-
-
- // READ ME: this code assumes that the values for boundType
- // enum will not changes. They are set so that the enum value
- // corresponds to the number of extra bytes each bound type
- // needs.
- if(result != NULL && resultLength >= sourceIndex+boundType) {
- uprv_memcpy(result, source, sourceIndex);
- switch(boundType) {
- // Lower bound just gets terminated. No extra bytes
- case UCOL_BOUND_LOWER: // = 0
- break;
- // Upper bound needs one extra byte
- case UCOL_BOUND_UPPER: // = 1
- result[sourceIndex++] = 2;
- break;
- // Upper long bound needs two extra bytes
- case UCOL_BOUND_UPPER_LONG: // = 2
- result[sourceIndex++] = 0xFF;
- result[sourceIndex++] = 0xFF;
- break;
- default:
- *status = U_ILLEGAL_ARGUMENT_ERROR;
- return 0;
- }
- result[sourceIndex++] = 0;
-
- return sourceIndex;
- } else {
- return sourceIndex+boundType+1;
- }
-}
-
-/****************************************************************************/
-/* Following are the functions that deal with the properties of a collator */
-/* there are new APIs and some compatibility APIs */
-/****************************************************************************/
-
-static inline void
-ucol_addLatinOneEntry(UCollator *coll, UChar ch, uint32_t CE,
- int32_t *primShift, int32_t *secShift, int32_t *terShift)
-{
- uint8_t primary1 = 0, primary2 = 0, secondary = 0, tertiary = 0;
- UBool reverseSecondary = FALSE;
- UBool continuation = isContinuation(CE);
- if(!continuation) {
- tertiary = (uint8_t)((CE & coll->tertiaryMask));
- tertiary ^= coll->caseSwitch;
- reverseSecondary = TRUE;
- } else {
- tertiary = (uint8_t)((CE & UCOL_REMOVE_CONTINUATION));
- tertiary &= UCOL_REMOVE_CASE;
- reverseSecondary = FALSE;
- }
-
- secondary = (uint8_t)((CE >>= 8) & UCOL_BYTE_SIZE_MASK);
- primary2 = (uint8_t)((CE >>= 8) & UCOL_BYTE_SIZE_MASK);
- primary1 = (uint8_t)(CE >> 8);
-
- if(primary1 != 0) {
- if (coll->leadBytePermutationTable != NULL && !continuation) {
- primary1 = coll->leadBytePermutationTable[primary1];
- }
-
- coll->latinOneCEs[ch] |= (primary1 << *primShift);
- *primShift -= 8;
- }
- if(primary2 != 0) {
- if(*primShift < 0) {
- coll->latinOneCEs[ch] = UCOL_BAIL_OUT_CE;
- coll->latinOneCEs[coll->latinOneTableLen+ch] = UCOL_BAIL_OUT_CE;
- coll->latinOneCEs[2*coll->latinOneTableLen+ch] = UCOL_BAIL_OUT_CE;
- return;
- }
- coll->latinOneCEs[ch] |= (primary2 << *primShift);
- *primShift -= 8;
- }
- if(secondary != 0) {
- if(reverseSecondary && coll->frenchCollation == UCOL_ON) { // reverse secondary
- coll->latinOneCEs[coll->latinOneTableLen+ch] >>= 8; // make space for secondary
- coll->latinOneCEs[coll->latinOneTableLen+ch] |= (secondary << 24);
- } else { // normal case
- coll->latinOneCEs[coll->latinOneTableLen+ch] |= (secondary << *secShift);
- }
- *secShift -= 8;
- }
- if(tertiary != 0) {
- coll->latinOneCEs[2*coll->latinOneTableLen+ch] |= (tertiary << *terShift);
- *terShift -= 8;
- }
-}
-
-static inline UBool
-ucol_resizeLatinOneTable(UCollator *coll, int32_t size, UErrorCode *status) {
- uint32_t *newTable = (uint32_t *)uprv_malloc(size*sizeof(uint32_t)*3);
- if(newTable == NULL) {
- *status = U_MEMORY_ALLOCATION_ERROR;
- coll->latinOneFailed = TRUE;
- return FALSE;
- }
- int32_t sizeToCopy = ((size<coll->latinOneTableLen)?size:coll->latinOneTableLen)*sizeof(uint32_t);
- uprv_memset(newTable, 0, size*sizeof(uint32_t)*3);
- uprv_memcpy(newTable, coll->latinOneCEs, sizeToCopy);
- uprv_memcpy(newTable+size, coll->latinOneCEs+coll->latinOneTableLen, sizeToCopy);
- uprv_memcpy(newTable+2*size, coll->latinOneCEs+2*coll->latinOneTableLen, sizeToCopy);
- coll->latinOneTableLen = size;
- uprv_free(coll->latinOneCEs);
- coll->latinOneCEs = newTable;
- return TRUE;
-}
-
-static UBool
-ucol_setUpLatinOne(UCollator *coll, UErrorCode *status) {
- UBool result = TRUE;
- if(coll->latinOneCEs == NULL) {
- coll->latinOneCEs = (uint32_t *)uprv_malloc(sizeof(uint32_t)*UCOL_LATINONETABLELEN*3);
- if(coll->latinOneCEs == NULL) {
- *status = U_MEMORY_ALLOCATION_ERROR;
- return FALSE;
- }
- coll->latinOneTableLen = UCOL_LATINONETABLELEN;
- }
- UChar ch = 0;
- UCollationElements *it = ucol_openElements(coll, &ch, 1, status);
- // Check for null pointer
- if (U_FAILURE(*status)) {
- return FALSE;
- }
- uprv_memset(coll->latinOneCEs, 0, sizeof(uint32_t)*coll->latinOneTableLen*3);
-
- int32_t primShift = 24, secShift = 24, terShift = 24;
- uint32_t CE = 0;
- int32_t contractionOffset = UCOL_ENDOFLATINONERANGE+1;
-
- // TODO: make safe if you get more than you wanted...
- for(ch = 0; ch <= UCOL_ENDOFLATINONERANGE; ch++) {
- primShift = 24; secShift = 24; terShift = 24;
- if(ch < 0x100) {
- CE = coll->latinOneMapping[ch];
- } else {
- CE = UTRIE_GET32_FROM_LEAD(&coll->mapping, ch);
- if(CE == UCOL_NOT_FOUND && coll->UCA) {
- CE = UTRIE_GET32_FROM_LEAD(&coll->UCA->mapping, ch);
- }
- }
- if(CE < UCOL_NOT_FOUND) {
- ucol_addLatinOneEntry(coll, ch, CE, &primShift, &secShift, &terShift);
- } else {
- switch (getCETag(CE)) {
- case EXPANSION_TAG:
- case DIGIT_TAG:
- ucol_setText(it, &ch, 1, status);
- while((int32_t)(CE = ucol_next(it, status)) != UCOL_NULLORDER) {
- if(primShift < 0 || secShift < 0 || terShift < 0) {
- coll->latinOneCEs[ch] = UCOL_BAIL_OUT_CE;
- coll->latinOneCEs[coll->latinOneTableLen+ch] = UCOL_BAIL_OUT_CE;
- coll->latinOneCEs[2*coll->latinOneTableLen+ch] = UCOL_BAIL_OUT_CE;
- break;
- }
- ucol_addLatinOneEntry(coll, ch, CE, &primShift, &secShift, &terShift);
- }
- break;
- case CONTRACTION_TAG:
- // here is the trick
- // F2 is contraction. We do something very similar to contractions
- // but have two indices, one in the real contraction table and the
- // other to where we stuffed things. This hopes that we don't have
- // many contractions (this should work for latin-1 tables).
- {
- if((CE & 0x00FFF000) != 0) {
- *status = U_UNSUPPORTED_ERROR;
- goto cleanup_after_failure;
- }
-
- const UChar *UCharOffset = (UChar *)coll->image+getContractOffset(CE);
-
- CE |= (contractionOffset & 0xFFF) << 12; // insert the offset in latin-1 table
-
- coll->latinOneCEs[ch] = CE;
- coll->latinOneCEs[coll->latinOneTableLen+ch] = CE;
- coll->latinOneCEs[2*coll->latinOneTableLen+ch] = CE;
-
- // We're going to jump into contraction table, pick the elements
- // and use them
- do {
- CE = *(coll->contractionCEs +
- (UCharOffset - coll->contractionIndex));
- if(CE > UCOL_NOT_FOUND && getCETag(CE) == EXPANSION_TAG) {
- uint32_t size;
- uint32_t i; /* general counter */
- uint32_t *CEOffset = (uint32_t *)coll->image+getExpansionOffset(CE); /* find the offset to expansion table */
- size = getExpansionCount(CE);
- //CE = *CEOffset++;
- if(size != 0) { /* if there are less than 16 elements in expansion, we don't terminate */
- for(i = 0; i<size; i++) {
- if(primShift < 0 || secShift < 0 || terShift < 0) {
- coll->latinOneCEs[(UChar)contractionOffset] = UCOL_BAIL_OUT_CE;
- coll->latinOneCEs[coll->latinOneTableLen+(UChar)contractionOffset] = UCOL_BAIL_OUT_CE;
- coll->latinOneCEs[2*coll->latinOneTableLen+(UChar)contractionOffset] = UCOL_BAIL_OUT_CE;
- break;
- }
- ucol_addLatinOneEntry(coll, (UChar)contractionOffset, *CEOffset++, &primShift, &secShift, &terShift);
- }
- } else { /* else, we do */
- while(*CEOffset != 0) {
- if(primShift < 0 || secShift < 0 || terShift < 0) {
- coll->latinOneCEs[(UChar)contractionOffset] = UCOL_BAIL_OUT_CE;
- coll->latinOneCEs[coll->latinOneTableLen+(UChar)contractionOffset] = UCOL_BAIL_OUT_CE;
- coll->latinOneCEs[2*coll->latinOneTableLen+(UChar)contractionOffset] = UCOL_BAIL_OUT_CE;
- break;
- }
- ucol_addLatinOneEntry(coll, (UChar)contractionOffset, *CEOffset++, &primShift, &secShift, &terShift);
- }
- }
- contractionOffset++;
- } else if(CE < UCOL_NOT_FOUND) {
- ucol_addLatinOneEntry(coll, (UChar)contractionOffset++, CE, &primShift, &secShift, &terShift);
- } else {
- coll->latinOneCEs[(UChar)contractionOffset] = UCOL_BAIL_OUT_CE;
- coll->latinOneCEs[coll->latinOneTableLen+(UChar)contractionOffset] = UCOL_BAIL_OUT_CE;
- coll->latinOneCEs[2*coll->latinOneTableLen+(UChar)contractionOffset] = UCOL_BAIL_OUT_CE;
- contractionOffset++;
- }
- UCharOffset++;
- primShift = 24; secShift = 24; terShift = 24;
- if(contractionOffset == coll->latinOneTableLen) { // we need to reallocate
- if(!ucol_resizeLatinOneTable(coll, 2*coll->latinOneTableLen, status)) {
- goto cleanup_after_failure;
- }
- }
- } while(*UCharOffset != 0xFFFF);
- }
- break;;
- case SPEC_PROC_TAG:
- {
- // 0xB7 is a precontext character defined in UCA5.1, a special
- // handle is implemeted in order to save LatinOne table for
- // most locales.
- if (ch==0xb7) {
- ucol_addLatinOneEntry(coll, ch, CE, &primShift, &secShift, &terShift);
- }
- else {
- goto cleanup_after_failure;
- }
- }
- break;
- default:
- goto cleanup_after_failure;
- }
- }
- }
- // compact table
- if(contractionOffset < coll->latinOneTableLen) {
- if(!ucol_resizeLatinOneTable(coll, contractionOffset, status)) {
- goto cleanup_after_failure;
- }
- }
- ucol_closeElements(it);
- return result;
-
-cleanup_after_failure:
- // status should already be set before arriving here.
- coll->latinOneFailed = TRUE;
- ucol_closeElements(it);
- return FALSE;
-}
-
-void ucol_updateInternalState(UCollator *coll, UErrorCode *status) {
- if(U_SUCCESS(*status)) {
- if(coll->caseFirst == UCOL_UPPER_FIRST) {
- coll->caseSwitch = UCOL_CASE_SWITCH;
- } else {
- coll->caseSwitch = UCOL_NO_CASE_SWITCH;
- }
-
- if(coll->caseLevel == UCOL_ON || coll->caseFirst == UCOL_OFF) {
- coll->tertiaryMask = UCOL_REMOVE_CASE;
- coll->tertiaryCommon = UCOL_COMMON3_NORMAL;
- coll->tertiaryAddition = (int8_t)UCOL_FLAG_BIT_MASK_CASE_SW_OFF; /* Should be 0x80 */
- coll->tertiaryTop = UCOL_COMMON_TOP3_CASE_SW_OFF;
- coll->tertiaryBottom = UCOL_COMMON_BOT3;
- } else {
- coll->tertiaryMask = UCOL_KEEP_CASE;
- coll->tertiaryAddition = UCOL_FLAG_BIT_MASK_CASE_SW_ON;
- if(coll->caseFirst == UCOL_UPPER_FIRST) {
- coll->tertiaryCommon = UCOL_COMMON3_UPPERFIRST;
- coll->tertiaryTop = UCOL_COMMON_TOP3_CASE_SW_UPPER;
- coll->tertiaryBottom = UCOL_COMMON_BOTTOM3_CASE_SW_UPPER;
- } else {
- coll->tertiaryCommon = UCOL_COMMON3_NORMAL;
- coll->tertiaryTop = UCOL_COMMON_TOP3_CASE_SW_LOWER;
- coll->tertiaryBottom = UCOL_COMMON_BOTTOM3_CASE_SW_LOWER;
- }
- }
-
- /* Set the compression values */
- uint8_t tertiaryTotal = (uint8_t)(coll->tertiaryTop - coll->tertiaryBottom - 1);
- coll->tertiaryTopCount = (uint8_t)(UCOL_PROPORTION3*tertiaryTotal); /* we multilply double with int, but need only int */
- coll->tertiaryBottomCount = (uint8_t)(tertiaryTotal - coll->tertiaryTopCount);
-
- if(coll->caseLevel == UCOL_OFF && coll->strength == UCOL_TERTIARY
- && coll->frenchCollation == UCOL_OFF && coll->alternateHandling == UCOL_NON_IGNORABLE)
- {
- coll->sortKeyGen = ucol_calcSortKeySimpleTertiary;
- } else {
- coll->sortKeyGen = ucol_calcSortKey;
- }
- if(coll->caseLevel == UCOL_OFF && coll->strength <= UCOL_TERTIARY && coll->numericCollation == UCOL_OFF
- && coll->alternateHandling == UCOL_NON_IGNORABLE && !coll->latinOneFailed)
- {
- if(coll->latinOneCEs == NULL || coll->latinOneRegenTable) {
- if(ucol_setUpLatinOne(coll, status)) { // if we succeed in building latin1 table, we'll use it
- //fprintf(stderr, "F");
- coll->latinOneUse = TRUE;
- } else {
- coll->latinOneUse = FALSE;
- }
- if(*status == U_UNSUPPORTED_ERROR) {
- *status = U_ZERO_ERROR;
- }
- } else { // latin1Table exists and it doesn't need to be regenerated, just use it
- coll->latinOneUse = TRUE;
- }
- } else {
- coll->latinOneUse = FALSE;
- }
- }
-}
-
-U_CAPI uint32_t U_EXPORT2
-ucol_setVariableTop(UCollator *coll, const UChar *varTop, int32_t len, UErrorCode *status) {
- if(U_FAILURE(*status) || coll == NULL) {
- return 0;
- }
- if(len == -1) {
- len = u_strlen(varTop);
- }
- if(len == 0) {
- *status = U_ILLEGAL_ARGUMENT_ERROR;
- return 0;
- }
-
- if(coll->delegate!=NULL) {
- return ((Collator*)coll->delegate)->setVariableTop(varTop, len, *status);
- }
-
-
- collIterate s;
- IInit_collIterate(coll, varTop, len, &s, status);
- if(U_FAILURE(*status)) {
- return 0;
- }
-
- uint32_t CE = ucol_IGetNextCE(coll, &s, status);
-
- /* here we check if we have consumed all characters */
- /* you can put in either one character or a contraction */
- /* you shouldn't put more... */
- if(s.pos != s.endp || CE == UCOL_NO_MORE_CES) {
- *status = U_CE_NOT_FOUND_ERROR;
- return 0;
- }
-
- uint32_t nextCE = ucol_IGetNextCE(coll, &s, status);
-
- if(isContinuation(nextCE) && (nextCE & UCOL_PRIMARYMASK) != 0) {
- *status = U_PRIMARY_TOO_LONG_ERROR;
- return 0;
- }
- if(coll->variableTopValue != (CE & UCOL_PRIMARYMASK)>>16) {
- coll->variableTopValueisDefault = FALSE;
- coll->variableTopValue = (CE & UCOL_PRIMARYMASK)>>16;
- }
-
- /* To avoid memory leak, free the offset buffer if necessary. */
- ucol_freeOffsetBuffer(&s);
-
- return CE & UCOL_PRIMARYMASK;
-}
-
-U_CAPI uint32_t U_EXPORT2 ucol_getVariableTop(const UCollator *coll, UErrorCode *status) {
- if(U_FAILURE(*status) || coll == NULL) {
- return 0;
- }
- if(coll->delegate!=NULL) {
- return ((const Collator*)coll->delegate)->getVariableTop(*status);
- }
- return coll->variableTopValue<<16;
-}
-
-U_CAPI void U_EXPORT2
-ucol_restoreVariableTop(UCollator *coll, const uint32_t varTop, UErrorCode *status) {
- if(U_FAILURE(*status) || coll == NULL) {
- return;
- }
-
- if(coll->variableTopValue != (varTop & UCOL_PRIMARYMASK)>>16) {
- coll->variableTopValueisDefault = FALSE;
- coll->variableTopValue = (varTop & UCOL_PRIMARYMASK)>>16;
- }
-}
-/* Attribute setter API */
-U_CAPI void U_EXPORT2
-ucol_setAttribute(UCollator *coll, UColAttribute attr, UColAttributeValue value, UErrorCode *status) {
- if(U_FAILURE(*status) || coll == NULL) {
- return;
- }
-
- if(coll->delegate != NULL) {
- ((Collator*)coll->delegate)->setAttribute(attr,value,*status);
- return;
- }
-
- UColAttributeValue oldFrench = coll->frenchCollation;
- UColAttributeValue oldCaseFirst = coll->caseFirst;
- switch(attr) {
- case UCOL_NUMERIC_COLLATION: /* sort substrings of digits as numbers */
- if(value == UCOL_ON) {
- coll->numericCollation = UCOL_ON;
- coll->numericCollationisDefault = FALSE;
- } else if (value == UCOL_OFF) {
- coll->numericCollation = UCOL_OFF;
- coll->numericCollationisDefault = FALSE;
- } else if (value == UCOL_DEFAULT) {
- coll->numericCollationisDefault = TRUE;
- coll->numericCollation = (UColAttributeValue)coll->options->numericCollation;
- } else {
- *status = U_ILLEGAL_ARGUMENT_ERROR;
- }
- break;
- case UCOL_HIRAGANA_QUATERNARY_MODE: /* special quaternary values for Hiragana */
- if(value == UCOL_ON) {
- coll->hiraganaQ = UCOL_ON;
- coll->hiraganaQisDefault = FALSE;
- } else if (value == UCOL_OFF) {
- coll->hiraganaQ = UCOL_OFF;
- coll->hiraganaQisDefault = FALSE;
- } else if (value == UCOL_DEFAULT) {
- coll->hiraganaQisDefault = TRUE;
- coll->hiraganaQ = (UColAttributeValue)coll->options->hiraganaQ;
- } else {
- *status = U_ILLEGAL_ARGUMENT_ERROR;
- }
- break;
- case UCOL_FRENCH_COLLATION: /* attribute for direction of secondary weights*/
- if(value == UCOL_ON) {
- coll->frenchCollation = UCOL_ON;
- coll->frenchCollationisDefault = FALSE;
- } else if (value == UCOL_OFF) {
- coll->frenchCollation = UCOL_OFF;
- coll->frenchCollationisDefault = FALSE;
- } else if (value == UCOL_DEFAULT) {
- coll->frenchCollationisDefault = TRUE;
- coll->frenchCollation = (UColAttributeValue)coll->options->frenchCollation;
- } else {
- *status = U_ILLEGAL_ARGUMENT_ERROR ;
- }
- break;
- case UCOL_ALTERNATE_HANDLING: /* attribute for handling variable elements*/
- if(value == UCOL_SHIFTED) {
- coll->alternateHandling = UCOL_SHIFTED;
- coll->alternateHandlingisDefault = FALSE;
- } else if (value == UCOL_NON_IGNORABLE) {
- coll->alternateHandling = UCOL_NON_IGNORABLE;
- coll->alternateHandlingisDefault = FALSE;
- } else if (value == UCOL_DEFAULT) {
- coll->alternateHandlingisDefault = TRUE;
- coll->alternateHandling = (UColAttributeValue)coll->options->alternateHandling ;
- } else {
- *status = U_ILLEGAL_ARGUMENT_ERROR ;
- }
- break;
- case UCOL_CASE_FIRST: /* who goes first, lower case or uppercase */
- if(value == UCOL_LOWER_FIRST) {
- coll->caseFirst = UCOL_LOWER_FIRST;
- coll->caseFirstisDefault = FALSE;
- } else if (value == UCOL_UPPER_FIRST) {
- coll->caseFirst = UCOL_UPPER_FIRST;
- coll->caseFirstisDefault = FALSE;
- } else if (value == UCOL_OFF) {
- coll->caseFirst = UCOL_OFF;
- coll->caseFirstisDefault = FALSE;
- } else if (value == UCOL_DEFAULT) {
- coll->caseFirst = (UColAttributeValue)coll->options->caseFirst;
- coll->caseFirstisDefault = TRUE;
- } else {
- *status = U_ILLEGAL_ARGUMENT_ERROR ;
- }
- break;
- case UCOL_CASE_LEVEL: /* do we have an extra case level */
- if(value == UCOL_ON) {
- coll->caseLevel = UCOL_ON;
- coll->caseLevelisDefault = FALSE;
- } else if (value == UCOL_OFF) {
- coll->caseLevel = UCOL_OFF;
- coll->caseLevelisDefault = FALSE;
- } else if (value == UCOL_DEFAULT) {
- coll->caseLevel = (UColAttributeValue)coll->options->caseLevel;
- coll->caseLevelisDefault = TRUE;
- } else {
- *status = U_ILLEGAL_ARGUMENT_ERROR ;
- }
- break;
- case UCOL_NORMALIZATION_MODE: /* attribute for normalization */
- if(value == UCOL_ON) {
- coll->normalizationMode = UCOL_ON;
- coll->normalizationModeisDefault = FALSE;
- initializeFCD(status);
- } else if (value == UCOL_OFF) {
- coll->normalizationMode = UCOL_OFF;
- coll->normalizationModeisDefault = FALSE;
- } else if (value == UCOL_DEFAULT) {
- coll->normalizationModeisDefault = TRUE;
- coll->normalizationMode = (UColAttributeValue)coll->options->normalizationMode;
- if(coll->normalizationMode == UCOL_ON) {
- initializeFCD(status);
- }
- } else {
- *status = U_ILLEGAL_ARGUMENT_ERROR ;
- }
- break;
- case UCOL_STRENGTH: /* attribute for strength */
- if (value == UCOL_DEFAULT) {
- coll->strengthisDefault = TRUE;
- coll->strength = (UColAttributeValue)coll->options->strength;
- } else if (value <= UCOL_IDENTICAL) {
- coll->strengthisDefault = FALSE;
- coll->strength = value;
- } else {
- *status = U_ILLEGAL_ARGUMENT_ERROR ;
- }
- break;
- case UCOL_ATTRIBUTE_COUNT:
- default:
- *status = U_ILLEGAL_ARGUMENT_ERROR;
- break;
- }
- if(oldFrench != coll->frenchCollation || oldCaseFirst != coll->caseFirst) {
- coll->latinOneRegenTable = TRUE;
- } else {
- coll->latinOneRegenTable = FALSE;
- }
- ucol_updateInternalState(coll, status);
-}
-
-U_CAPI UColAttributeValue U_EXPORT2
-ucol_getAttribute(const UCollator *coll, UColAttribute attr, UErrorCode *status) {
- if(U_FAILURE(*status) || coll == NULL) {
- return UCOL_DEFAULT;
- }
-
- if(coll->delegate != NULL) {
- return ((Collator*)coll->delegate)->getAttribute(attr,*status);
- }
-
- switch(attr) {
- case UCOL_NUMERIC_COLLATION:
- return coll->numericCollation;
- case UCOL_HIRAGANA_QUATERNARY_MODE:
- return coll->hiraganaQ;
- case UCOL_FRENCH_COLLATION: /* attribute for direction of secondary weights*/
- return coll->frenchCollation;
- case UCOL_ALTERNATE_HANDLING: /* attribute for handling variable elements*/
- return coll->alternateHandling;
- case UCOL_CASE_FIRST: /* who goes first, lower case or uppercase */
- return coll->caseFirst;
- case UCOL_CASE_LEVEL: /* do we have an extra case level */
- return coll->caseLevel;
- case UCOL_NORMALIZATION_MODE: /* attribute for normalization */
- return coll->normalizationMode;
- case UCOL_STRENGTH: /* attribute for strength */
- return coll->strength;
- case UCOL_ATTRIBUTE_COUNT:
- default:
- *status = U_ILLEGAL_ARGUMENT_ERROR;
- break;
- }
- return UCOL_DEFAULT;
-}
-
-U_CAPI void U_EXPORT2
-ucol_setStrength( UCollator *coll,
- UCollationStrength strength)
-{
- UErrorCode status = U_ZERO_ERROR;
- ucol_setAttribute(coll, UCOL_STRENGTH, strength, &status);
-}
-
-U_CAPI UCollationStrength U_EXPORT2
-ucol_getStrength(const UCollator *coll)
-{
- UErrorCode status = U_ZERO_ERROR;
- return ucol_getAttribute(coll, UCOL_STRENGTH, &status);
-}
-
-U_DRAFT int32_t U_EXPORT2
-ucol_getReorderCodes(const UCollator *coll,
- int32_t *dest,
- int32_t destCapacity,
- UErrorCode *status) {
- if (U_FAILURE(*status)) {
- return 0;
- }
-
- if(coll->delegate!=NULL) {
- return ((const Collator*)coll->delegate)->getReorderCodes(dest, destCapacity, *status);
- }
-
- if (destCapacity < 0 || (destCapacity > 0 && dest == NULL)) {
- *status = U_ILLEGAL_ARGUMENT_ERROR;
- return 0;
- }
-
-#ifdef UCOL_DEBUG
- printf("coll->reorderCodesLength = %d\n", coll->reorderCodesLength);
- printf("coll->defaultReorderCodesLength = %d\n", coll->defaultReorderCodesLength);
-#endif
-
- if (coll->reorderCodesLength > destCapacity) {
- *status = U_BUFFER_OVERFLOW_ERROR;
- return coll->reorderCodesLength;
- }
- for (int32_t i = 0; i < coll->reorderCodesLength; i++) {
- dest[i] = coll->reorderCodes[i];
- }
- return coll->reorderCodesLength;
-}
-
-U_DRAFT void U_EXPORT2
-ucol_setReorderCodes(UCollator* coll,
- const int32_t* reorderCodes,
- int32_t reorderCodesLength,
- UErrorCode *status) {
- if (U_FAILURE(*status)) {
- return;
- }
-
- if (reorderCodesLength < 0 || (reorderCodesLength > 0 && reorderCodes == NULL)) {
- *status = U_ILLEGAL_ARGUMENT_ERROR;
- return;
- }
-
- if(coll->delegate!=NULL) {
- ((Collator*)coll->delegate)->setReorderCodes(reorderCodes, reorderCodesLength, *status);
- return;
- }
-
- if (coll->reorderCodes != NULL && coll->freeReorderCodesOnClose == TRUE) {
- uprv_free(coll->reorderCodes);
- }
- coll->reorderCodes = NULL;
- coll->reorderCodesLength = 0;
- if (reorderCodesLength == 0) {
- if (coll->leadBytePermutationTable != NULL && coll->freeLeadBytePermutationTableOnClose == TRUE) {
- uprv_free(coll->leadBytePermutationTable);
- }
- coll->leadBytePermutationTable = NULL;
- return;
- }
- coll->reorderCodes = (int32_t*) uprv_malloc(reorderCodesLength * sizeof(int32_t));
- if (coll->reorderCodes == NULL) {
- *status = U_MEMORY_ALLOCATION_ERROR;
- return;
- }
- coll->freeReorderCodesOnClose = TRUE;
- for (int32_t i = 0; i < reorderCodesLength; i++) {
- coll->reorderCodes[i] = reorderCodes[i];
- }
- coll->reorderCodesLength = reorderCodesLength;
- ucol_buildPermutationTable(coll, status);
-}
-
-U_DRAFT int32_t U_EXPORT2
-ucol_getEquivalentReorderCodes(int32_t reorderCode,
- int32_t* dest,
- int32_t destCapacity,
- UErrorCode *pErrorCode) {
- bool equivalentCodesSet[USCRIPT_CODE_LIMIT];
- uint16_t leadBytes[256];
- int leadBytesCount;
- int leadByteIndex;
- int16_t reorderCodesForLeadByte[USCRIPT_CODE_LIMIT];
- int reorderCodesForLeadByteCount;
- int reorderCodeIndex;
-
- int32_t equivalentCodesCount = 0;
- int setIndex;
-
- if (U_FAILURE(*pErrorCode)) {
- return 0;
- }
-
- if (destCapacity < 0 || (destCapacity > 0 && dest == NULL)) {
- *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR;
- return 0;
- }
-
- uprv_memset(equivalentCodesSet, 0, USCRIPT_CODE_LIMIT * sizeof(bool));
-
- const UCollator* uca = ucol_initUCA(pErrorCode);
- if (U_FAILURE(*pErrorCode)) {
- return 0;
- }
- leadBytesCount = ucol_getLeadBytesForReorderCode(uca, reorderCode, leadBytes, 256);
- for (leadByteIndex = 0; leadByteIndex < leadBytesCount; leadByteIndex++) {
- reorderCodesForLeadByteCount = ucol_getReorderCodesForLeadByte(
- uca, leadBytes[leadByteIndex], reorderCodesForLeadByte, USCRIPT_CODE_LIMIT);
- for (reorderCodeIndex = 0; reorderCodeIndex < reorderCodesForLeadByteCount; reorderCodeIndex++) {
- equivalentCodesSet[reorderCodesForLeadByte[reorderCodeIndex]] = true;
- }
- }
-
- for (setIndex = 0; setIndex < USCRIPT_CODE_LIMIT; setIndex++) {
- if (equivalentCodesSet[setIndex] == true) {
- equivalentCodesCount++;
- }
- }
-
- if (destCapacity == 0) {
- return equivalentCodesCount;
- }
-
- equivalentCodesCount = 0;
- for (setIndex = 0; setIndex < USCRIPT_CODE_LIMIT; setIndex++) {
- if (equivalentCodesSet[setIndex] == true) {
- dest[equivalentCodesCount++] = setIndex;
- if (equivalentCodesCount >= destCapacity) {
- break;
- }
- }
- }
- return equivalentCodesCount;
-}
-
-
-/****************************************************************************/
-/* Following are misc functions */
-/* there are new APIs and some compatibility APIs */
-/****************************************************************************/
-
-U_CAPI void U_EXPORT2
-ucol_getVersion(const UCollator* coll,
- UVersionInfo versionInfo)
-{
- if(coll->delegate!=NULL) {
- ((const Collator*)coll->delegate)->getVersion(versionInfo);
- return;
- }
- /* RunTime version */
- uint8_t rtVersion = UCOL_RUNTIME_VERSION;
- /* Builder version*/
- uint8_t bdVersion = coll->image->version[0];
-
- /* Charset Version. Need to get the version from cnv files
- * makeconv should populate cnv files with version and
- * an api has to be provided in ucnv.h to obtain this version
- */
- uint8_t csVersion = 0;
-
- /* combine the version info */
- uint16_t cmbVersion = (uint16_t)((rtVersion<<11) | (bdVersion<<6) | (csVersion));
-
- /* Tailoring rules */
- versionInfo[0] = (uint8_t)(cmbVersion>>8);
- versionInfo[1] = (uint8_t)cmbVersion;
- versionInfo[2] = coll->image->version[1];
- if(coll->UCA) {
- /* Include the minor number when getting the UCA version. (major & 1f) << 3 | (minor & 7) */
- versionInfo[3] = (coll->UCA->image->UCAVersion[0] & 0x1f) << 3 | (coll->UCA->image->UCAVersion[1] & 0x07);
- } else {
- versionInfo[3] = 0;
- }
-}
-
-
-/* This internal API checks whether a character is tailored or not */
-U_CAPI UBool U_EXPORT2
-ucol_isTailored(const UCollator *coll, const UChar u, UErrorCode *status) {
- if(U_FAILURE(*status) || coll == NULL || coll == coll->UCA) {
- return FALSE;
- }
-
- uint32_t CE = UCOL_NOT_FOUND;
- const UChar *ContractionStart = NULL;
- if(u < 0x100) { /* latin-1 */
- CE = coll->latinOneMapping[u];
- if(coll->UCA && CE == coll->UCA->latinOneMapping[u]) {
- return FALSE;
- }
- } else { /* regular */
- CE = UTRIE_GET32_FROM_LEAD(&coll->mapping, u);
- }
-
- if(isContraction(CE)) {
- ContractionStart = (UChar *)coll->image+getContractOffset(CE);
- CE = *(coll->contractionCEs + (ContractionStart- coll->contractionIndex));
- }
-
- return (UBool)(CE != UCOL_NOT_FOUND);
-}
-
-
-/****************************************************************************/
-/* Following are the string compare functions */
-/* */
-/****************************************************************************/
-
-
-/* ucol_checkIdent internal function. Does byte level string compare. */
-/* Used by strcoll if strength == identical and strings */
-/* are otherwise equal. */
-/* */
-/* Comparison must be done on NFD normalized strings. */
-/* FCD is not good enough. */
-
-static
-UCollationResult ucol_checkIdent(collIterate *sColl, collIterate *tColl, UBool normalize, UErrorCode *status)
-{
- // When we arrive here, we can have normal strings or UCharIterators. Currently they are both
- // of same type, but that doesn't really mean that it will stay that way.
- int32_t comparison;
-
- if (sColl->flags & UCOL_USE_ITERATOR) {
- // The division for the array length may truncate the array size to
- // a little less than UNORM_ITER_SIZE, but that size is dimensioned too high
- // for all platforms anyway.
- UAlignedMemory stackNormIter1[UNORM_ITER_SIZE/sizeof(UAlignedMemory)];
- UAlignedMemory stackNormIter2[UNORM_ITER_SIZE/sizeof(UAlignedMemory)];
- UNormIterator *sNIt = NULL, *tNIt = NULL;
- sNIt = unorm_openIter(stackNormIter1, sizeof(stackNormIter1), status);
- tNIt = unorm_openIter(stackNormIter2, sizeof(stackNormIter2), status);
- sColl->iterator->move(sColl->iterator, 0, UITER_START);
- tColl->iterator->move(tColl->iterator, 0, UITER_START);
- UCharIterator *sIt = unorm_setIter(sNIt, sColl->iterator, UNORM_NFD, status);
- UCharIterator *tIt = unorm_setIter(tNIt, tColl->iterator, UNORM_NFD, status);
- comparison = u_strCompareIter(sIt, tIt, TRUE);
- unorm_closeIter(sNIt);
- unorm_closeIter(tNIt);
- } else {
- int32_t sLen = (sColl->flags & UCOL_ITER_HASLEN) ? (int32_t)(sColl->endp - sColl->string) : -1;
- const UChar *sBuf = sColl->string;
- int32_t tLen = (tColl->flags & UCOL_ITER_HASLEN) ? (int32_t)(tColl->endp - tColl->string) : -1;
- const UChar *tBuf = tColl->string;
-
- if (normalize) {
- *status = U_ZERO_ERROR;
- // Note: We could use Normalizer::compare() or similar, but for short strings
- // which may not be in FCD it might be faster to just NFD them.
- // Note: spanQuickCheckYes() + normalizeSecondAndAppend() rather than
- // NFD'ing immediately might be faster for long strings,
- // but string comparison is usually done on relatively short strings.
- sColl->nfd->normalize(UnicodeString((sColl->flags & UCOL_ITER_HASLEN) == 0, sBuf, sLen),
- sColl->writableBuffer,
- *status);
- tColl->nfd->normalize(UnicodeString((tColl->flags & UCOL_ITER_HASLEN) == 0, tBuf, tLen),
- tColl->writableBuffer,
- *status);
- if(U_FAILURE(*status)) {
- return UCOL_LESS;
- }
- comparison = sColl->writableBuffer.compareCodePointOrder(tColl->writableBuffer);
- } else {
- comparison = u_strCompare(sBuf, sLen, tBuf, tLen, TRUE);
- }
- }
-
- if (comparison < 0) {
- return UCOL_LESS;
- } else if (comparison == 0) {
- return UCOL_EQUAL;
- } else /* comparison > 0 */ {
- return UCOL_GREATER;
- }
-}
-
-/* CEBuf - A struct and some inline functions to handle the saving */
-/* of CEs in a buffer within ucol_strcoll */
-
-#define UCOL_CEBUF_SIZE 512
-typedef struct ucol_CEBuf {
- uint32_t *buf;
- uint32_t *endp;
- uint32_t *pos;
- uint32_t localArray[UCOL_CEBUF_SIZE];
-} ucol_CEBuf;
-
-
-static
-inline void UCOL_INIT_CEBUF(ucol_CEBuf *b) {
- (b)->buf = (b)->pos = (b)->localArray;
- (b)->endp = (b)->buf + UCOL_CEBUF_SIZE;
-}
-
-static
-void ucol_CEBuf_Expand(ucol_CEBuf *b, collIterate *ci, UErrorCode *status) {
- uint32_t oldSize;
- uint32_t newSize;
- uint32_t *newBuf;
-
- ci->flags |= UCOL_ITER_ALLOCATED;
- oldSize = (uint32_t)(b->pos - b->buf);
- newSize = oldSize * 2;
- newBuf = (uint32_t *)uprv_malloc(newSize * sizeof(uint32_t));
- if(newBuf == NULL) {
- *status = U_MEMORY_ALLOCATION_ERROR;
- }
- else {
- uprv_memcpy(newBuf, b->buf, oldSize * sizeof(uint32_t));
- if (b->buf != b->localArray) {
- uprv_free(b->buf);
- }
- b->buf = newBuf;
- b->endp = b->buf + newSize;
- b->pos = b->buf + oldSize;
- }
-}
-
-static
-inline void UCOL_CEBUF_PUT(ucol_CEBuf *b, uint32_t ce, collIterate *ci, UErrorCode *status) {
- if (b->pos == b->endp) {
- ucol_CEBuf_Expand(b, ci, status);
- }
- if (U_SUCCESS(*status)) {
- *(b)->pos++ = ce;
- }
-}
-
-/* This is a trick string compare function that goes in and uses sortkeys to compare */
-/* It is used when compare gets in trouble and needs to bail out */
-static UCollationResult ucol_compareUsingSortKeys(collIterate *sColl,
- collIterate *tColl,
- UErrorCode *status)
-{
- uint8_t sourceKey[UCOL_MAX_BUFFER], targetKey[UCOL_MAX_BUFFER];
- uint8_t *sourceKeyP = sourceKey;
- uint8_t *targetKeyP = targetKey;
- int32_t sourceKeyLen = UCOL_MAX_BUFFER, targetKeyLen = UCOL_MAX_BUFFER;
- const UCollator *coll = sColl->coll;
- const UChar *source = NULL;
- const UChar *target = NULL;
- int32_t result = UCOL_EQUAL;
- UnicodeString sourceString, targetString;
- int32_t sourceLength;
- int32_t targetLength;
-
- if(sColl->flags & UCOL_USE_ITERATOR) {
- sColl->iterator->move(sColl->iterator, 0, UITER_START);
- tColl->iterator->move(tColl->iterator, 0, UITER_START);
- UChar32 c;
- while((c=sColl->iterator->next(sColl->iterator))>=0) {
- sourceString.append((UChar)c);
- }
- while((c=tColl->iterator->next(tColl->iterator))>=0) {
- targetString.append((UChar)c);
- }
- source = sourceString.getBuffer();
- sourceLength = sourceString.length();
- target = targetString.getBuffer();
- targetLength = targetString.length();
- } else { // no iterators
- sourceLength = (sColl->flags&UCOL_ITER_HASLEN)?(int32_t)(sColl->endp-sColl->string):-1;
- targetLength = (tColl->flags&UCOL_ITER_HASLEN)?(int32_t)(tColl->endp-tColl->string):-1;
- source = sColl->string;
- target = tColl->string;
- }
-
-
-
- sourceKeyLen = ucol_getSortKey(coll, source, sourceLength, sourceKeyP, sourceKeyLen);
- if(sourceKeyLen > UCOL_MAX_BUFFER) {
- sourceKeyP = (uint8_t*)uprv_malloc(sourceKeyLen*sizeof(uint8_t));
- if(sourceKeyP == NULL) {
- *status = U_MEMORY_ALLOCATION_ERROR;
- goto cleanup_and_do_compare;
- }
- sourceKeyLen = ucol_getSortKey(coll, source, sourceLength, sourceKeyP, sourceKeyLen);
- }
-
- targetKeyLen = ucol_getSortKey(coll, target, targetLength, targetKeyP, targetKeyLen);
- if(targetKeyLen > UCOL_MAX_BUFFER) {
- targetKeyP = (uint8_t*)uprv_malloc(targetKeyLen*sizeof(uint8_t));
- if(targetKeyP == NULL) {
- *status = U_MEMORY_ALLOCATION_ERROR;
- goto cleanup_and_do_compare;
- }
- targetKeyLen = ucol_getSortKey(coll, target, targetLength, targetKeyP, targetKeyLen);
- }
-
- result = uprv_strcmp((const char*)sourceKeyP, (const char*)targetKeyP);
-
-cleanup_and_do_compare:
- if(sourceKeyP != NULL && sourceKeyP != sourceKey) {
- uprv_free(sourceKeyP);
- }
-
- if(targetKeyP != NULL && targetKeyP != targetKey) {
- uprv_free(targetKeyP);
- }
-
- if(result<0) {
- return UCOL_LESS;
- } else if(result>0) {
- return UCOL_GREATER;
- } else {
- return UCOL_EQUAL;
- }
-}
-
-
-static UCollationResult
-ucol_strcollRegular(collIterate *sColl, collIterate *tColl, UErrorCode *status)
-{
- U_ALIGN_CODE(16);
-
- const UCollator *coll = sColl->coll;
-
-
- // setting up the collator parameters
- UColAttributeValue strength = coll->strength;
- UBool initialCheckSecTer = (strength >= UCOL_SECONDARY);
-
- UBool checkSecTer = initialCheckSecTer;
- UBool checkTertiary = (strength >= UCOL_TERTIARY);
- UBool checkQuad = (strength >= UCOL_QUATERNARY);
- UBool checkIdent = (strength == UCOL_IDENTICAL);
- UBool checkCase = (coll->caseLevel == UCOL_ON);
- UBool isFrenchSec = (coll->frenchCollation == UCOL_ON) && checkSecTer;
- UBool shifted = (coll->alternateHandling == UCOL_SHIFTED);
- UBool qShifted = shifted && checkQuad;
- UBool doHiragana = (coll->hiraganaQ == UCOL_ON) && checkQuad;
-
- if(doHiragana && shifted) {
- return (ucol_compareUsingSortKeys(sColl, tColl, status));
- }
- uint8_t caseSwitch = coll->caseSwitch;
- uint8_t tertiaryMask = coll->tertiaryMask;
-
- // This is the lowest primary value that will not be ignored if shifted
- uint32_t LVT = (shifted)?(coll->variableTopValue<<16):0;
-
- UCollationResult result = UCOL_EQUAL;
- UCollationResult hirResult = UCOL_EQUAL;
-
- // Preparing the CE buffers. They will be filled during the primary phase
- ucol_CEBuf sCEs;
- ucol_CEBuf tCEs;
- UCOL_INIT_CEBUF(&sCEs);
- UCOL_INIT_CEBUF(&tCEs);
-
- uint32_t secS = 0, secT = 0;
- uint32_t sOrder=0, tOrder=0;
-
- // Non shifted primary processing is quite simple
- if(!shifted) {
- for(;;) {
-
- // We fetch CEs until we hit a non ignorable primary or end.
- do {
- // We get the next CE
- sOrder = ucol_IGetNextCE(coll, sColl, status);
- // Stuff it in the buffer
- UCOL_CEBUF_PUT(&sCEs, sOrder, sColl, status);
- // And keep just the primary part.
- sOrder &= UCOL_PRIMARYMASK;
- } while(sOrder == 0);
-
- // see the comments on the above block
- do {
- tOrder = ucol_IGetNextCE(coll, tColl, status);
- UCOL_CEBUF_PUT(&tCEs, tOrder, tColl, status);
- tOrder &= UCOL_PRIMARYMASK;
- } while(tOrder == 0);
-
- // if both primaries are the same
- if(sOrder == tOrder) {
- // and there are no more CEs, we advance to the next level
- if(sOrder == UCOL_NO_MORE_CES_PRIMARY) {
- break;
- }
- if(doHiragana && hirResult == UCOL_EQUAL) {
- if((sColl->flags & UCOL_WAS_HIRAGANA) != (tColl->flags & UCOL_WAS_HIRAGANA)) {
- hirResult = ((sColl->flags & UCOL_WAS_HIRAGANA) > (tColl->flags & UCOL_WAS_HIRAGANA))
- ? UCOL_LESS:UCOL_GREATER;
- }
- }
- } else {
- // only need to check one for continuation
- // if one is then the other must be or the preceding CE would be a prefix of the other
- if (coll->leadBytePermutationTable != NULL && !isContinuation(sOrder)) {
- sOrder = (coll->leadBytePermutationTable[sOrder>>24] << 24) | (sOrder & 0x00FFFFFF);
- tOrder = (coll->leadBytePermutationTable[tOrder>>24] << 24) | (tOrder & 0x00FFFFFF);
- }
- // if two primaries are different, we are done
- result = (sOrder < tOrder) ? UCOL_LESS: UCOL_GREATER;
- goto commonReturn;
- }
- } // no primary difference... do the rest from the buffers
- } else { // shifted - do a slightly more complicated processing :)
- for(;;) {
- UBool sInShifted = FALSE;
- UBool tInShifted = FALSE;
- // This version of code can be refactored. However, it seems easier to understand this way.
- // Source loop. Sam as the target loop.
- for(;;) {
- sOrder = ucol_IGetNextCE(coll, sColl, status);
- if(sOrder == UCOL_NO_MORE_CES) {
- UCOL_CEBUF_PUT(&sCEs, sOrder, sColl, status);
- break;
- } else if(sOrder == 0 || (sInShifted && (sOrder & UCOL_PRIMARYMASK) == 0)) {
- /* UCA amendment - ignore ignorables that follow shifted code points */
- continue;
- } else if(isContinuation(sOrder)) {
- if((sOrder & UCOL_PRIMARYMASK) > 0) { /* There is primary value */
- if(sInShifted) {
- sOrder = (sOrder & UCOL_PRIMARYMASK) | 0xC0; /* preserve interesting continuation */
- UCOL_CEBUF_PUT(&sCEs, sOrder, sColl, status);
- continue;
- } else {
- UCOL_CEBUF_PUT(&sCEs, sOrder, sColl, status);
- break;
- }
- } else { /* Just lower level values */
- if(sInShifted) {
- continue;
- } else {
- UCOL_CEBUF_PUT(&sCEs, sOrder, sColl, status);
- continue;
- }
- }
- } else { /* regular */
- if(coll->leadBytePermutationTable != NULL){
- sOrder = (coll->leadBytePermutationTable[sOrder>>24] << 24) | (sOrder & 0x00FFFFFF);
- }
- if((sOrder & UCOL_PRIMARYMASK) > LVT) {
- UCOL_CEBUF_PUT(&sCEs, sOrder, sColl, status);
- break;
- } else {
- if((sOrder & UCOL_PRIMARYMASK) > 0) {
- sInShifted = TRUE;
- sOrder &= UCOL_PRIMARYMASK;
- UCOL_CEBUF_PUT(&sCEs, sOrder, sColl, status);
- continue;
- } else {
- UCOL_CEBUF_PUT(&sCEs, sOrder, sColl, status);
- sInShifted = FALSE;
- continue;
- }
- }
- }
- }
- sOrder &= UCOL_PRIMARYMASK;
- sInShifted = FALSE;
-
- for(;;) {
- tOrder = ucol_IGetNextCE(coll, tColl, status);
- if(tOrder == UCOL_NO_MORE_CES) {
- UCOL_CEBUF_PUT(&tCEs, tOrder, tColl, status);
- break;
- } else if(tOrder == 0 || (tInShifted && (tOrder & UCOL_PRIMARYMASK) == 0)) {
- /* UCA amendment - ignore ignorables that follow shifted code points */
- continue;
- } else if(isContinuation(tOrder)) {
- if((tOrder & UCOL_PRIMARYMASK) > 0) { /* There is primary value */
- if(tInShifted) {
- tOrder = (tOrder & UCOL_PRIMARYMASK) | 0xC0; /* preserve interesting continuation */
- UCOL_CEBUF_PUT(&tCEs, tOrder, tColl, status);
- continue;
- } else {
- UCOL_CEBUF_PUT(&tCEs, tOrder, tColl, status);
- break;
- }
- } else { /* Just lower level values */
- if(tInShifted) {
- continue;
- } else {
- UCOL_CEBUF_PUT(&tCEs, tOrder, tColl, status);
- continue;
- }
- }
- } else { /* regular */
- if(coll->leadBytePermutationTable != NULL){
- tOrder = (coll->leadBytePermutationTable[tOrder>>24] << 24) | (tOrder & 0x00FFFFFF);
- }
- if((tOrder & UCOL_PRIMARYMASK) > LVT) {
- UCOL_CEBUF_PUT(&tCEs, tOrder, tColl, status);
- break;
- } else {
- if((tOrder & UCOL_PRIMARYMASK) > 0) {
- tInShifted = TRUE;
- tOrder &= UCOL_PRIMARYMASK;
- UCOL_CEBUF_PUT(&tCEs, tOrder, tColl, status);
- continue;
- } else {
- UCOL_CEBUF_PUT(&tCEs, tOrder, tColl, status);
- tInShifted = FALSE;
- continue;
- }
- }
- }
- }
- tOrder &= UCOL_PRIMARYMASK;
- tInShifted = FALSE;
-
- if(sOrder == tOrder) {
- /*
- if(doHiragana && hirResult == UCOL_EQUAL) {
- if((sColl.flags & UCOL_WAS_HIRAGANA) != (tColl.flags & UCOL_WAS_HIRAGANA)) {
- hirResult = ((sColl.flags & UCOL_WAS_HIRAGANA) > (tColl.flags & UCOL_WAS_HIRAGANA))
- ? UCOL_LESS:UCOL_GREATER;
- }
- }
- */
- if(sOrder == UCOL_NO_MORE_CES_PRIMARY) {
- break;
- } else {
- sOrder = 0;
- tOrder = 0;
- continue;
- }
- } else {
- result = (sOrder < tOrder) ? UCOL_LESS : UCOL_GREATER;
- goto commonReturn;
- }
- } /* no primary difference... do the rest from the buffers */
- }
-
- /* now, we're gonna reexamine collected CEs */
- uint32_t *sCE;
- uint32_t *tCE;
-
- /* This is the secondary level of comparison */
- if(checkSecTer) {
- if(!isFrenchSec) { /* normal */
- sCE = sCEs.buf;
- tCE = tCEs.buf;
- for(;;) {
- while (secS == 0) {
- secS = *(sCE++) & UCOL_SECONDARYMASK;
- }
-
- while(secT == 0) {
- secT = *(tCE++) & UCOL_SECONDARYMASK;
- }
-
- if(secS == secT) {
- if(secS == UCOL_NO_MORE_CES_SECONDARY) {
- break;
- } else {
- secS = 0; secT = 0;
- continue;
- }
- } else {
- result = (secS < secT) ? UCOL_LESS : UCOL_GREATER;
- goto commonReturn;
- }
- }
- } else { /* do the French */
- uint32_t *sCESave = NULL;
- uint32_t *tCESave = NULL;
- sCE = sCEs.pos-2; /* this could also be sCEs-- if needs to be optimized */
- tCE = tCEs.pos-2;
- for(;;) {
- while (secS == 0 && sCE >= sCEs.buf) {
- if(sCESave == NULL) {
- secS = *(sCE--);
- if(isContinuation(secS)) {
- while(isContinuation(secS = *(sCE--)))
- ;
- /* after this, secS has the start of continuation, and sCEs points before that */
- sCESave = sCE; /* we save it, so that we know where to come back AND that we need to go forward */
- sCE+=2; /* need to point to the first continuation CP */
- /* However, now you can just continue doing stuff */
- }
- } else {
- secS = *(sCE++);
- if(!isContinuation(secS)) { /* This means we have finished with this cont */
- sCE = sCESave; /* reset the pointer to before continuation */
- sCESave = NULL;
- secS = 0; /* Fetch a fresh CE before the continuation sequence. */
- continue;
- }
- }
- secS &= UCOL_SECONDARYMASK; /* remove the continuation bit */
- }
-
- while(secT == 0 && tCE >= tCEs.buf) {
- if(tCESave == NULL) {
- secT = *(tCE--);
- if(isContinuation(secT)) {
- while(isContinuation(secT = *(tCE--)))
- ;
- /* after this, secS has the start of continuation, and sCEs points before that */
- tCESave = tCE; /* we save it, so that we know where to come back AND that we need to go forward */
- tCE+=2; /* need to point to the first continuation CP */
- /* However, now you can just continue doing stuff */
- }
- } else {
- secT = *(tCE++);
- if(!isContinuation(secT)) { /* This means we have finished with this cont */
- tCE = tCESave; /* reset the pointer to before continuation */
- tCESave = NULL;
- secT = 0; /* Fetch a fresh CE before the continuation sequence. */
- continue;
- }
- }
- secT &= UCOL_SECONDARYMASK; /* remove the continuation bit */
- }
-
- if(secS == secT) {
- if(secS == UCOL_NO_MORE_CES_SECONDARY || (sCE < sCEs.buf && tCE < tCEs.buf)) {
- break;
- } else {
- secS = 0; secT = 0;
- continue;
- }
- } else {
- result = (secS < secT) ? UCOL_LESS : UCOL_GREATER;
- goto commonReturn;
- }
- }
- }
projects / apple / icu.git / blobdiff