icuSources/i18n/affixpatternparser.cpp

   1 /*
   2  * Copyright (C) 2015, International Business Machines
   3  * Corporation and others.  All Rights Reserved.
   4  *
   5  * file name: affixpatternparser.cpp
   6  */
   7
   8 #include "unicode/utypes.h"
   9
  10 #if !UCONFIG_NO_FORMATTING
  11
  12 #include "unicode/dcfmtsym.h"
  13 #include "unicode/plurrule.h"
  14 #include "unicode/ucurr.h"
  15 #include "affixpatternparser.h"
  16 #include "charstr.h"
  17 #include "precision.h"
  18 #include "uassert.h"
  19 #include "unistrappender.h"
  20
  21         static UChar gDefaultSymbols[] = {0xa4, 0xa4, 0xa4};
  22
  23 static UChar gPercent = 0x25;
  24 static UChar gPerMill = 0x2030;
  25 static UChar gNegative = 0x2D;
  26 static UChar gPositive = 0x2B;
  27
  28 #define PACK_TOKEN_AND_LENGTH(t, l) ((UChar) (((t) << 8) | (l & 0xFF)))
  29
  30 #define UNPACK_TOKEN(c) ((AffixPattern::ETokenType) (((c) >> 8) & 0x7F))
  31
  32 #define UNPACK_LONG(c) (((c) >> 8) & 0x80)
  33
  34 #define UNPACK_LENGTH(c) ((c) & 0xFF)
  35
  36 U_NAMESPACE_BEGIN
  37
  38 static int32_t
  39 nextToken(const UChar *buffer, int32_t idx, int32_t len, UChar *token) {
  40     if (buffer[idx] != 0x27 || idx + 1 == len) {
  41         *token = buffer[idx];
  42         return 1;
  43     }
  44     *token = buffer[idx + 1];
  45     if (buffer[idx + 1] == 0xA4) {
  46         int32_t i = 2;
  47         for (; idx + i < len && i < 4 && buffer[idx + i] == buffer[idx + 1]; ++i);
  48         return i;
  49     }
  50     return 2;
  51 }
  52
  53 static int32_t
  54 nextUserToken(const UChar *buffer, int32_t idx, int32_t len, UChar *token) {
  55     *token = buffer[idx];
  56     int32_t max;
  57     switch (buffer[idx]) {
  58     case 0x27:
  59         max = 2;
  60         break;
  61     case 0xA4:
  62         max = 3;
  63         break;
  64     default:
  65         max = 1;
  66         break;
  67     }
  68     int32_t i = 1;
  69     for (; idx + i < len && i < max && buffer[idx + i] == buffer[idx]; ++i);
  70     return i;
  71 }
  72
  73 CurrencyAffixInfo::CurrencyAffixInfo()
  74         : fSymbol(gDefaultSymbols, 1),
  75           fISO(gDefaultSymbols, 2),
  76           fLong(DigitAffix(gDefaultSymbols, 3)),
  77           fIsDefault(TRUE) {
  78 }
  79
  80 void
  81 CurrencyAffixInfo::set(
  82         const char *locale,
  83         const PluralRules *rules,
  84         const UChar *currency,
  85         UErrorCode &status) {
  86     if (U_FAILURE(status)) {
  87         return;
  88     }
  89     fIsDefault = FALSE;
  90     if (currency == NULL) {
  91         fSymbol.setTo(gDefaultSymbols, 1);
  92         fISO.setTo(gDefaultSymbols, 2);
  93         fLong.remove();
  94         fLong.append(gDefaultSymbols, 3);
  95         fIsDefault = TRUE;
  96         return;
  97     }
  98     int32_t len;
  99     UBool unusedIsChoice;
 100     const UChar *symbol = ucurr_getName(
 101             currency, locale, UCURR_SYMBOL_NAME, &unusedIsChoice,
 102             &len, &status);
 103     if (U_FAILURE(status)) {
 104         return;
 105     }
 106     fSymbol.setTo(symbol, len);
 107     fISO.setTo(currency, u_strlen(currency));
 108     fLong.remove();
 109     StringEnumeration* keywords = rules->getKeywords(status);
 110     if (U_FAILURE(status)) {
 111         return;
 112     }
 113     const UnicodeString* pluralCount;
 114     while ((pluralCount = keywords->snext(status)) != NULL) {
 115         CharString pCount;
 116         pCount.appendInvariantChars(*pluralCount, status);
 117         const UChar *pluralName = ucurr_getPluralName(
 118             currency, locale, &unusedIsChoice, pCount.data(),
 119             &len, &status);
 120         fLong.setVariant(pCount.data(), UnicodeString(pluralName, len), status);
 121     }
 122     delete keywords;
 123 }
 124
 125 void
 126 CurrencyAffixInfo::adjustPrecision(
 127         const UChar *currency, const UCurrencyUsage usage,
 128         FixedPrecision &precision, UErrorCode &status) {
 129     if (U_FAILURE(status)) {
 130         return;
 131     }
 132
 133     int32_t digitCount = ucurr_getDefaultFractionDigitsForUsage(
 134             currency, usage, &status);
 135     precision.fMin.setFracDigitCount(digitCount);
 136     precision.fMax.setFracDigitCount(digitCount);
 137     double increment = ucurr_getRoundingIncrementForUsage(
 138             currency, usage, &status);
 139     if (increment == 0.0) {
 140         precision.fRoundingIncrement.clear();
 141     } else {
 142         precision.fRoundingIncrement.set(increment);
 143         // guard against round-off error
 144         precision.fRoundingIncrement.round(6);
 145     }
 146 }
 147
 148 void
 149 AffixPattern::addLiteral(
 150         const UChar *literal, int32_t start, int32_t len) {
 151     char32Count += u_countChar32(literal + start, len);
 152     literals.append(literal, start, len);
 153     int32_t tlen = tokens.length();
 154     // Takes 4 UChars to encode maximum literal length.
 155     UChar *tokenChars = tokens.getBuffer(tlen + 4);
 156
 157     // find start of literal size. May be tlen if there is no literal.
 158     // While finding start of literal size, compute literal length
 159     int32_t literalLength = 0;
 160     int32_t tLiteralStart = tlen;
 161     while (tLiteralStart > 0 && UNPACK_TOKEN(tokenChars[tLiteralStart - 1]) == kLiteral) {
 162         tLiteralStart--;
 163         literalLength <<= 8;
 164         literalLength |= UNPACK_LENGTH(tokenChars[tLiteralStart]);
 165     }
 166     // Add number of chars we just added to literal
 167     literalLength += len;
 168
 169     // Now encode the new length starting at tLiteralStart
 170     tlen = tLiteralStart;
 171     tokenChars[tlen++] = PACK_TOKEN_AND_LENGTH(kLiteral, literalLength & 0xFF);
 172     literalLength >>= 8;
 173     while (literalLength) {
 174         tokenChars[tlen++] = PACK_TOKEN_AND_LENGTH(kLiteral | 0x80, literalLength & 0xFF);
 175         literalLength >>= 8;
 176     }
 177     tokens.releaseBuffer(tlen);
 178 }
 179
 180 void
 181 AffixPattern::add(ETokenType t) {
 182     add(t, 1);
 183 }
 184
 185 void
 186 AffixPattern::addCurrency(uint8_t count) {
 187     add(kCurrency, count);
 188 }
 189
 190 void
 191 AffixPattern::add(ETokenType t, uint8_t count) {
 192     U_ASSERT(t != kLiteral);
 193     char32Count += count;
 194     switch (t) {
 195     case kCurrency:
 196         hasCurrencyToken = TRUE;
 197         break;
 198     case kPercent:
 199         hasPercentToken = TRUE;
 200         break;
 201     case kPerMill:
 202         hasPermillToken = TRUE;
 203         break;
 204     default:
 205         // Do nothing
 206         break;
 207     }
 208     tokens.append(PACK_TOKEN_AND_LENGTH(t, count));
 209 }
 210
 211 AffixPattern &
 212 AffixPattern::append(const AffixPattern &other) {
 213     AffixPatternIterator iter;
 214     other.iterator(iter);
 215     UnicodeString literal;
 216     while (iter.nextToken()) {
 217         switch (iter.getTokenType()) {
 218         case kLiteral:
 219             iter.getLiteral(literal);
 220             addLiteral(literal.getBuffer(), 0, literal.length());
 221             break;
 222         case kCurrency:
 223             addCurrency(iter.getTokenLength());
 224             break;
 225         default:
 226             add(iter.getTokenType());
 227             break;
 228         }
 229     }
 230     return *this;
 231 }
 232
 233 void
 234 AffixPattern::remove() {
 235     tokens.remove();
 236     literals.remove();
 237     hasCurrencyToken = FALSE;
 238     hasPercentToken = FALSE;
 239     hasPermillToken = FALSE;
 240     char32Count = 0;
 241 }
 242
 243 // escapes literals for strings where special characters are NOT escaped
 244 // except for apostrophe.
 245 static void escapeApostropheInLiteral(
 246         const UnicodeString &literal, UnicodeStringAppender &appender) {
 247     int32_t len = literal.length();
 248     const UChar *buffer = literal.getBuffer();
 249     for (int32_t i = 0; i < len; ++i) {
 250         UChar ch = buffer[i];
 251         switch (ch) {
 252             case 0x27:
 253                 appender.append((UChar) 0x27);
 254                 appender.append((UChar) 0x27);
 255                 break;
 256             default:
 257                 appender.append(ch);
 258                 break;
 259         }
 260     }
 261 }
 262
 263
 264 // escapes literals for user strings where special characters in literals
 265 // are escaped with apostrophe.
 266 static void escapeLiteral(
 267         const UnicodeString &literal, UnicodeStringAppender &appender) {
 268     int32_t len = literal.length();
 269     const UChar *buffer = literal.getBuffer();
 270     for (int32_t i = 0; i < len; ++i) {
 271         UChar ch = buffer[i];
 272         switch (ch) {
 273             case 0x27:
 274                 appender.append((UChar) 0x27);
 275                 appender.append((UChar) 0x27);
 276                 break;
 277             case 0x25:
 278                 appender.append((UChar) 0x27);
 279                 appender.append((UChar) 0x25);
 280                 appender.append((UChar) 0x27);
 281                 break;
 282             case 0x2030:
 283                 appender.append((UChar) 0x27);
 284                 appender.append((UChar) 0x2030);
 285                 appender.append((UChar) 0x27);
 286                 break;
 287             case 0xA4:
 288                 appender.append((UChar) 0x27);
 289                 appender.append((UChar) 0xA4);
 290                 appender.append((UChar) 0x27);
 291                 break;
 292             case 0x2D:
 293                 appender.append((UChar) 0x27);
 294                 appender.append((UChar) 0x2D);
 295                 appender.append((UChar) 0x27);
 296                 break;
 297             case 0x2B:
 298                 appender.append((UChar) 0x27);
 299                 appender.append((UChar) 0x2B);
 300                 appender.append((UChar) 0x27);
 301                 break;
 302             default:
 303                 appender.append(ch);
 304                 break;
 305         }
 306     }
 307 }
 308
 309 UnicodeString &
 310 AffixPattern::toString(UnicodeString &appendTo) const {
 311     AffixPatternIterator iter;
 312     iterator(iter);
 313     UnicodeStringAppender appender(appendTo);
 314     UnicodeString literal;
 315     while (iter.nextToken()) {
 316         switch (iter.getTokenType()) {
 317         case kLiteral:
 318             escapeApostropheInLiteral(iter.getLiteral(literal), appender);
 319             break;
 320         case kPercent:
 321             appender.append((UChar) 0x27);
 322             appender.append((UChar) 0x25);
 323             break;
 324         case kPerMill:
 325             appender.append((UChar) 0x27);
 326             appender.append((UChar) 0x2030);
 327             break;
 328         case kCurrency:
 329             {
 330                 appender.append((UChar) 0x27);
 331                 int32_t cl = iter.getTokenLength();
 332                 for (int32_t i = 0; i < cl; ++i) {
 333                     appender.append((UChar) 0xA4);
 334                 }
 335             }
 336             break;
 337         case kNegative:
 338             appender.append((UChar) 0x27);
 339             appender.append((UChar) 0x2D);
 340             break;
 341         case kPositive:
 342             appender.append((UChar) 0x27);
 343             appender.append((UChar) 0x2B);
 344             break;
 345         default:
 346             U_ASSERT(FALSE);
 347             break;
 348         }
 349     }
 350     return appendTo;
 351 }
 352
 353 UnicodeString &
 354 AffixPattern::toUserString(UnicodeString &appendTo) const {
 355     AffixPatternIterator iter;
 356     iterator(iter);
 357     UnicodeStringAppender appender(appendTo);
 358     UnicodeString literal;
 359     while (iter.nextToken()) {
 360         switch (iter.getTokenType()) {
 361         case kLiteral:
 362             escapeLiteral(iter.getLiteral(literal), appender);
 363             break;
 364         case kPercent:
 365             appender.append((UChar) 0x25);
 366             break;
 367         case kPerMill:
 368             appender.append((UChar) 0x2030);
 369             break;
 370         case kCurrency:
 371             {
 372                 int32_t cl = iter.getTokenLength();
 373                 for (int32_t i = 0; i < cl; ++i) {
 374                     appender.append((UChar) 0xA4);
 375                 }
 376             }
 377             break;
 378         case kNegative:
 379             appender.append((UChar) 0x2D);
 380             break;
 381         case kPositive:
 382             appender.append((UChar) 0x2B);
 383             break;
 384         default:
 385             U_ASSERT(FALSE);
 386             break;
 387         }
 388     }
 389     return appendTo;
 390 }
 391
 392 class AffixPatternAppender : public UMemory {
 393 public:
 394     AffixPatternAppender(AffixPattern &dest) : fDest(&dest), fIdx(0) { }
 395
 396     inline void append(UChar x) {
 397         if (fIdx == UPRV_LENGTHOF(fBuffer)) {
 398             fDest->addLiteral(fBuffer, 0, fIdx);
 399             fIdx = 0;
 400         }
 401         fBuffer[fIdx++] = x;
 402     }
 403
 404     inline void append(UChar32 x) {
 405         if (fIdx >= UPRV_LENGTHOF(fBuffer) - 1) {
 406             fDest->addLiteral(fBuffer, 0, fIdx);
 407             fIdx = 0;
 408         }
 409         U16_APPEND_UNSAFE(fBuffer, fIdx, x);
 410     }
 411
 412     inline void flush() {
 413         if (fIdx) {
 414             fDest->addLiteral(fBuffer, 0, fIdx);
 415         }
 416         fIdx = 0;
 417     }
 418
 419     /**
 420      * flush the buffer when we go out of scope.
 421      */
 422     ~AffixPatternAppender() {
 423         flush();
 424     }
 425 private:
 426     AffixPattern *fDest;
 427     int32_t fIdx;
 428     UChar fBuffer[32];
 429     AffixPatternAppender(const AffixPatternAppender &other);
 430     AffixPatternAppender &operator=(const AffixPatternAppender &other);
 431 };
 432
 433
 434 AffixPattern &
 435 AffixPattern::parseUserAffixString(
 436         const UnicodeString &affixStr,
 437         AffixPattern &appendTo,
 438         UErrorCode &status) {
 439     if (U_FAILURE(status)) {
 440         return appendTo;
 441     }
 442     int32_t len = affixStr.length();
 443     const UChar *buffer = affixStr.getBuffer();
 444     // 0 = not quoted; 1 = quoted.
 445     int32_t state = 0;
 446     AffixPatternAppender appender(appendTo);
 447     for (int32_t i = 0; i < len; ) {
 448         UChar token;
 449         int32_t tokenSize = nextUserToken(buffer, i, len, &token);
 450         i += tokenSize;
 451         if (token == 0x27 && tokenSize == 1) { // quote
 452             state = 1 - state;
 453             continue;
 454         }
 455         if (state == 0) {
 456             switch (token) {
 457             case 0x25:
 458                 appender.flush();
 459                 appendTo.add(kPercent, 1);
 460                 break;
 461             case 0x27:  // double quote
 462                 appender.append((UChar) 0x27);
 463                 break;
 464             case 0x2030:
 465                 appender.flush();
 466                 appendTo.add(kPerMill, 1);
 467                 break;
 468             case 0x2D:
 469                 appender.flush();
 470                 appendTo.add(kNegative, 1);
 471                 break;
 472             case 0x2B:
 473                 appender.flush();
 474                 appendTo.add(kPositive, 1);
 475                 break;
 476             case 0xA4:
 477                 appender.flush();
 478                 appendTo.add(kCurrency, tokenSize);
 479                 break;
 480             default:
 481                 appender.append(token);
 482                 break;
 483             }
 484         } else {
 485             switch (token) {
 486             case 0x27:  // double quote
 487                 appender.append((UChar) 0x27);
 488                 break;
 489             case 0xA4: // included b/c tokenSize can be > 1
 490                 for (int32_t j = 0; j < tokenSize; ++j) {
 491                     appender.append((UChar) 0xA4);
 492                 }
 493                 break;
 494             default:
 495                 appender.append(token);
 496                 break;
 497             }
 498         }
 499     }
 500     return appendTo;
 501 }
 502
 503 AffixPattern &
 504 AffixPattern::parseAffixString(
 505         const UnicodeString &affixStr,
 506         AffixPattern &appendTo,
 507         UErrorCode &status) {
 508     if (U_FAILURE(status)) {
 509         return appendTo;
 510     }
 511     int32_t len = affixStr.length();
 512     const UChar *buffer = affixStr.getBuffer();
 513     for (int32_t i = 0; i < len; ) {
 514         UChar token;
 515         int32_t tokenSize = nextToken(buffer, i, len, &token);
 516         if (tokenSize == 1) {
 517             int32_t literalStart = i;
 518             ++i;
 519             while (i < len && (tokenSize = nextToken(buffer, i, len, &token)) == 1) {
 520                 ++i;
 521             }
 522             appendTo.addLiteral(buffer, literalStart, i - literalStart);
 523
 524             // If we reached end of string, we are done
 525             if (i == len) {
 526                 return appendTo;
 527             }
 528         }
 529         i += tokenSize;
 530         switch (token) {
 531         case 0x25:
 532             appendTo.add(kPercent, 1);
 533             break;
 534         case 0x2030:
 535             appendTo.add(kPerMill, 1);
 536             break;
 537         case 0x2D:
 538             appendTo.add(kNegative, 1);
 539             break;
 540         case 0x2B:
 541             appendTo.add(kPositive, 1);
 542             break;
 543         case 0xA4:
 544             {
 545                 if (tokenSize - 1 > 3) {
 546                     status = U_PARSE_ERROR;
 547                     return appendTo;
 548                 }
 549                 appendTo.add(kCurrency, tokenSize - 1);
 550             }
 551             break;
 552         default:
 553             appendTo.addLiteral(&token, 0, 1);
 554             break;
 555         }
 556     }
 557     return appendTo;
 558 }
 559
 560 AffixPatternIterator &
 561 AffixPattern::iterator(AffixPatternIterator &result) const {
 562     result.nextLiteralIndex = 0;
 563     result.lastLiteralLength = 0;
 564     result.nextTokenIndex = 0;
 565     result.tokens = &tokens;
 566     result.literals = &literals;
 567     return result;
 568 }
 569
 570 UBool
 571 AffixPatternIterator::nextToken() {
 572     int32_t tlen = tokens->length();
 573     if (nextTokenIndex == tlen) {
 574         return FALSE;
 575     }
 576     ++nextTokenIndex;
 577     const UChar *tokenBuffer = tokens->getBuffer();
 578     if (UNPACK_TOKEN(tokenBuffer[nextTokenIndex - 1]) ==
 579             AffixPattern::kLiteral) {
 580         while (nextTokenIndex < tlen &&
 581                 UNPACK_LONG(tokenBuffer[nextTokenIndex])) {
 582             ++nextTokenIndex;
 583         }
 584         lastLiteralLength = 0;
 585         int32_t i = nextTokenIndex - 1;
 586         for (; UNPACK_LONG(tokenBuffer[i]); --i) {
 587             lastLiteralLength <<= 8;
 588             lastLiteralLength |= UNPACK_LENGTH(tokenBuffer[i]);
 589         }
 590         lastLiteralLength <<= 8;
 591         lastLiteralLength |= UNPACK_LENGTH(tokenBuffer[i]);
 592         nextLiteralIndex += lastLiteralLength;
 593     }
 594     return TRUE;
 595 }
 596
 597 AffixPattern::ETokenType
 598 AffixPatternIterator::getTokenType() const {
 599     return UNPACK_TOKEN(tokens->charAt(nextTokenIndex - 1));
 600 }
 601
 602 UnicodeString &
 603 AffixPatternIterator::getLiteral(UnicodeString &result) const {
 604     const UChar *buffer = literals->getBuffer();
 605     result.setTo(buffer + (nextLiteralIndex - lastLiteralLength), lastLiteralLength);
 606     return result;
 607 }
 608
 609 int32_t
 610 AffixPatternIterator::getTokenLength() const {
 611     const UChar *tokenBuffer = tokens->getBuffer();
 612     AffixPattern::ETokenType type = UNPACK_TOKEN(tokenBuffer[nextTokenIndex - 1]);
 613     return type == AffixPattern::kLiteral ? lastLiteralLength : UNPACK_LENGTH(tokenBuffer[nextTokenIndex - 1]);
 614 }
 615
 616 AffixPatternParser::AffixPatternParser()
 617         : fPercent(gPercent), fPermill(gPerMill), fNegative(gNegative), fPositive(gPositive) {
 618 }
 619
 620 AffixPatternParser::AffixPatternParser(
 621         const DecimalFormatSymbols &symbols) {
 622     setDecimalFormatSymbols(symbols);
 623 }
 624
 625 void
 626 AffixPatternParser::setDecimalFormatSymbols(
 627         const DecimalFormatSymbols &symbols) {
 628     fPercent = symbols.getConstSymbol(DecimalFormatSymbols::kPercentSymbol);
 629     fPermill = symbols.getConstSymbol(DecimalFormatSymbols::kPerMillSymbol);
 630     fNegative = symbols.getConstSymbol(DecimalFormatSymbols::kMinusSignSymbol);
 631     fPositive = symbols.getConstSymbol(DecimalFormatSymbols::kPlusSignSymbol);
 632 }
 633
 634 PluralAffix &
 635 AffixPatternParser::parse(
 636         const AffixPattern &affixPattern,
 637         const CurrencyAffixInfo &currencyAffixInfo,
 638         PluralAffix &appendTo,
 639         UErrorCode &status) const {
 640     if (U_FAILURE(status)) {
 641         return appendTo;
 642     }
 643     AffixPatternIterator iter;
 644     affixPattern.iterator(iter);
 645     UnicodeString literal;
 646     while (iter.nextToken()) {
 647         switch (iter.getTokenType()) {
 648         case AffixPattern::kPercent:
 649             appendTo.append(fPercent, UNUM_PERCENT_FIELD);
 650             break;
 651         case AffixPattern::kPerMill:
 652             appendTo.append(fPermill, UNUM_PERMILL_FIELD);
 653             break;
 654         case AffixPattern::kNegative:
 655             appendTo.append(fNegative, UNUM_SIGN_FIELD);
 656             break;
 657         case AffixPattern::kPositive:
 658             appendTo.append(fPositive, UNUM_SIGN_FIELD);
 659             break;
 660         case AffixPattern::kCurrency:
 661             switch (iter.getTokenLength()) {
 662                 case 1:
 663                     appendTo.append(
 664                             currencyAffixInfo.getSymbol(), UNUM_CURRENCY_FIELD);
 665                     break;
 666                 case 2:
 667                     appendTo.append(
 668                             currencyAffixInfo.getISO(), UNUM_CURRENCY_FIELD);
 669                     break;
 670                 case 3:
 671                     appendTo.append(
 672                             currencyAffixInfo.getLong(), UNUM_CURRENCY_FIELD, status);
 673                     break;
 674                 default:
 675                     U_ASSERT(FALSE);
 676                     break;
 677             }
 678             break;
 679         case AffixPattern::kLiteral:
 680             appendTo.append(iter.getLiteral(literal));
 681             break;
 682         default:
 683             U_ASSERT(FALSE);
 684             break;
 685         }
 686     }
 687     return appendTo;
 688 }
 689
 690
 691 U_NAMESPACE_END
 692 #endif /* #if !UCONFIG_NO_FORMATTING */