icuSources/i18n/numparse_affixes.cpp

   1 // © 2018 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3
   4 #include "unicode/utypes.h"
   5
   6 #if !UCONFIG_NO_FORMATTING
   7
   8 // Allow implicit conversion from char16_t* to UnicodeString for this file:
   9 // Helpful in toString methods and elsewhere.
  10 #define UNISTR_FROM_STRING_EXPLICIT
  11
  12 #include "numparse_types.h"
  13 #include "numparse_affixes.h"
  14 #include "numparse_utils.h"
  15 #include "number_utils.h"
  16
  17 using namespace icu;
  18 using namespace icu::numparse;
  19 using namespace icu::numparse::impl;
  20 using namespace icu::number;
  21 using namespace icu::number::impl;
  22
  23
  24 namespace {
  25
  26 /**
  27  * Helper method to return whether the given AffixPatternMatcher equals the given pattern string.
  28  * Either both arguments must be null or the pattern string inside the AffixPatternMatcher must equal
  29  * the given pattern string.
  30  */
  31 static bool matched(const AffixPatternMatcher* affix, const UnicodeString& patternString) {
  32     return (affix == nullptr && patternString.isBogus()) ||
  33            (affix != nullptr && affix->getPattern() == patternString);
  34 }
  35
  36 /**
  37  * Helper method to return the length of the given AffixPatternMatcher. Returns 0 for null.
  38  */
  39 static int32_t length(const AffixPatternMatcher* matcher) {
  40     return matcher == nullptr ? 0 : matcher->getPattern().length();
  41 }
  42
  43 /**
  44  * Helper method to return whether (1) both lhs and rhs are null/invalid, or (2) if they are both
  45  * valid, whether they are equal according to operator==.  Similar to Java Objects.equals()
  46  */
  47 static bool equals(const AffixPatternMatcher* lhs, const AffixPatternMatcher* rhs) {
  48     if (lhs == nullptr && rhs == nullptr) {
  49         return true;
  50     }
  51     if (lhs == nullptr || rhs == nullptr) {
  52         return false;
  53     }
  54     return *lhs == *rhs;
  55 }
  56
  57 }
  58
  59
  60 AffixPatternMatcherBuilder::AffixPatternMatcherBuilder(const UnicodeString& pattern,
  61                                                        AffixTokenMatcherWarehouse& warehouse,
  62                                                        IgnorablesMatcher* ignorables)
  63         : fMatchersLen(0),
  64           fLastTypeOrCp(0),
  65           fPattern(pattern),
  66           fWarehouse(warehouse),
  67           fIgnorables(ignorables) {}
  68
  69 void AffixPatternMatcherBuilder::consumeToken(AffixPatternType type, UChar32 cp, UErrorCode& status) {
  70     // This is called by AffixUtils.iterateWithConsumer() for each token.
  71
  72     // Add an ignorables matcher between tokens except between two literals, and don't put two
  73     // ignorables matchers in a row.
  74     if (fIgnorables != nullptr && fMatchersLen > 0 &&
  75         (fLastTypeOrCp < 0 || !fIgnorables->getSet()->contains(fLastTypeOrCp))) {
  76         addMatcher(*fIgnorables);
  77     }
  78
  79     if (type != TYPE_CODEPOINT) {
  80         // Case 1: the token is a symbol.
  81         switch (type) {
  82             case TYPE_MINUS_SIGN:
  83                 addMatcher(fWarehouse.minusSign());
  84                 break;
  85             case TYPE_PLUS_SIGN:
  86                 addMatcher(fWarehouse.plusSign());
  87                 break;
  88             case TYPE_PERCENT:
  89                 addMatcher(fWarehouse.percent());
  90                 break;
  91             case TYPE_PERMILLE:
  92                 addMatcher(fWarehouse.permille());
  93                 break;
  94             case TYPE_CURRENCY_SINGLE:
  95             case TYPE_CURRENCY_DOUBLE:
  96             case TYPE_CURRENCY_TRIPLE:
  97             case TYPE_CURRENCY_QUAD:
  98             case TYPE_CURRENCY_QUINT:
  99                 // All currency symbols use the same matcher
 100                 addMatcher(fWarehouse.currency(status));
 101                 break;
 102             default:
 103                 U_ASSERT(FALSE);
 104         }
 105
 106     } else if (fIgnorables != nullptr && fIgnorables->getSet()->contains(cp)) {
 107         // Case 2: the token is an ignorable literal.
 108         // No action necessary: the ignorables matcher has already been added.
 109
 110     } else {
 111         // Case 3: the token is a non-ignorable literal.
 112         addMatcher(fWarehouse.nextCodePointMatcher(cp));
 113     }
 114     fLastTypeOrCp = type != TYPE_CODEPOINT ? type : cp;
 115 }
 116
 117 void AffixPatternMatcherBuilder::addMatcher(NumberParseMatcher& matcher) {
 118     if (fMatchersLen >= fMatchers.getCapacity()) {
 119         fMatchers.resize(fMatchersLen * 2, fMatchersLen);
 120     }
 121     fMatchers[fMatchersLen++] = &matcher;
 122 }
 123
 124 AffixPatternMatcher AffixPatternMatcherBuilder::build() {
 125     return AffixPatternMatcher(fMatchers, fMatchersLen, fPattern);
 126 }
 127
 128
 129 CodePointMatcherWarehouse::CodePointMatcherWarehouse()
 130         : codePointCount(0), codePointNumBatches(0) {}
 131
 132 CodePointMatcherWarehouse::~CodePointMatcherWarehouse() {
 133     // Delete the variable number of batches of code point matchers
 134     for (int32_t i = 0; i < codePointNumBatches; i++) {
 135         delete[] codePointsOverflow[i];
 136     }
 137 }
 138
 139 CodePointMatcherWarehouse::CodePointMatcherWarehouse(CodePointMatcherWarehouse&& src) U_NOEXCEPT
 140         : codePoints(std::move(src.codePoints)),
 141           codePointsOverflow(std::move(src.codePointsOverflow)),
 142           codePointCount(src.codePointCount),
 143           codePointNumBatches(src.codePointNumBatches) {}
 144
 145 CodePointMatcherWarehouse&
 146 CodePointMatcherWarehouse::operator=(CodePointMatcherWarehouse&& src) U_NOEXCEPT {
 147     codePoints = std::move(src.codePoints);
 148     codePointsOverflow = std::move(src.codePointsOverflow);
 149     codePointCount = src.codePointCount;
 150     codePointNumBatches = src.codePointNumBatches;
 151     return *this;
 152 }
 153
 154 NumberParseMatcher& CodePointMatcherWarehouse::nextCodePointMatcher(UChar32 cp) {
 155     if (codePointCount < CODE_POINT_STACK_CAPACITY) {
 156         return codePoints[codePointCount++] = {cp};
 157     }
 158     int32_t totalCapacity = CODE_POINT_STACK_CAPACITY + codePointNumBatches * CODE_POINT_BATCH_SIZE;
 159     if (codePointCount >= totalCapacity) {
 160         // Need a new batch
 161         auto* nextBatch = new CodePointMatcher[CODE_POINT_BATCH_SIZE];
 162         if (codePointNumBatches >= codePointsOverflow.getCapacity()) {
 163             // Need more room for storing pointers to batches
 164             codePointsOverflow.resize(codePointNumBatches * 2, codePointNumBatches);
 165         }
 166         codePointsOverflow[codePointNumBatches++] = nextBatch;
 167     }
 168     return codePointsOverflow[codePointNumBatches - 1][(codePointCount++ - CODE_POINT_STACK_CAPACITY) %
 169                                                        CODE_POINT_BATCH_SIZE] = {cp};
 170 }
 171
 172
 173 AffixTokenMatcherWarehouse::AffixTokenMatcherWarehouse(const AffixTokenMatcherSetupData* setupData)
 174         : fSetupData(setupData) {}
 175
 176 NumberParseMatcher& AffixTokenMatcherWarehouse::minusSign() {
 177     return fMinusSign = {fSetupData->dfs, true};
 178 }
 179
 180 NumberParseMatcher& AffixTokenMatcherWarehouse::plusSign() {
 181     return fPlusSign = {fSetupData->dfs, true};
 182 }
 183
 184 NumberParseMatcher& AffixTokenMatcherWarehouse::percent() {
 185     return fPercent = {fSetupData->dfs};
 186 }
 187
 188 NumberParseMatcher& AffixTokenMatcherWarehouse::permille() {
 189     return fPermille = {fSetupData->dfs};
 190 }
 191
 192 NumberParseMatcher& AffixTokenMatcherWarehouse::currency(UErrorCode& status) {
 193     return fCurrency = {fSetupData->currencySymbols, fSetupData->dfs, fSetupData->parseFlags, status};
 194 }
 195
 196 IgnorablesMatcher& AffixTokenMatcherWarehouse::ignorables() {
 197     return fSetupData->ignorables;
 198 }
 199
 200 NumberParseMatcher& AffixTokenMatcherWarehouse::nextCodePointMatcher(UChar32 cp) {
 201     return fCodePoints.nextCodePointMatcher(cp);
 202 }
 203
 204
 205 CodePointMatcher::CodePointMatcher(UChar32 cp)
 206         : fCp(cp) {}
 207
 208 bool CodePointMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode&) const {
 209     if (segment.startsWith(fCp)) {
 210         segment.adjustOffsetByCodePoint();
 211         result.setCharsConsumed(segment);
 212     }
 213     return false;
 214 }
 215
 216 bool CodePointMatcher::smokeTest(const StringSegment& segment) const {
 217     return segment.startsWith(fCp);
 218 }
 219
 220 UnicodeString CodePointMatcher::toString() const {
 221     return u"<CodePoint>";
 222 }
 223
 224
 225 AffixPatternMatcher AffixPatternMatcher::fromAffixPattern(const UnicodeString& affixPattern,
 226                                                           AffixTokenMatcherWarehouse& tokenWarehouse,
 227                                                           parse_flags_t parseFlags, bool* success,
 228                                                           UErrorCode& status) {
 229     if (affixPattern.isEmpty()) {
 230         *success = false;
 231         return {};
 232     }
 233     *success = true;
 234
 235     IgnorablesMatcher* ignorables;
 236     if (0 != (parseFlags & PARSE_FLAG_EXACT_AFFIX)) {
 237         ignorables = nullptr;
 238     } else {
 239         ignorables = &tokenWarehouse.ignorables();
 240     }
 241
 242     AffixPatternMatcherBuilder builder(affixPattern, tokenWarehouse, ignorables);
 243     AffixUtils::iterateWithConsumer(affixPattern, builder, status);
 244     return builder.build();
 245 }
 246
 247 AffixPatternMatcher::AffixPatternMatcher(MatcherArray& matchers, int32_t matchersLen,
 248                                          const UnicodeString& pattern)
 249         : ArraySeriesMatcher(matchers, matchersLen), fPattern(pattern) {}
 250
 251 UnicodeString AffixPatternMatcher::getPattern() const {
 252     return fPattern.toAliasedUnicodeString();
 253 }
 254
 255 bool AffixPatternMatcher::operator==(const AffixPatternMatcher& other) const {
 256     return fPattern == other.fPattern;
 257 }
 258
 259
 260 AffixMatcherWarehouse::AffixMatcherWarehouse(AffixTokenMatcherWarehouse* tokenWarehouse)
 261         : fTokenWarehouse(tokenWarehouse) {
 262 }
 263
 264 bool AffixMatcherWarehouse::isInteresting(const AffixPatternProvider& patternInfo,
 265                                           const IgnorablesMatcher& ignorables, parse_flags_t parseFlags,
 266                                           UErrorCode& status) {
 267     UnicodeString posPrefixString = patternInfo.getString(AffixPatternProvider::AFFIX_POS_PREFIX);
 268     UnicodeString posSuffixString = patternInfo.getString(AffixPatternProvider::AFFIX_POS_SUFFIX);
 269     UnicodeString negPrefixString;
 270     UnicodeString negSuffixString;
 271     if (patternInfo.hasNegativeSubpattern()) {
 272         negPrefixString = patternInfo.getString(AffixPatternProvider::AFFIX_NEG_PREFIX);
 273         negSuffixString = patternInfo.getString(AffixPatternProvider::AFFIX_NEG_SUFFIX);
 274     }
 275
 276     if (0 == (parseFlags & PARSE_FLAG_USE_FULL_AFFIXES) &&
 277         AffixUtils::containsOnlySymbolsAndIgnorables(posPrefixString, *ignorables.getSet(), status) &&
 278         AffixUtils::containsOnlySymbolsAndIgnorables(posSuffixString, *ignorables.getSet(), status) &&
 279         AffixUtils::containsOnlySymbolsAndIgnorables(negPrefixString, *ignorables.getSet(), status) &&
 280         AffixUtils::containsOnlySymbolsAndIgnorables(negSuffixString, *ignorables.getSet(), status)
 281         // HACK: Plus and minus sign are a special case: we accept them trailing only if they are
 282         // trailing in the pattern string.
 283         && !AffixUtils::containsType(posSuffixString, TYPE_PLUS_SIGN, status) &&
 284         !AffixUtils::containsType(posSuffixString, TYPE_MINUS_SIGN, status) &&
 285         !AffixUtils::containsType(negSuffixString, TYPE_PLUS_SIGN, status) &&
 286         !AffixUtils::containsType(negSuffixString, TYPE_MINUS_SIGN, status)) {
 287         // The affixes contain only symbols and ignorables.
 288         // No need to generate affix matchers.
 289         return false;
 290     }
 291     return true;
 292 }
 293
 294 void AffixMatcherWarehouse::createAffixMatchers(const AffixPatternProvider& patternInfo,
 295                                                 MutableMatcherCollection& output,
 296                                                 const IgnorablesMatcher& ignorables,
 297                                                 parse_flags_t parseFlags, UErrorCode& status) {
 298     if (!isInteresting(patternInfo, ignorables, parseFlags, status)) {
 299         return;
 300     }
 301
 302     // The affixes have interesting characters, or we are in strict mode.
 303     // Use initial capacity of 6, the highest possible number of AffixMatchers.
 304     UnicodeString sb;
 305     bool includeUnpaired = 0 != (parseFlags & PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES);
 306     UNumberSignDisplay signDisplay = (0 != (parseFlags & PARSE_FLAG_PLUS_SIGN_ALLOWED)) ? UNUM_SIGN_ALWAYS
 307                                                                                         : UNUM_SIGN_AUTO;
 308
 309     int32_t numAffixMatchers = 0;
 310     int32_t numAffixPatternMatchers = 0;
 311
 312     AffixPatternMatcher* posPrefix = nullptr;
 313     AffixPatternMatcher* posSuffix = nullptr;
 314
 315     // Pre-process the affix strings to resolve LDML rules like sign display.
 316     for (int8_t signum = 1; signum >= -1; signum--) {
 317         // Generate Prefix
 318         bool hasPrefix = false;
 319         PatternStringUtils::patternInfoToStringBuilder(
 320                 patternInfo, true, signum, signDisplay, StandardPlural::OTHER, false, sb);
 321         fAffixPatternMatchers[numAffixPatternMatchers] = AffixPatternMatcher::fromAffixPattern(
 322                 sb, *fTokenWarehouse, parseFlags, &hasPrefix, status);
 323         AffixPatternMatcher* prefix = hasPrefix ? &fAffixPatternMatchers[numAffixPatternMatchers++]
 324                                                 : nullptr;
 325
 326         // Generate Suffix
 327         bool hasSuffix = false;
 328         PatternStringUtils::patternInfoToStringBuilder(
 329                 patternInfo, false, signum, signDisplay, StandardPlural::OTHER, false, sb);
 330         fAffixPatternMatchers[numAffixPatternMatchers] = AffixPatternMatcher::fromAffixPattern(
 331                 sb, *fTokenWarehouse, parseFlags, &hasSuffix, status);
 332         AffixPatternMatcher* suffix = hasSuffix ? &fAffixPatternMatchers[numAffixPatternMatchers++]
 333                                                 : nullptr;
 334
 335         if (signum == 1) {
 336             posPrefix = prefix;
 337             posSuffix = suffix;
 338         } else if (equals(prefix, posPrefix) && equals(suffix, posSuffix)) {
 339             // Skip adding these matchers (we already have equivalents)
 340             continue;
 341         }
 342
 343         // Flags for setting in the ParsedNumber; the token matchers may add more.
 344         int flags = (signum == -1) ? FLAG_NEGATIVE : 0;
 345
 346         // Note: it is indeed possible for posPrefix and posSuffix to both be null.
 347         // We still need to add that matcher for strict mode to work.
 348         fAffixMatchers[numAffixMatchers++] = {prefix, suffix, flags};
 349         if (includeUnpaired && prefix != nullptr && suffix != nullptr) {
 350             // The following if statements are designed to prevent adding two identical matchers.
 351             if (signum == 1 || !equals(prefix, posPrefix)) {
 352                 fAffixMatchers[numAffixMatchers++] = {prefix, nullptr, flags};
 353             }
 354             if (signum == 1 || !equals(suffix, posSuffix)) {
 355                 fAffixMatchers[numAffixMatchers++] = {nullptr, suffix, flags};
 356             }
 357         }
 358     }
 359
 360     // Put the AffixMatchers in order, and then add them to the output.
 361     // Since there are at most 9 elements, do a simple-to-implement bubble sort.
 362     bool madeChanges;
 363     do {
 364         madeChanges = false;
 365         for (int32_t i = 1; i < numAffixMatchers; i++) {
 366             if (fAffixMatchers[i - 1].compareTo(fAffixMatchers[i]) > 0) {
 367                 madeChanges = true;
 368                 AffixMatcher temp = std::move(fAffixMatchers[i - 1]);
 369                 fAffixMatchers[i - 1] = std::move(fAffixMatchers[i]);
 370                 fAffixMatchers[i] = std::move(temp);
 371             }
 372         }
 373     } while (madeChanges);
 374
 375     for (int32_t i = 0; i < numAffixMatchers; i++) {
 376         // Enable the following line to debug affixes
 377         //std::cout << "Adding affix matcher: " << CStr(fAffixMatchers[i].toString())() << std::endl;
 378         output.addMatcher(fAffixMatchers[i]);
 379     }
 380 }
 381
 382
 383 AffixMatcher::AffixMatcher(AffixPatternMatcher* prefix, AffixPatternMatcher* suffix, result_flags_t flags)
 384         : fPrefix(prefix), fSuffix(suffix), fFlags(flags) {}
 385
 386 bool AffixMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const {
 387     if (!result.seenNumber()) {
 388         // Prefix
 389         // Do not match if:
 390         // 1. We have already seen a prefix (result.prefix != null)
 391         // 2. The prefix in this AffixMatcher is empty (prefix == null)
 392         if (!result.prefix.isBogus() || fPrefix == nullptr) {
 393             return false;
 394         }
 395
 396         // Attempt to match the prefix.
 397         int initialOffset = segment.getOffset();
 398         bool maybeMore = fPrefix->match(segment, result, status);
 399         if (initialOffset != segment.getOffset()) {
 400             result.prefix = fPrefix->getPattern();
 401         }
 402         return maybeMore;
 403
 404     } else {
 405         // Suffix
 406         // Do not match if:
 407         // 1. We have already seen a suffix (result.suffix != null)
 408         // 2. The suffix in this AffixMatcher is empty (suffix == null)
 409         // 3. The matched prefix does not equal this AffixMatcher's prefix
 410         if (!result.suffix.isBogus() || fSuffix == nullptr || !matched(fPrefix, result.prefix)) {
 411             return false;
 412         }
 413
 414         // Attempt to match the suffix.
 415         int initialOffset = segment.getOffset();
 416         bool maybeMore = fSuffix->match(segment, result, status);
 417         if (initialOffset != segment.getOffset()) {
 418             result.suffix = fSuffix->getPattern();
 419         }
 420         return maybeMore;
 421     }
 422 }
 423
 424 bool AffixMatcher::smokeTest(const StringSegment& segment) const {
 425     return (fPrefix != nullptr && fPrefix->smokeTest(segment)) ||
 426            (fSuffix != nullptr && fSuffix->smokeTest(segment));
 427 }
 428
 429 void AffixMatcher::postProcess(ParsedNumber& result) const {
 430     // Check to see if our affix is the one that was matched. If so, set the flags in the result.
 431     if (matched(fPrefix, result.prefix) && matched(fSuffix, result.suffix)) {
 432         // Fill in the result prefix and suffix with non-null values (empty string).
 433         // Used by strict mode to determine whether an entire affix pair was matched.
 434         if (result.prefix.isBogus()) {
 435             result.prefix = UnicodeString();
 436         }
 437         if (result.suffix.isBogus()) {
 438             result.suffix = UnicodeString();
 439         }
 440         result.flags |= fFlags;
 441         if (fPrefix != nullptr) {
 442             fPrefix->postProcess(result);
 443         }
 444         if (fSuffix != nullptr) {
 445             fSuffix->postProcess(result);
 446         }
 447     }
 448 }
 449
 450 int8_t AffixMatcher::compareTo(const AffixMatcher& rhs) const {
 451     const AffixMatcher& lhs = *this;
 452     if (length(lhs.fPrefix) != length(rhs.fPrefix)) {
 453         return length(lhs.fPrefix) > length(rhs.fPrefix) ? -1 : 1;
 454     } else if (length(lhs.fSuffix) != length(rhs.fSuffix)) {
 455         return length(lhs.fSuffix) > length(rhs.fSuffix) ? -1 : 1;
 456     } else {
 457         return 0;
 458     }
 459 }
 460
 461 UnicodeString AffixMatcher::toString() const {
 462     bool isNegative = 0 != (fFlags & FLAG_NEGATIVE);
 463     return UnicodeString(u"<Affix") + (isNegative ? u":negative " : u" ") +
 464            (fPrefix ? fPrefix->getPattern() : u"null") + u"#" +
 465            (fSuffix ? fSuffix->getPattern() : u"null") + u">";
 466
 467 }
 468
 469
 470 #endif /* #if !UCONFIG_NO_FORMATTING */
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495