icuSources/i18n/cpdtrans.cpp

   1 /*
   2 **********************************************************************
   3 *   Copyright (C) 1999-2011, International Business Machines
   4 *   Corporation and others.  All Rights Reserved.
   5 **********************************************************************
   6 *   Date        Name        Description
   7 *   11/17/99    aliu        Creation.
   8 **********************************************************************
   9 */
  10
  11 #include "unicode/utypes.h"
  12
  13 #if !UCONFIG_NO_TRANSLITERATION
  14
  15 #include "unicode/unifilt.h"
  16 #include "unicode/uniset.h"
  17 #include "cpdtrans.h"
  18 #include "uvector.h"
  19 #include "tridpars.h"
  20 #include "cmemory.h"
  21
  22 // keep in sync with Transliterator
  23 //static const UChar ID_SEP   = 0x002D; /*-*/
  24 static const UChar ID_DELIM = 0x003B; /*;*/
  25 static const UChar NEWLINE  = 10;
  26
  27 static const UChar COLON_COLON[] = {0x3A, 0x3A, 0}; //"::"
  28
  29 U_NAMESPACE_BEGIN
  30
  31 const UChar CompoundTransliterator::PASS_STRING[] = { 0x0025, 0x0050, 0x0061, 0x0073, 0x0073, 0 }; // "%Pass"
  32
  33 UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CompoundTransliterator)
  34
  35 /**
  36  * Constructs a new compound transliterator given an array of
  37  * transliterators.  The array of transliterators may be of any
  38  * length, including zero or one, however, useful compound
  39  * transliterators have at least two components.
  40  * @param transliterators array of <code>Transliterator</code>
  41  * objects
  42  * @param transliteratorCount The number of
  43  * <code>Transliterator</code> objects in transliterators.
  44  * @param filter the filter.  Any character for which
  45  * <tt>filter.contains()</tt> returns <tt>false</tt> will not be
  46  * altered by this transliterator.  If <tt>filter</tt> is
  47  * <tt>null</tt> then no filtering is applied.
  48  */
  49 CompoundTransliterator::CompoundTransliterator(
  50                            Transliterator* const transliterators[],
  51                            int32_t transliteratorCount,
  52                            UnicodeFilter* adoptedFilter) :
  53     Transliterator(joinIDs(transliterators, transliteratorCount), adoptedFilter),
  54     trans(0), count(0), numAnonymousRBTs(0)  {
  55     setTransliterators(transliterators, transliteratorCount);
  56 }
  57
  58 /**
  59  * Splits an ID of the form "ID;ID;..." into a compound using each
  60  * of the IDs.
  61  * @param id of above form
  62  * @param forward if false, does the list in reverse order, and
  63  * takes the inverse of each ID.
  64  */
  65 CompoundTransliterator::CompoundTransliterator(const UnicodeString& id,
  66                               UTransDirection direction,
  67                               UnicodeFilter* adoptedFilter,
  68                               UParseError& /*parseError*/,
  69                               UErrorCode& status) :
  70     Transliterator(id, adoptedFilter),
  71     trans(0), numAnonymousRBTs(0) {
  72     // TODO add code for parseError...currently unused, but
  73     // later may be used by parsing code...
  74     init(id, direction, TRUE, status);
  75 }
  76
  77 CompoundTransliterator::CompoundTransliterator(const UnicodeString& id,
  78                               UParseError& /*parseError*/,
  79                               UErrorCode& status) :
  80     Transliterator(id, 0), // set filter to 0 here!
  81     trans(0), numAnonymousRBTs(0) {
  82     // TODO add code for parseError...currently unused, but
  83     // later may be used by parsing code...
  84     init(id, UTRANS_FORWARD, TRUE, status);
  85 }
  86
  87
  88 /**
  89  * Private constructor for use of TransliteratorAlias
  90  */
  91 CompoundTransliterator::CompoundTransliterator(const UnicodeString& newID,
  92                                               UVector& list,
  93                                               UnicodeFilter* adoptedFilter,
  94                                               int32_t anonymousRBTs,
  95                                               UParseError& /*parseError*/,
  96                                               UErrorCode& status) :
  97     Transliterator(newID, adoptedFilter),
  98     trans(0), numAnonymousRBTs(anonymousRBTs)
  99 {
 100     init(list, UTRANS_FORWARD, FALSE, status);
 101 }
 102
 103 /**
 104  * Private constructor for Transliterator from a vector of
 105  * transliterators.  The caller is responsible for fixing up the
 106  * ID.
 107  */
 108 CompoundTransliterator::CompoundTransliterator(UVector& list,
 109                                                UParseError& /*parseError*/,
 110                                                UErrorCode& status) :
 111     Transliterator(UnicodeString(), NULL),
 112     trans(0), numAnonymousRBTs(0)
 113 {
 114     // TODO add code for parseError...currently unused, but
 115     // later may be used by parsing code...
 116     init(list, UTRANS_FORWARD, FALSE, status);
 117     // assume caller will fixup ID
 118 }
 119
 120 CompoundTransliterator::CompoundTransliterator(UVector& list,
 121                                                int32_t anonymousRBTs,
 122                                                UParseError& /*parseError*/,
 123                                                UErrorCode& status) :
 124     Transliterator(UnicodeString(), NULL),
 125     trans(0), numAnonymousRBTs(anonymousRBTs)
 126 {
 127     init(list, UTRANS_FORWARD, FALSE, status);
 128 }
 129
 130 /**
 131  * Finish constructing a transliterator: only to be called by
 132  * constructors.  Before calling init(), set trans and filter to NULL.
 133  * @param id the id containing ';'-separated entries
 134  * @param direction either FORWARD or REVERSE
 135  * @param idSplitPoint the index into id at which the
 136  * adoptedSplitTransliterator should be inserted, if there is one, or
 137  * -1 if there is none.
 138  * @param adoptedSplitTransliterator a transliterator to be inserted
 139  * before the entry at offset idSplitPoint in the id string.  May be
 140  * NULL to insert no entry.
 141  * @param fixReverseID if TRUE, then reconstruct the ID of reverse
 142  * entries by calling getID() of component entries.  Some constructors
 143  * do not require this because they apply a facade ID anyway.
 144  * @param status the error code indicating success or failure
 145  */
 146 void CompoundTransliterator::init(const UnicodeString& id,
 147                                   UTransDirection direction,
 148                                   UBool fixReverseID,
 149                                   UErrorCode& status) {
 150     // assert(trans == 0);
 151
 152     if (U_FAILURE(status)) {
 153         return;
 154     }
 155
 156     UVector list(status);
 157     UnicodeSet* compoundFilter = NULL;
 158     UnicodeString regenID;
 159     if (!TransliteratorIDParser::parseCompoundID(id, direction,
 160                                       regenID, list, compoundFilter)) {
 161         status = U_INVALID_ID;
 162         delete compoundFilter;
 163         return;
 164     }
 165
 166     TransliteratorIDParser::instantiateList(list, status);
 167
 168     init(list, direction, fixReverseID, status);
 169
 170     if (compoundFilter != NULL) {
 171         adoptFilter(compoundFilter);
 172     }
 173 }
 174
 175 /**
 176  * Finish constructing a transliterator: only to be called by
 177  * constructors.  Before calling init(), set trans and filter to NULL.
 178  * @param list a vector of transliterator objects to be adopted.  It
 179  * should NOT be empty.  The list should be in declared order.  That
 180  * is, it should be in the FORWARD order; if direction is REVERSE then
 181  * the list order will be reversed.
 182  * @param direction either FORWARD or REVERSE
 183  * @param fixReverseID if TRUE, then reconstruct the ID of reverse
 184  * entries by calling getID() of component entries.  Some constructors
 185  * do not require this because they apply a facade ID anyway.
 186  * @param status the error code indicating success or failure
 187  */
 188 void CompoundTransliterator::init(UVector& list,
 189                                   UTransDirection direction,
 190                                   UBool fixReverseID,
 191                                   UErrorCode& status) {
 192     // assert(trans == 0);
 193
 194     // Allocate array
 195     if (U_SUCCESS(status)) {
 196         count = list.size();
 197         trans = (Transliterator **)uprv_malloc(count * sizeof(Transliterator *));
 198         /* test for NULL */
 199         if (trans == 0) {
 200             status = U_MEMORY_ALLOCATION_ERROR;
 201             return;
 202         }
 203     }
 204
 205     if (U_FAILURE(status) || trans == 0) {
 206          // assert(trans == 0);
 207         return;
 208     }
 209
 210     // Move the transliterators from the vector into an array.
 211     // Reverse the order if necessary.
 212     int32_t i;
 213     for (i=0; i<count; ++i) {
 214         int32_t j = (direction == UTRANS_FORWARD) ? i : count - 1 - i;
 215         trans[i] = (Transliterator*) list.elementAt(j);
 216     }
 217
 218     // If the direction is UTRANS_REVERSE then we may need to fix the
 219     // ID.
 220     if (direction == UTRANS_REVERSE && fixReverseID) {
 221         UnicodeString newID;
 222         for (i=0; i<count; ++i) {
 223             if (i > 0) {
 224                 newID.append(ID_DELIM);
 225             }
 226             newID.append(trans[i]->getID());
 227         }
 228         setID(newID);
 229     }
 230
 231     computeMaximumContextLength();
 232 }
 233
 234 /**
 235  * Return the IDs of the given list of transliterators, concatenated
 236  * with ID_DELIM delimiting them.  Equivalent to the perlish expression
 237  * join(ID_DELIM, map($_.getID(), transliterators).
 238  */
 239 UnicodeString CompoundTransliterator::joinIDs(Transliterator* const transliterators[],
 240                                               int32_t transCount) {
 241     UnicodeString id;
 242     for (int32_t i=0; i<transCount; ++i) {
 243         if (i > 0) {
 244             id.append(ID_DELIM);
 245         }
 246         id.append(transliterators[i]->getID());
 247     }
 248     return id; // Return temporary
 249 }
 250
 251 /**
 252  * Copy constructor.
 253  */
 254 CompoundTransliterator::CompoundTransliterator(const CompoundTransliterator& t) :
 255     Transliterator(t), trans(0), count(0), numAnonymousRBTs(-1) {
 256     *this = t;
 257 }
 258
 259 /**
 260  * Destructor
 261  */
 262 CompoundTransliterator::~CompoundTransliterator() {
 263     freeTransliterators();
 264 }
 265
 266 void CompoundTransliterator::freeTransliterators(void) {
 267     if (trans != 0) {
 268         for (int32_t i=0; i<count; ++i) {
 269             delete trans[i];
 270         }
 271         uprv_free(trans);
 272     }
 273     trans = 0;
 274     count = 0;
 275 }
 276
 277 /**
 278  * Assignment operator.
 279  */
 280 CompoundTransliterator& CompoundTransliterator::operator=(
 281                                              const CompoundTransliterator& t)
 282 {
 283     Transliterator::operator=(t);
 284     int32_t i = 0;
 285     UBool failed = FALSE;
 286     if (trans != NULL) {
 287         for (i=0; i<count; ++i) {
 288             delete trans[i];
 289             trans[i] = 0;
 290         }
 291     }
 292     if (t.count > count) {
 293         if (trans != NULL) {
 294             uprv_free(trans);
 295         }
 296         trans = (Transliterator **)uprv_malloc(t.count * sizeof(Transliterator *));
 297     }
 298     count = t.count;
 299     if (trans != NULL) {
 300         for (i=0; i<count; ++i) {
 301             trans[i] = t.trans[i]->clone();
 302             if (trans[i] == NULL) {
 303                 failed = TRUE;
 304                 break;
 305             }
 306         }
 307     }
 308
 309     // if memory allocation failed delete backwards trans array
 310     if (failed && i > 0) {
 311         int32_t n;
 312         for (n = i-1; n >= 0; n--) {
 313             uprv_free(trans[n]);
 314             trans[n] = NULL;
 315         }
 316     }
 317     numAnonymousRBTs = t.numAnonymousRBTs;
 318     return *this;
 319 }
 320
 321 /**
 322  * Transliterator API.
 323  */
 324 Transliterator* CompoundTransliterator::clone(void) const {
 325     return new CompoundTransliterator(*this);
 326 }
 327
 328 /**
 329  * Returns the number of transliterators in this chain.
 330  * @return number of transliterators in this chain.
 331  */
 332 int32_t CompoundTransliterator::getCount(void) const {
 333     return count;
 334 }
 335
 336 /**
 337  * Returns the transliterator at the given index in this chain.
 338  * @param index index into chain, from 0 to <code>getCount() - 1</code>
 339  * @return transliterator at the given index
 340  */
 341 const Transliterator& CompoundTransliterator::getTransliterator(int32_t index) const {
 342     return *trans[index];
 343 }
 344
 345 void CompoundTransliterator::setTransliterators(Transliterator* const transliterators[],
 346                                                 int32_t transCount) {
 347     Transliterator** a = (Transliterator **)uprv_malloc(transCount * sizeof(Transliterator *));
 348     if (a == NULL) {
 349         return;
 350     }
 351     int32_t i = 0;
 352     UBool failed = FALSE;
 353     for (i=0; i<transCount; ++i) {
 354         a[i] = transliterators[i]->clone();
 355         if (a[i] == NULL) {
 356             failed = TRUE;
 357             break;
 358         }
 359     }
 360     if (failed && i > 0) {
 361         int32_t n;
 362         for (n = i-1; n >= 0; n--) {
 363             uprv_free(a[n]);
 364             a[n] = NULL;
 365         }
 366         return;
 367     }
 368     adoptTransliterators(a, transCount);
 369 }
 370
 371 void CompoundTransliterator::adoptTransliterators(Transliterator* adoptedTransliterators[],
 372                                                   int32_t transCount) {
 373     // First free trans[] and set count to zero.  Once this is done,
 374     // orphan the filter.  Set up the new trans[].
 375     freeTransliterators();
 376     trans = adoptedTransliterators;
 377     count = transCount;
 378     computeMaximumContextLength();
 379     setID(joinIDs(trans, count));
 380 }
 381
 382 /**
 383  * Append c to buf, unless buf is empty or buf already ends in c.
 384  */
 385 static void _smartAppend(UnicodeString& buf, UChar c) {
 386     if (buf.length() != 0 &&
 387         buf.charAt(buf.length() - 1) != c) {
 388         buf.append(c);
 389     }
 390 }
 391
 392 UnicodeString& CompoundTransliterator::toRules(UnicodeString& rulesSource,
 393                                                UBool escapeUnprintable) const {
 394     // We do NOT call toRules() on our component transliterators, in
 395     // general.  If we have several rule-based transliterators, this
 396     // yields a concatenation of the rules -- not what we want.  We do
 397     // handle compound RBT transliterators specially -- those for which
 398     // compoundRBTIndex >= 0.  For the transliterator at compoundRBTIndex,
 399     // we do call toRules() recursively.
 400     rulesSource.truncate(0);
 401     if (numAnonymousRBTs >= 1 && getFilter() != NULL) {
 402         // If we are a compound RBT and if we have a global
 403         // filter, then emit it at the top.
 404         UnicodeString pat;
 405         rulesSource.append(COLON_COLON, 2).append(getFilter()->toPattern(pat, escapeUnprintable)).append(ID_DELIM);
 406     }
 407     for (int32_t i=0; i<count; ++i) {
 408         UnicodeString rule;
 409
 410         // Anonymous RuleBasedTransliterators (inline rules and
 411         // ::BEGIN/::END blocks) are given IDs that begin with
 412         // "%Pass": use toRules() to write all the rules to the output
 413         // (and insert "::Null;" if we have two in a row)
 414         if (trans[i]->getID().startsWith(PASS_STRING, 5)) {
 415             trans[i]->toRules(rule, escapeUnprintable);
 416             if (numAnonymousRBTs > 1 && i > 0 && trans[i - 1]->getID().startsWith(PASS_STRING, 5))
 417                 rule = UNICODE_STRING_SIMPLE("::Null;") + rule;
 418
 419         // we also use toRules() on CompoundTransliterators (which we
 420         // check for by looking for a semicolon in the ID)-- this gets
 421         // the list of their child transliterators output in the right
 422         // format
 423         } else if (trans[i]->getID().indexOf(ID_DELIM) >= 0) {
 424             trans[i]->toRules(rule, escapeUnprintable);
 425
 426         // for everything else, use Transliterator::toRules()
 427         } else {
 428             trans[i]->Transliterator::toRules(rule, escapeUnprintable);
 429         }
 430         _smartAppend(rulesSource, NEWLINE);
 431         rulesSource.append(rule);
 432         _smartAppend(rulesSource, ID_DELIM);
 433     }
 434     return rulesSource;
 435 }
 436
 437 /**
 438  * Implement Transliterator framework
 439  */
 440 void CompoundTransliterator::handleGetSourceSet(UnicodeSet& result) const {
 441     UnicodeSet set;
 442     result.clear();
 443     for (int32_t i=0; i<count; ++i) {
 444     result.addAll(trans[i]->getSourceSet(set));
 445     // Take the example of Hiragana-Latin.  This is really
 446     // Hiragana-Katakana; Katakana-Latin.  The source set of
 447     // these two is roughly [:Hiragana:] and [:Katakana:].
 448     // But the source set for the entire transliterator is
 449     // actually [:Hiragana:] ONLY -- that is, the first
 450     // non-empty source set.
 451
 452     // This is a heuristic, and not 100% reliable.
 453     if (!result.isEmpty()) {
 454         break;
 455     }
 456     }
 457 }
 458
 459 /**
 460  * Override Transliterator framework
 461  */
 462 UnicodeSet& CompoundTransliterator::getTargetSet(UnicodeSet& result) const {
 463     UnicodeSet set;
 464     result.clear();
 465     for (int32_t i=0; i<count; ++i) {
 466     // This is a heuristic, and not 100% reliable.
 467     result.addAll(trans[i]->getTargetSet(set));
 468     }
 469     return result;
 470 }
 471
 472 /**
 473  * Implements {@link Transliterator#handleTransliterate}.
 474  */
 475 void CompoundTransliterator::handleTransliterate(Replaceable& text, UTransPosition& index,
 476                                                  UBool incremental) const {
 477     /* Call each transliterator with the same contextStart and
 478      * start, but with the limit as modified
 479      * by preceding transliterators.  The start index must be
 480      * reset for each transliterator to give each a chance to
 481      * transliterate the text.  The initial contextStart index is known
 482      * to still point to the same place after each transliterator
 483      * is called because each transliterator will not change the
 484      * text between contextStart and the initial start index.
 485      *
 486      * IMPORTANT: After the first transliterator, each subsequent
 487      * transliterator only gets to transliterate text committed by
 488      * preceding transliterators; that is, the start (output
 489      * value) of transliterator i becomes the limit (input value)
 490      * of transliterator i+1.  Finally, the overall limit is fixed
 491      * up before we return.
 492      *
 493      * Assumptions we make here:
 494      * (1) contextStart <= start <= limit <= contextLimit <= text.length()
 495      * (2) start <= start' <= limit'  ;cursor doesn't move back
 496      * (3) start <= limit'            ;text before cursor unchanged
 497      * - start' is the value of start after calling handleKT
 498      * - limit' is the value of limit after calling handleKT
 499      */
 500
 501     /**
 502      * Example: 3 transliterators.  This example illustrates the
 503      * mechanics we need to implement.  C, S, and L are the contextStart,
 504      * start, and limit.  gl is the globalLimit.  contextLimit is
 505      * equal to limit throughout.
 506      *
 507      * 1. h-u, changes hex to Unicode
 508      *
 509      *    4  7  a  d  0      4  7  a
 510      *    abc/u0061/u    =>  abca/u
 511      *    C  S       L       C   S L   gl=f->a
 512      *
 513      * 2. upup, changes "x" to "XX"
 514      *
 515      *    4  7  a       4  7  a
 516      *    abca/u    =>  abcAA/u
 517      *    C  SL         C    S
 518      *                       L    gl=a->b
 519      * 3. u-h, changes Unicode to hex
 520      *
 521      *    4  7  a        4  7  a  d  0  3
 522      *    abcAA/u    =>  abc/u0041/u0041/u
 523      *    C  S L         C              S
 524      *                                  L   gl=b->15
 525      * 4. return
 526      *
 527      *    4  7  a  d  0  3
 528      *    abc/u0041/u0041/u
 529      *    C S L
 530      */
 531
 532     if (count < 1) {
 533         index.start = index.limit;
 534         return; // Short circuit for empty compound transliterators
 535     }
 536
 537     // compoundLimit is the limit value for the entire compound
 538     // operation.  We overwrite index.limit with the previous
 539     // index.start.  After each transliteration, we update
 540     // compoundLimit for insertions or deletions that have happened.
 541     int32_t compoundLimit = index.limit;
 542
 543     // compoundStart is the start for the entire compound
 544     // operation.
 545     int32_t compoundStart = index.start;
 546
 547     int32_t delta = 0; // delta in length
 548
 549     // Give each transliterator a crack at the run of characters.
 550     // See comments at the top of the method for more detail.
 551     for (int32_t i=0; i<count; ++i) {
 552         index.start = compoundStart; // Reset start
 553         int32_t limit = index.limit;
 554
 555         if (index.start == index.limit) {
 556             // Short circuit for empty range
 557             break;
 558         }
 559
 560         trans[i]->filteredTransliterate(text, index, incremental);
 561
 562         // In a properly written transliterator, start == limit after
 563         // handleTransliterate() returns when incremental is false.
 564         // Catch cases where the subclass doesn't do this, and throw
 565         // an exception.  (Just pinning start to limit is a bad idea,
 566         // because what's probably happening is that the subclass
 567         // isn't transliterating all the way to the end, and it should
 568         // in non-incremental mode.)
 569         if (!incremental && index.start != index.limit) {
 570             // We can't throw an exception, so just fudge things
 571             index.start = index.limit;
 572         }
 573
 574         // Cumulative delta for insertions/deletions
 575         delta += index.limit - limit;
 576
 577         if (incremental) {
 578             // In the incremental case, only allow subsequent
 579             // transliterators to modify what has already been
 580             // completely processed by prior transliterators.  In the
 581             // non-incrmental case, allow each transliterator to
 582             // process the entire text.
 583             index.limit = index.start;
 584         }
 585     }
 586
 587     compoundLimit += delta;
 588
 589     // Start is good where it is -- where the last transliterator left
 590     // it.  Limit needs to be put back where it was, modulo
 591     // adjustments for deletions/insertions.
 592     index.limit = compoundLimit;
 593 }
 594
 595 /**
 596  * Sets the length of the longest context required by this transliterator.
 597  * This is <em>preceding</em> context.
 598  */
 599 void CompoundTransliterator::computeMaximumContextLength(void) {
 600     int32_t max = 0;
 601     for (int32_t i=0; i<count; ++i) {
 602         int32_t len = trans[i]->getMaximumContextLength();
 603         if (len > max) {
 604             max = len;
 605         }
 606     }
 607     setMaximumContextLength(max);
 608 }
 609
 610 U_NAMESPACE_END
 611
 612 #endif /* #if !UCONFIG_NO_TRANSLITERATION */
 613
 614 /* eof */