icuSources/common/rbbidata.cpp

   1 /*
   2 ***************************************************************************
   3 *   Copyright (C) 1999-2010 International Business Machines Corporation   *
   4 *   and others. All rights reserved.                                      *
   5 ***************************************************************************
   6 */
   7
   8 #include "unicode/utypes.h"
   9
  10 #if !UCONFIG_NO_BREAK_ITERATION
  11
  12 #include "unicode/utypes.h"
  13 #include "rbbidata.h"
  14 #include "rbbirb.h"
  15 #include "utrie.h"
  16 #include "udatamem.h"
  17 #include "cmemory.h"
  18 #include "cstring.h"
  19 #include "umutex.h"
  20
  21 #include "uassert.h"
  22
  23
  24 //-----------------------------------------------------------------------------------
  25 //
  26 //   Trie access folding function.  Copied as-is from properties code in uchar.c
  27 //
  28 //-----------------------------------------------------------------------------------
  29 U_CDECL_BEGIN
  30 static int32_t U_CALLCONV
  31 getFoldingOffset(uint32_t data) {
  32     /* if bit 15 is set, then the folding offset is in bits 14..0 of the 16-bit trie result */
  33     if(data&0x8000) {
  34         return (int32_t)(data&0x7fff);
  35     } else {
  36         return 0;
  37     }
  38 }
  39 U_CDECL_END
  40
  41 U_NAMESPACE_BEGIN
  42
  43 //-----------------------------------------------------------------------------
  44 //
  45 //    Constructors.
  46 //
  47 //-----------------------------------------------------------------------------
  48 RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, UErrorCode &status) {
  49     init(data, status);
  50 }
  51
  52 RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, enum EDontAdopt, UErrorCode &status) {
  53     init(data, status);
  54     fDontFreeData = TRUE;
  55 }
  56
  57 RBBIDataWrapper::RBBIDataWrapper(UDataMemory* udm, UErrorCode &status) {
  58     const RBBIDataHeader *d = (const RBBIDataHeader *)
  59         // ((char *)&(udm->pHeader->info) + udm->pHeader->info.size);
  60         // taking into consideration the padding added in by udata_write
  61         ((char *)(udm->pHeader) + udm->pHeader->dataHeader.headerSize);
  62     init(d, status);
  63     fUDataMem = udm;
  64 }
  65
  66 //-----------------------------------------------------------------------------
  67 //
  68 //    init().   Does most of the work of construction, shared between the
  69 //              constructors.
  70 //
  71 //-----------------------------------------------------------------------------
  72 void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) {
  73     if (U_FAILURE(status)) {
  74         return;
  75     }
  76     fHeader = data;
  77     if (fHeader->fMagic != 0xb1a0 || fHeader->fFormatVersion[0] != 3)
  78     {
  79         status = U_INVALID_FORMAT_ERROR;
  80         return;
  81     }
  82     // Note: in ICU version 3.2 and earlier, there was a formatVersion 1
  83     //       that is no longer supported.  At that time fFormatVersion was
  84     //       an int32_t field, rather than an array of 4 bytes.
  85
  86     fDontFreeData = FALSE;
  87     fUDataMem     = NULL;
  88     fReverseTable = NULL;
  89     fSafeFwdTable = NULL;
  90     fSafeRevTable = NULL;
  91     if (data->fFTableLen != 0) {
  92         fForwardTable = (RBBIStateTable *)((char *)data + fHeader->fFTable);
  93     }
  94     if (data->fRTableLen != 0) {
  95         fReverseTable = (RBBIStateTable *)((char *)data + fHeader->fRTable);
  96     }
  97     if (data->fSFTableLen != 0) {
  98         fSafeFwdTable = (RBBIStateTable *)((char *)data + fHeader->fSFTable);
  99     }
 100     if (data->fSRTableLen != 0) {
 101         fSafeRevTable = (RBBIStateTable *)((char *)data + fHeader->fSRTable);
 102     }
 103
 104
 105     utrie_unserialize(&fTrie,
 106                        (uint8_t *)data + fHeader->fTrie,
 107                        fHeader->fTrieLen,
 108                        &status);
 109     if (U_FAILURE(status)) {
 110         return;
 111     }
 112     fTrie.getFoldingOffset=getFoldingOffset;
 113
 114
 115     fRuleSource   = (UChar *)((char *)data + fHeader->fRuleSource);
 116     fRuleString.setTo(TRUE, fRuleSource, -1);
 117     U_ASSERT(data->fRuleSourceLen > 0);
 118
 119     fRuleStatusTable = (int32_t *)((char *)data + fHeader->fStatusTable);
 120     fStatusMaxIdx    = data->fStatusTableLen / sizeof(int32_t);
 121
 122     fRefCount = 1;
 123
 124 #ifdef RBBI_DEBUG
 125     char *debugEnv = getenv("U_RBBIDEBUG");
 126     if (debugEnv && uprv_strstr(debugEnv, "data")) {this->printData();}
 127 #endif
 128 }
 129
 130
 131 //-----------------------------------------------------------------------------
 132 //
 133 //    Destructor.     Don't call this - use removeReference() instead.
 134 //
 135 //-----------------------------------------------------------------------------
 136 RBBIDataWrapper::~RBBIDataWrapper() {
 137     U_ASSERT(fRefCount == 0);
 138     if (fUDataMem) {
 139         udata_close(fUDataMem);
 140     } else if (!fDontFreeData) {
 141         uprv_free((void *)fHeader);
 142     }
 143 }
 144
 145
 146
 147 //-----------------------------------------------------------------------------
 148 //
 149 //   Operator ==    Consider two RBBIDataWrappers to be equal if they
 150 //                  refer to the same underlying data.  Although
 151 //                  the data wrappers are normally shared between
 152 //                  iterator instances, it's possible to independently
 153 //                  open the same data twice, and get two instances, which
 154 //                  should still be ==.
 155 //
 156 //-----------------------------------------------------------------------------
 157 UBool RBBIDataWrapper::operator ==(const RBBIDataWrapper &other) const {
 158     if (fHeader == other.fHeader) {
 159         return TRUE;
 160     }
 161     if (fHeader->fLength != other.fHeader->fLength) {
 162         return FALSE;
 163     }
 164     if (uprv_memcmp(fHeader, other.fHeader, fHeader->fLength) == 0) {
 165         return TRUE;
 166     }
 167     return FALSE;
 168 }
 169
 170 int32_t  RBBIDataWrapper::hashCode() {
 171     return fHeader->fFTableLen;
 172 }
 173
 174
 175
 176 //-----------------------------------------------------------------------------
 177 //
 178 //    Reference Counting.   A single RBBIDataWrapper object is shared among
 179 //                          however many RulesBasedBreakIterator instances are
 180 //                          referencing the same data.
 181 //
 182 //-----------------------------------------------------------------------------
 183 void RBBIDataWrapper::removeReference() {
 184     if (umtx_atomic_dec(&fRefCount) == 0) {
 185         delete this;
 186     }
 187 }
 188
 189
 190 RBBIDataWrapper *RBBIDataWrapper::addReference() {
 191    umtx_atomic_inc(&fRefCount);
 192    return this;
 193 }
 194
 195
 196
 197 //-----------------------------------------------------------------------------
 198 //
 199 //  getRuleSourceString
 200 //
 201 //-----------------------------------------------------------------------------
 202 const UnicodeString &RBBIDataWrapper::getRuleSourceString() const {
 203     return fRuleString;
 204 }
 205
 206
 207 //-----------------------------------------------------------------------------
 208 //
 209 //  print   -  debugging function to dump the runtime data tables.
 210 //
 211 //-----------------------------------------------------------------------------
 212 #ifdef RBBI_DEBUG
 213 void  RBBIDataWrapper::printTable(const char *heading, const RBBIStateTable *table) {
 214     uint32_t   c;
 215     uint32_t   s;
 216
 217     RBBIDebugPrintf("   %s\n", heading);
 218
 219     RBBIDebugPrintf("State |  Acc  LA TagIx");
 220     for (c=0; c<fHeader->fCatCount; c++) {RBBIDebugPrintf("%3d ", c);}
 221     RBBIDebugPrintf("\n------|---------------"); for (c=0;c<fHeader->fCatCount; c++) {
 222         RBBIDebugPrintf("----");
 223     }
 224     RBBIDebugPrintf("\n");
 225
 226     if (table == NULL) {
 227         RBBIDebugPrintf("         N U L L   T A B L E\n\n");
 228         return;
 229     }
 230     for (s=0; s<table->fNumStates; s++) {
 231         RBBIStateTableRow *row = (RBBIStateTableRow *)
 232                                   (table->fTableData + (table->fRowLen * s));
 233         RBBIDebugPrintf("%4d  |  %3d %3d %3d ", s, row->fAccepting, row->fLookAhead, row->fTagIdx);
 234         for (c=0; c<fHeader->fCatCount; c++)  {
 235             RBBIDebugPrintf("%3d ", row->fNextState[c]);
 236         }
 237         RBBIDebugPrintf("\n");
 238     }
 239     RBBIDebugPrintf("\n");
 240 }
 241 #endif
 242
 243
 244 #ifdef RBBI_DEBUG
 245 void  RBBIDataWrapper::printData() {
 246     RBBIDebugPrintf("RBBI Data at %p\n", (void *)fHeader);
 247     RBBIDebugPrintf("   Version = {%d %d %d %d}\n", fHeader->fFormatVersion[0], fHeader->fFormatVersion[1],
 248                                                     fHeader->fFormatVersion[2], fHeader->fFormatVersion[3]);
 249     RBBIDebugPrintf("   total length of data  = %d\n", fHeader->fLength);
 250     RBBIDebugPrintf("   number of character categories = %d\n\n", fHeader->fCatCount);
 251
 252     printTable("Forward State Transition Table", fForwardTable);
 253     printTable("Reverse State Transition Table", fReverseTable);
 254     printTable("Safe Forward State Transition Table", fSafeFwdTable);
 255     printTable("Safe Reverse State Transition Table", fSafeRevTable);
 256
 257     RBBIDebugPrintf("\nOrignal Rules source:\n");
 258     for (int32_t c=0; fRuleSource[c] != 0; c++) {
 259         RBBIDebugPrintf("%c", fRuleSource[c]);
 260     }
 261     RBBIDebugPrintf("\n\n");
 262 }
 263 #endif
 264
 265
 266 U_NAMESPACE_END
 267 U_NAMESPACE_USE
 268
 269 //-----------------------------------------------------------------------------
 270 //
 271 //  ubrk_swap   -  byte swap and char encoding swap of RBBI data
 272 //
 273 //-----------------------------------------------------------------------------
 274
 275 U_CAPI int32_t U_EXPORT2
 276 ubrk_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
 277            UErrorCode *status) {
 278
 279     if (status == NULL || U_FAILURE(*status)) {
 280         return 0;
 281     }
 282     if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) {
 283         *status=U_ILLEGAL_ARGUMENT_ERROR;
 284         return 0;
 285     }
 286
 287     //
 288     //  Check that the data header is for for break data.
 289     //    (Header contents are defined in genbrk.cpp)
 290     //
 291     const UDataInfo *pInfo = (const UDataInfo *)((const char *)inData+4);
 292     if(!(  pInfo->dataFormat[0]==0x42 &&   /* dataFormat="Brk " */
 293            pInfo->dataFormat[1]==0x72 &&
 294            pInfo->dataFormat[2]==0x6b &&
 295            pInfo->dataFormat[3]==0x20 &&
 296            pInfo->formatVersion[0]==3  )) {
 297         udata_printError(ds, "ubrk_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized\n",
 298                          pInfo->dataFormat[0], pInfo->dataFormat[1],
 299                          pInfo->dataFormat[2], pInfo->dataFormat[3],
 300                          pInfo->formatVersion[0]);
 301         *status=U_UNSUPPORTED_ERROR;
 302         return 0;
 303     }
 304
 305     //
 306     // Swap the data header.  (This is the generic ICU Data Header, not the RBBI Specific
 307     //                         RBBIDataHeader).  This swap also conveniently gets us
 308     //                         the size of the ICU d.h., which lets us locate the start
 309     //                         of the RBBI specific data.
 310     //
 311     int32_t headerSize=udata_swapDataHeader(ds, inData, length, outData, status);
 312
 313
 314     //
 315     // Get the RRBI Data Header, and check that it appears to be OK.
 316     //
 317     //    Note:  ICU 3.2 and earlier, RBBIDataHeader::fDataFormat was actually
 318     //           an int32_t with a value of 1.  Starting with ICU 3.4,
 319     //           RBBI's fDataFormat matches the dataFormat field from the
 320     //           UDataInfo header, four int8_t bytes.  The value is {3,1,0,0}
 321     //
 322     const uint8_t  *inBytes =(const uint8_t *)inData+headerSize;
 323     RBBIDataHeader *rbbiDH = (RBBIDataHeader *)inBytes;
 324     if (ds->readUInt32(rbbiDH->fMagic) != 0xb1a0 ||
 325         rbbiDH->fFormatVersion[0] != 3 ||
 326         ds->readUInt32(rbbiDH->fLength)  <  sizeof(RBBIDataHeader))
 327     {
 328         udata_printError(ds, "ubrk_swap(): RBBI Data header is invalid.\n");
 329         *status=U_UNSUPPORTED_ERROR;
 330         return 0;
 331     }
 332
 333     //
 334     // Prefight operation?  Just return the size
 335     //
 336     int32_t breakDataLength = ds->readUInt32(rbbiDH->fLength);
 337     int32_t totalSize = headerSize + breakDataLength;
 338     if (length < 0) {
 339         return totalSize;
 340     }
 341
 342     //
 343     // Check that length passed in is consistent with length from RBBI data header.
 344     //
 345     if (length < totalSize) {
 346         udata_printError(ds, "ubrk_swap(): too few bytes (%d after ICU Data header) for break data.\n",
 347                             breakDataLength);
 348         *status=U_INDEX_OUTOFBOUNDS_ERROR;
 349         return 0;
 350         }
 351
 352
 353     //
 354     // Swap the Data.  Do the data itself first, then the RBBI Data Header, because
 355     //                 we need to reference the header to locate the data, and an
 356     //                 inplace swap of the header leaves it unusable.
 357     //
 358     uint8_t         *outBytes = (uint8_t *)outData + headerSize;
 359     RBBIDataHeader  *outputDH = (RBBIDataHeader *)outBytes;
 360
 361     int32_t   tableStartOffset;
 362     int32_t   tableLength;
 363
 364     //
 365     // If not swapping in place, zero out the output buffer before starting.
 366     //    Individual tables and other data items within are aligned to 8 byte boundaries
 367     //    when originally created.  Any unused space between items needs to be zero.
 368     //
 369     if (inBytes != outBytes) {
 370         uprv_memset(outBytes, 0, breakDataLength);
 371     }
 372
 373     //
 374     // Each state table begins with several 32 bit fields.  Calculate the size
 375     //   in bytes of these.
 376     //
 377     int32_t         topSize = offsetof(RBBIStateTable, fTableData);
 378
 379     // Forward state table.
 380     tableStartOffset = ds->readUInt32(rbbiDH->fFTable);
 381     tableLength      = ds->readUInt32(rbbiDH->fFTableLen);
 382
 383     if (tableLength > 0) {
 384         ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
 385                             outBytes+tableStartOffset, status);
 386         ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
 387                             outBytes+tableStartOffset+topSize, status);
 388     }
 389
 390     // Reverse state table.  Same layout as forward table, above.
 391     tableStartOffset = ds->readUInt32(rbbiDH->fRTable);
 392     tableLength      = ds->readUInt32(rbbiDH->fRTableLen);
 393
 394     if (tableLength > 0) {
 395         ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
 396                             outBytes+tableStartOffset, status);
 397         ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
 398                             outBytes+tableStartOffset+topSize, status);
 399     }
 400
 401     // Safe Forward state table.  Same layout as forward table, above.
 402     tableStartOffset = ds->readUInt32(rbbiDH->fSFTable);
 403     tableLength      = ds->readUInt32(rbbiDH->fSFTableLen);
 404
 405     if (tableLength > 0) {
 406         ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
 407                             outBytes+tableStartOffset, status);
 408         ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
 409                             outBytes+tableStartOffset+topSize, status);
 410     }
 411
 412     // Safe Reverse state table.  Same layout as forward table, above.
 413     tableStartOffset = ds->readUInt32(rbbiDH->fSRTable);
 414     tableLength      = ds->readUInt32(rbbiDH->fSRTableLen);
 415
 416     if (tableLength > 0) {
 417         ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
 418                             outBytes+tableStartOffset, status);
 419         ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
 420                             outBytes+tableStartOffset+topSize, status);
 421     }
 422
 423     // Trie table for character categories
 424     utrie_swap(ds, inBytes+ds->readUInt32(rbbiDH->fTrie), ds->readUInt32(rbbiDH->fTrieLen),
 425                             outBytes+ds->readUInt32(rbbiDH->fTrie), status);
 426
 427     // Source Rules Text.  It's UChar data
 428     ds->swapArray16(ds, inBytes+ds->readUInt32(rbbiDH->fRuleSource), ds->readUInt32(rbbiDH->fRuleSourceLen),
 429                         outBytes+ds->readUInt32(rbbiDH->fRuleSource), status);
 430
 431     // Table of rule status values.  It's all int_32 values
 432     ds->swapArray32(ds, inBytes+ds->readUInt32(rbbiDH->fStatusTable), ds->readUInt32(rbbiDH->fStatusTableLen),
 433                         outBytes+ds->readUInt32(rbbiDH->fStatusTable), status);
 434
 435     // And, last, the header.
 436     //   It is all int32_t values except for fFormataVersion, which is an array of four bytes.
 437     //   Swap the whole thing as int32_t, then re-swap the one field.
 438     //
 439     ds->swapArray32(ds, inBytes, sizeof(RBBIDataHeader), outBytes, status);
 440     ds->swapArray32(ds, outputDH->fFormatVersion, 4, outputDH->fFormatVersion, status);
 441
 442     return totalSize;
 443 }
 444
 445
 446 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */