X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/b75a7d8f3b4adbae880cab104ce2c6a50eee4db2..a01113dcd0f39d5da295ef82785beff9ed86fe38:/icuSources/common/rbbidata.cpp diff --git a/icuSources/common/rbbidata.cpp b/icuSources/common/rbbidata.cpp index 1a78537c..1d4c9e58 100644 --- a/icuSources/common/rbbidata.cpp +++ b/icuSources/common/rbbidata.cpp @@ -1,6 +1,8 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html /* *************************************************************************** -* Copyright (C) 1999-2003 International Business Machines Corporation * +* Copyright (C) 1999-2014 International Business Machines Corporation * * and others. All rights reserved. * *************************************************************************** */ @@ -12,7 +14,7 @@ #include "unicode/utypes.h" #include "rbbidata.h" #include "rbbirb.h" -#include "utrie.h" +#include "utrie2.h" #include "udatamem.h" #include "cmemory.h" #include "cstring.h" @@ -21,23 +23,6 @@ #include "uassert.h" -//----------------------------------------------------------------------------------- -// -// Trie access folding function. Copied as-is from properties code in uchar.c -// -//----------------------------------------------------------------------------------- -U_CDECL_BEGIN -static int32_t U_CALLCONV -getFoldingOffset(uint32_t data) { - /* if bit 15 is set, then the folding offset is in bits 14..0 of the 16-bit trie result */ - if(data&0x8000) { - return (int32_t)(data&0x7fff); - } else { - return 0; - } -} -U_CDECL_END - U_NAMESPACE_BEGIN //----------------------------------------------------------------------------- @@ -46,52 +31,100 @@ U_NAMESPACE_BEGIN // //----------------------------------------------------------------------------- RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, UErrorCode &status) { + init0(); init(data, status); } +RBBIDataWrapper::RBBIDataWrapper(const RBBIDataHeader *data, enum EDontAdopt, UErrorCode &status) { + init0(); + init(data, status); + fDontFreeData = TRUE; +} + RBBIDataWrapper::RBBIDataWrapper(UDataMemory* udm, UErrorCode &status) { - const RBBIDataHeader *d = (const RBBIDataHeader *) - ((char *)&(udm->pHeader->info) + udm->pHeader->info.size); - init(d, status); + init0(); + if (U_FAILURE(status)) { + return; + } + const DataHeader *dh = udm->pHeader; + int32_t headerSize = dh->dataHeader.headerSize; + if ( !(headerSize >= 20 && + dh->info.isBigEndian == U_IS_BIG_ENDIAN && + dh->info.charsetFamily == U_CHARSET_FAMILY && + dh->info.dataFormat[0] == 0x42 && // dataFormat="Brk " + dh->info.dataFormat[1] == 0x72 && + dh->info.dataFormat[2] == 0x6b && + dh->info.dataFormat[3] == 0x20 && + isDataVersionAcceptable(dh->info.formatVersion)) + ) { + status = U_INVALID_FORMAT_ERROR; + return; + } + const char *dataAsBytes = reinterpret_cast(dh); + const RBBIDataHeader *rbbidh = reinterpret_cast(dataAsBytes + headerSize); + init(rbbidh, status); fUDataMem = udm; } +UBool RBBIDataWrapper::isDataVersionAcceptable(const UVersionInfo version) { + return RBBI_DATA_FORMAT_VERSION[0] == version[0]; +} + + //----------------------------------------------------------------------------- // // init(). Does most of the work of construction, shared between the // constructors. // //----------------------------------------------------------------------------- +void RBBIDataWrapper::init0() { + fHeader = NULL; + fForwardTable = NULL; + fReverseTable = NULL; + fRuleSource = NULL; + fRuleStatusTable = NULL; + fTrie = NULL; + fUDataMem = NULL; + fRefCount = 0; + fDontFreeData = TRUE; +} + void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) { if (U_FAILURE(status)) { return; } fHeader = data; - if (fHeader->fMagic != 0xb1a0) { - status = U_BRK_INTERNAL_ERROR; + if (fHeader->fMagic != 0xb1a0 || !isDataVersionAcceptable(fHeader->fFormatVersion)) { + status = U_INVALID_FORMAT_ERROR; return; } + // Note: in ICU version 3.2 and earlier, there was a formatVersion 1 + // that is no longer supported. At that time fFormatVersion was + // an int32_t field, rather than an array of 4 bytes. - fUDataMem = NULL; - fForwardTable = (RBBIStateTable *)((char *)data + fHeader->fFTable); - fReverseTable = NULL; + fDontFreeData = FALSE; + if (data->fFTableLen != 0) { + fForwardTable = (RBBIStateTable *)((char *)data + fHeader->fFTable); + } if (data->fRTableLen != 0) { fReverseTable = (RBBIStateTable *)((char *)data + fHeader->fRTable); } - - utrie_unserialize(&fTrie, - (uint8_t *)data + fHeader->fTrie, - fHeader->fTrieLen, - &status); + fTrie = utrie2_openFromSerialized(UTRIE2_16_VALUE_BITS, + (uint8_t *)data + fHeader->fTrie, + fHeader->fTrieLen, + NULL, // *actual length + &status); if (U_FAILURE(status)) { return; } - fTrie.getFoldingOffset=getFoldingOffset; - fRuleSource = (UChar *)((char *)data + fHeader->fRuleSource); fRuleString.setTo(TRUE, fRuleSource, -1); + U_ASSERT(data->fRuleSourceLen > 0); + + fRuleStatusTable = (int32_t *)((char *)data + fHeader->fStatusTable); + fStatusMaxIdx = data->fStatusTableLen / sizeof(int32_t); fRefCount = 1; @@ -104,14 +137,16 @@ void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) { //----------------------------------------------------------------------------- // -// Destructor. Don't call this - use removeReferenc() instead. +// Destructor. Don't call this - use removeReference() instead. // //----------------------------------------------------------------------------- RBBIDataWrapper::~RBBIDataWrapper() { U_ASSERT(fRefCount == 0); + utrie2_close(fTrie); + fTrie = NULL; if (fUDataMem) { udata_close(fUDataMem); - } else { + } else if (!fDontFreeData) { uprv_free((void *)fHeader); } } @@ -173,7 +208,7 @@ RBBIDataWrapper *RBBIDataWrapper::addReference() { // getRuleSourceString // //----------------------------------------------------------------------------- -const UnicodeString &RBBIDataWrapper::getRuleSourceString() { +const UnicodeString &RBBIDataWrapper::getRuleSourceString() const { return fRuleString; } @@ -183,45 +218,208 @@ const UnicodeString &RBBIDataWrapper::getRuleSourceString() { // print - debugging function to dump the runtime data tables. // //----------------------------------------------------------------------------- -void RBBIDataWrapper::printData() { #ifdef RBBI_DEBUG - uint32_t c, s; +void RBBIDataWrapper::printTable(const char *heading, const RBBIStateTable *table) { + uint32_t c; + uint32_t s; - RBBIDebugPrintf("RBBI Data at %p\n", (void *)fHeader); - RBBIDebugPrintf(" Version = %d\n", fHeader->fVersion); - RBBIDebugPrintf(" total length of data = %d\n", fHeader->fLength); - RBBIDebugPrintf(" number of character categories = %d\n\n", fHeader->fCatCount); + RBBIDebugPrintf(" %s\n", heading); - RBBIDebugPrintf(" Forward State Transition Table\n"); - RBBIDebugPrintf("State | Acc LA Tag"); + RBBIDebugPrintf("State | Acc LA TagIx"); for (c=0; cfCatCount; c++) {RBBIDebugPrintf("%3d ", c);} - RBBIDebugPrintf("\n------|---------------"); for (c=0;cfCatCount; c++) {RBBIDebugPrintf("----");} + RBBIDebugPrintf("\n------|---------------"); for (c=0;cfCatCount; c++) { + RBBIDebugPrintf("----"); + } RBBIDebugPrintf("\n"); - for (s=0; sfNumStates; s++) { + if (table == NULL) { + RBBIDebugPrintf(" N U L L T A B L E\n\n"); + return; + } + for (s=0; sfNumStates; s++) { RBBIStateTableRow *row = (RBBIStateTableRow *) - (fForwardTable->fTableData + (fForwardTable->fRowLen * s)); - RBBIDebugPrintf("%4d | %3d %3d %3d ", s, row->fAccepting, row->fLookAhead, row->fTag); + (table->fTableData + (table->fRowLen * s)); + RBBIDebugPrintf("%4d | %3d %3d %3d ", s, row->fAccepting, row->fLookAhead, row->fTagIdx); for (c=0; cfCatCount; c++) { RBBIDebugPrintf("%3d ", row->fNextState[c]); } RBBIDebugPrintf("\n"); } + RBBIDebugPrintf("\n"); +} +#endif + + +void RBBIDataWrapper::printData() { +#ifdef RBBI_DEBUG + RBBIDebugPrintf("RBBI Data at %p\n", (void *)fHeader); + RBBIDebugPrintf(" Version = {%d %d %d %d}\n", fHeader->fFormatVersion[0], fHeader->fFormatVersion[1], + fHeader->fFormatVersion[2], fHeader->fFormatVersion[3]); + RBBIDebugPrintf(" total length of data = %d\n", fHeader->fLength); + RBBIDebugPrintf(" number of character categories = %d\n\n", fHeader->fCatCount); + + printTable("Forward State Transition Table", fForwardTable); + printTable("Reverse State Transition Table", fReverseTable); RBBIDebugPrintf("\nOrignal Rules source:\n"); - c = 0; - for (;;) { - if (fRuleSource[c] == 0) - break; + for (int32_t c=0; fRuleSource[c] != 0; c++) { RBBIDebugPrintf("%c", fRuleSource[c]); - c++; } RBBIDebugPrintf("\n\n"); #endif } - U_NAMESPACE_END +U_NAMESPACE_USE + +//----------------------------------------------------------------------------- +// +// ubrk_swap - byte swap and char encoding swap of RBBI data +// +//----------------------------------------------------------------------------- + +U_CAPI int32_t U_EXPORT2 +ubrk_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, + UErrorCode *status) { + + if (status == NULL || U_FAILURE(*status)) { + return 0; + } + if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) { + *status=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + // + // Check that the data header is for for break data. + // (Header contents are defined in genbrk.cpp) + // + const UDataInfo *pInfo = (const UDataInfo *)((const char *)inData+4); + if(!( pInfo->dataFormat[0]==0x42 && /* dataFormat="Brk " */ + pInfo->dataFormat[1]==0x72 && + pInfo->dataFormat[2]==0x6b && + pInfo->dataFormat[3]==0x20 && + RBBIDataWrapper::isDataVersionAcceptable(pInfo->formatVersion) )) { + udata_printError(ds, "ubrk_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *status=U_UNSUPPORTED_ERROR; + return 0; + } + + // + // Swap the data header. (This is the generic ICU Data Header, not the RBBI Specific + // RBBIDataHeader). This swap also conveniently gets us + // the size of the ICU d.h., which lets us locate the start + // of the RBBI specific data. + // + int32_t headerSize=udata_swapDataHeader(ds, inData, length, outData, status); + + + // + // Get the RRBI Data Header, and check that it appears to be OK. + // + const uint8_t *inBytes =(const uint8_t *)inData+headerSize; + RBBIDataHeader *rbbiDH = (RBBIDataHeader *)inBytes; + if (ds->readUInt32(rbbiDH->fMagic) != 0xb1a0 || + !RBBIDataWrapper::isDataVersionAcceptable(rbbiDH->fFormatVersion) || + ds->readUInt32(rbbiDH->fLength) < sizeof(RBBIDataHeader)) { + udata_printError(ds, "ubrk_swap(): RBBI Data header is invalid.\n"); + *status=U_UNSUPPORTED_ERROR; + return 0; + } + + // + // Prefight operation? Just return the size + // + int32_t breakDataLength = ds->readUInt32(rbbiDH->fLength); + int32_t totalSize = headerSize + breakDataLength; + if (length < 0) { + return totalSize; + } + + // + // Check that length passed in is consistent with length from RBBI data header. + // + if (length < totalSize) { + udata_printError(ds, "ubrk_swap(): too few bytes (%d after ICU Data header) for break data.\n", + breakDataLength); + *status=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + + // + // Swap the Data. Do the data itself first, then the RBBI Data Header, because + // we need to reference the header to locate the data, and an + // inplace swap of the header leaves it unusable. + // + uint8_t *outBytes = (uint8_t *)outData + headerSize; + RBBIDataHeader *outputDH = (RBBIDataHeader *)outBytes; + + int32_t tableStartOffset; + int32_t tableLength; + + // + // If not swapping in place, zero out the output buffer before starting. + // Individual tables and other data items within are aligned to 8 byte boundaries + // when originally created. Any unused space between items needs to be zero. + // + if (inBytes != outBytes) { + uprv_memset(outBytes, 0, breakDataLength); + } + + // + // Each state table begins with several 32 bit fields. Calculate the size + // in bytes of these. + // + int32_t topSize = offsetof(RBBIStateTable, fTableData); + + // Forward state table. + tableStartOffset = ds->readUInt32(rbbiDH->fFTable); + tableLength = ds->readUInt32(rbbiDH->fFTableLen); + + if (tableLength > 0) { + ds->swapArray32(ds, inBytes+tableStartOffset, topSize, + outBytes+tableStartOffset, status); + ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize, + outBytes+tableStartOffset+topSize, status); + } + + // Reverse state table. Same layout as forward table, above. + tableStartOffset = ds->readUInt32(rbbiDH->fRTable); + tableLength = ds->readUInt32(rbbiDH->fRTableLen); + + if (tableLength > 0) { + ds->swapArray32(ds, inBytes+tableStartOffset, topSize, + outBytes+tableStartOffset, status); + ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize, + outBytes+tableStartOffset+topSize, status); + } + + // Trie table for character categories + utrie2_swap(ds, inBytes+ds->readUInt32(rbbiDH->fTrie), ds->readUInt32(rbbiDH->fTrieLen), + outBytes+ds->readUInt32(rbbiDH->fTrie), status); + + // Source Rules Text. It's UChar data + ds->swapArray16(ds, inBytes+ds->readUInt32(rbbiDH->fRuleSource), ds->readUInt32(rbbiDH->fRuleSourceLen), + outBytes+ds->readUInt32(rbbiDH->fRuleSource), status); + + // Table of rule status values. It's all int_32 values + ds->swapArray32(ds, inBytes+ds->readUInt32(rbbiDH->fStatusTable), ds->readUInt32(rbbiDH->fStatusTableLen), + outBytes+ds->readUInt32(rbbiDH->fStatusTable), status); + + // And, last, the header. + // It is all int32_t values except for fFormataVersion, which is an array of four bytes. + // Swap the whole thing as int32_t, then re-swap the one field. + // + ds->swapArray32(ds, inBytes, sizeof(RBBIDataHeader), outBytes, status); + ds->swapArray32(ds, outputDH->fFormatVersion, 4, outputDH->fFormatVersion, status); + + return totalSize; +} + #endif /* #if !UCONFIG_NO_BREAK_ITERATION */