icuSources/tools/gennorm2/norms.cpp

   1 // © 2017 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3
   4 // norms.cpp
   5 // created: 2017jun04 Markus W. Scherer
   6 // (pulled out of n2builder.cpp)
   7
   8 #include "unicode/utypes.h"
   9
  10 #if !UCONFIG_NO_NORMALIZATION
  11
  12 #include <stdio.h>
  13 #include <stdlib.h>
  14 #include "unicode/errorcode.h"
  15 #include "unicode/unistr.h"
  16 #include "unicode/utf16.h"
  17 #include "normalizer2impl.h"
  18 #include "norms.h"
  19 #include "toolutil.h"
  20 #include "utrie2.h"
  21 #include "uvectr32.h"
  22
  23 U_NAMESPACE_BEGIN
  24
  25 void BuilderReorderingBuffer::append(UChar32 c, uint8_t cc) {
  26     if(cc==0 || fLength==0 || ccAt(fLength-1)<=cc) {
  27         if(cc==0) {
  28             fLastStarterIndex=fLength;
  29         }
  30         fArray[fLength++]=(c<<8)|cc;
  31         return;
  32     }
  33     // Let this character bubble back to its canonical order.
  34     int32_t i=fLength-1;
  35     while(i>fLastStarterIndex && ccAt(i)>cc) {
  36         --i;
  37     }
  38     ++i;  // after the last starter or prevCC<=cc
  39     // Move this and the following characters forward one to make space.
  40     for(int32_t j=fLength; i<j; --j) {
  41         fArray[j]=fArray[j-1];
  42     }
  43     fArray[i]=(c<<8)|cc;
  44     ++fLength;
  45     fDidReorder=TRUE;
  46 }
  47
  48 void BuilderReorderingBuffer::toString(UnicodeString &dest) const {
  49     dest.remove();
  50     for(int32_t i=0; i<fLength; ++i) {
  51         dest.append(charAt(i));
  52     }
  53 }
  54
  55 UChar32 Norm::combine(UChar32 trail) const {
  56     int32_t length;
  57     const CompositionPair *pairs=getCompositionPairs(length);
  58     for(int32_t i=0; i<length; ++i) {
  59         if(trail==pairs[i].trail) {
  60             return pairs[i].composite;
  61         }
  62         if(trail<pairs[i].trail) {
  63             break;
  64         }
  65     }
  66     return U_SENTINEL;
  67 }
  68
  69 Norms::Norms(UErrorCode &errorCode) {
  70     normTrie=utrie2_open(0, 0, &errorCode);
  71     normMem=utm_open("gennorm2 normalization structs", 10000, 0x110100, sizeof(Norm));
  72     // Default "inert" Norm struct at index 0. Practically immutable.
  73     norms=allocNorm();
  74     norms->type=Norm::INERT;
  75 }
  76
  77 Norms::~Norms() {
  78     utrie2_close(normTrie);
  79     int32_t normsLength=utm_countItems(normMem);
  80     for(int32_t i=1; i<normsLength; ++i) {
  81         delete norms[i].mapping;
  82         delete norms[i].rawMapping;
  83         delete norms[i].compositions;
  84     }
  85     utm_close(normMem);
  86 }
  87
  88 Norm *Norms::allocNorm() {
  89     Norm *p=(Norm *)utm_alloc(normMem);
  90     norms=(Norm *)utm_getStart(normMem);  // in case it got reallocated
  91     return p;
  92 }
  93
  94 Norm *Norms::getNorm(UChar32 c) {
  95     uint32_t i=utrie2_get32(normTrie, c);
  96     if(i==0) {
  97         return nullptr;
  98     }
  99     return norms+i;
 100 }
 101
 102 const Norm *Norms::getNorm(UChar32 c) const {
 103     uint32_t i=utrie2_get32(normTrie, c);
 104     if(i==0) {
 105         return nullptr;
 106     }
 107     return norms+i;
 108 }
 109
 110 const Norm &Norms::getNormRef(UChar32 c) const {
 111     return norms[utrie2_get32(normTrie, c)];
 112 }
 113
 114 Norm *Norms::createNorm(UChar32 c) {
 115     uint32_t i=utrie2_get32(normTrie, c);
 116     if(i!=0) {
 117         return norms+i;
 118     } else {
 119         /* allocate Norm */
 120         Norm *p=allocNorm();
 121         IcuToolErrorCode errorCode("gennorm2/createNorm()");
 122         utrie2_set32(normTrie, c, (uint32_t)(p-norms), errorCode);
 123         return p;
 124     }
 125 }
 126
 127 void Norms::reorder(UnicodeString &mapping, BuilderReorderingBuffer &buffer) const {
 128     int32_t length=mapping.length();
 129     U_ASSERT(length<=Normalizer2Impl::MAPPING_LENGTH_MASK);
 130     const char16_t *s=mapping.getBuffer();
 131     int32_t i=0;
 132     UChar32 c;
 133     while(i<length) {
 134         U16_NEXT(s, i, length, c);
 135         buffer.append(c, getCC(c));
 136     }
 137     if(buffer.didReorder()) {
 138         buffer.toString(mapping);
 139     }
 140 }
 141
 142 UBool Norms::combinesWithCCBetween(const Norm &norm, uint8_t lowCC, int32_t highCC) const {
 143     if((highCC-lowCC)>=2) {
 144         int32_t length;
 145         const CompositionPair *pairs=norm.getCompositionPairs(length);
 146         for(int32_t i=0; i<length; ++i) {
 147             uint8_t trailCC=getCC(pairs[i].trail);
 148             if(lowCC<trailCC && trailCC<highCC) {
 149                 return TRUE;
 150             }
 151         }
 152     }
 153     return FALSE;
 154 }
 155
 156 U_CDECL_BEGIN
 157
 158 static UBool U_CALLCONV
 159 enumRangeHandler(const void *context, UChar32 start, UChar32 end, uint32_t value) {
 160     return ((Norms::Enumerator *)context)->rangeHandler(start, end, value);
 161 }
 162
 163 U_CDECL_END
 164
 165 void Norms::enumRanges(Enumerator &e) {
 166     utrie2_enum(normTrie, nullptr, enumRangeHandler, &e);
 167 }
 168
 169 Norms::Enumerator::~Enumerator() {}
 170
 171 UBool Norms::Enumerator::rangeHandler(UChar32 start, UChar32 end, uint32_t value) {
 172     if(value!=0) {
 173         rangeHandler(start, end, norms.getNormRefByIndex(value));
 174     }
 175     return TRUE;
 176 }
 177
 178 void CompositionBuilder::rangeHandler(UChar32 start, UChar32 end, Norm &norm) {
 179     if(norm.mappingType!=Norm::ROUND_TRIP) { return; }
 180     if(start!=end) {
 181         fprintf(stderr,
 182                 "gennorm2 error: same round-trip mapping for "
 183                 "more than 1 code point U+%04lX..U+%04lX\n",
 184                 (long)start, (long)end);
 185         exit(U_INVALID_FORMAT_ERROR);
 186     }
 187     if(norm.cc!=0) {
 188         fprintf(stderr,
 189                 "gennorm2 error: "
 190                 "U+%04lX has a round-trip mapping and ccc!=0, "
 191                 "not possible in Unicode normalization\n",
 192                 (long)start);
 193         exit(U_INVALID_FORMAT_ERROR);
 194     }
 195     // setRoundTripMapping() ensured that there are exactly two code points.
 196     const UnicodeString &m=*norm.mapping;
 197     UChar32 lead=m.char32At(0);
 198     UChar32 trail=m.char32At(m.length()-1);
 199     if(norms.getCC(lead)!=0) {
 200         fprintf(stderr,
 201                 "gennorm2 error: "
 202                 "U+%04lX's round-trip mapping's starter U+%04lX has ccc!=0, "
 203                 "not possible in Unicode normalization\n",
 204                 (long)start, (long)lead);
 205         exit(U_INVALID_FORMAT_ERROR);
 206     }
 207     // Flag for trailing character.
 208     norms.createNorm(trail)->combinesBack=TRUE;
 209     // Insert (trail, composite) pair into compositions list for the lead character.
 210     IcuToolErrorCode errorCode("gennorm2/addComposition()");
 211     Norm *leadNorm=norms.createNorm(lead);
 212     UVector32 *compositions=leadNorm->compositions;
 213     int32_t i;
 214     if(compositions==nullptr) {
 215         compositions=leadNorm->compositions=new UVector32(errorCode);
 216         i=0;  // "insert" the first pair at index 0
 217     } else {
 218         // Insertion sort, and check for duplicate trail characters.
 219         int32_t length;
 220         const CompositionPair *pairs=leadNorm->getCompositionPairs(length);
 221         for(i=0; i<length; ++i) {
 222             if(trail==pairs[i].trail) {
 223                 fprintf(stderr,
 224                         "gennorm2 error: same round-trip mapping for "
 225                         "more than 1 code point (e.g., U+%04lX) to U+%04lX + U+%04lX\n",
 226                         (long)start, (long)lead, (long)trail);
 227                 exit(U_INVALID_FORMAT_ERROR);
 228             }
 229             if(trail<pairs[i].trail) {
 230                 break;
 231             }
 232         }
 233     }
 234     compositions->insertElementAt(trail, 2*i, errorCode);
 235     compositions->insertElementAt(start, 2*i+1, errorCode);
 236 }
 237
 238 void Decomposer::rangeHandler(UChar32 start, UChar32 end, Norm &norm) {
 239     if(!norm.hasMapping()) { return; }
 240     const UnicodeString &m=*norm.mapping;
 241     UnicodeString *decomposed=nullptr;
 242     const UChar *s=toUCharPtr(m.getBuffer());
 243     int32_t length=m.length();
 244     int32_t prev, i=0;
 245     UChar32 c;
 246     while(i<length) {
 247         prev=i;
 248         U16_NEXT(s, i, length, c);
 249         if(start<=c && c<=end) {
 250             fprintf(stderr,
 251                     "gennorm2 error: U+%04lX maps to itself directly or indirectly\n",
 252                     (long)c);
 253             exit(U_INVALID_FORMAT_ERROR);
 254         }
 255         const Norm &cNorm=norms.getNormRef(c);
 256         if(cNorm.hasMapping()) {
 257             if(norm.mappingType==Norm::ROUND_TRIP) {
 258                 if(prev==0) {
 259                     if(cNorm.mappingType!=Norm::ROUND_TRIP) {
 260                         fprintf(stderr,
 261                                 "gennorm2 error: "
 262                                 "U+%04lX's round-trip mapping's starter "
 263                                 "U+%04lX one-way-decomposes, "
 264                                 "not possible in Unicode normalization\n",
 265                                 (long)start, (long)c);
 266                         exit(U_INVALID_FORMAT_ERROR);
 267                     }
 268                     uint8_t myTrailCC=norms.getCC(m.char32At(i));
 269                     UChar32 cTrailChar=cNorm.mapping->char32At(cNorm.mapping->length()-1);
 270                     uint8_t cTrailCC=norms.getCC(cTrailChar);
 271                     if(cTrailCC>myTrailCC) {
 272                         fprintf(stderr,
 273                                 "gennorm2 error: "
 274                                 "U+%04lX's round-trip mapping's starter "
 275                                 "U+%04lX decomposes and the "
 276                                 "inner/earlier tccc=%hu > outer/following tccc=%hu, "
 277                                 "not possible in Unicode normalization\n",
 278                                 (long)start, (long)c,
 279                                 (short)cTrailCC, (short)myTrailCC);
 280                         exit(U_INVALID_FORMAT_ERROR);
 281                     }
 282                 } else {
 283                     fprintf(stderr,
 284                             "gennorm2 error: "
 285                             "U+%04lX's round-trip mapping's non-starter "
 286                             "U+%04lX decomposes, "
 287                             "not possible in Unicode normalization\n",
 288                             (long)start, (long)c);
 289                     exit(U_INVALID_FORMAT_ERROR);
 290                 }
 291             }
 292             if(decomposed==nullptr) {
 293                 decomposed=new UnicodeString(m, 0, prev);
 294             }
 295             decomposed->append(*cNorm.mapping);
 296         } else if(Hangul::isHangul(c)) {
 297             UChar buffer[3];
 298             int32_t hangulLength=Hangul::decompose(c, buffer);
 299             if(norm.mappingType==Norm::ROUND_TRIP && prev!=0) {
 300                 fprintf(stderr,
 301                         "gennorm2 error: "
 302                         "U+%04lX's round-trip mapping's non-starter "
 303                         "U+%04lX decomposes, "
 304                         "not possible in Unicode normalization\n",
 305                         (long)start, (long)c);
 306                 exit(U_INVALID_FORMAT_ERROR);
 307             }
 308             if(decomposed==nullptr) {
 309                 decomposed=new UnicodeString(m, 0, prev);
 310             }
 311             decomposed->append(buffer, hangulLength);
 312         } else if(decomposed!=nullptr) {
 313             decomposed->append(m, prev, i-prev);
 314         }
 315     }
 316     if(decomposed!=nullptr) {
 317         if(norm.rawMapping==nullptr) {
 318             // Remember the original mapping when decomposing recursively.
 319             norm.rawMapping=norm.mapping;
 320         } else {
 321             delete norm.mapping;
 322         }
 323         norm.mapping=decomposed;
 324         // Not  norm.setMappingCP();  because the original mapping
 325         // is most likely to be encodable as a delta.
 326         didDecompose|=TRUE;
 327     }
 328 }
 329
 330 U_NAMESPACE_END
 331
 332 #endif // #if !UCONFIG_NO_NORMALIZATION