icuSources/tools/toolutil/ppucd.cpp

   1 /*
   2 *******************************************************************************
   3 *   Copyright (C) 2011-2014, International Business Machines
   4 *   Corporation and others.  All Rights Reserved.
   5 *******************************************************************************
   6 *   file name:  ppucd.cpp
   7 *   encoding:   US-ASCII
   8 *   tab size:   8 (not used)
   9 *   indentation:4
  10 *
  11 *   created on: 2011dec11
  12 *   created by: Markus W. Scherer
  13 */
  14
  15 #include "unicode/utypes.h"
  16 #include "unicode/uchar.h"
  17 #include "charstr.h"
  18 #include "cstring.h"
  19 #include "ppucd.h"
  20 #include "uassert.h"
  21 #include "uparse.h"
  22
  23 #include <stdio.h>
  24 #include <string.h>
  25
  26 U_NAMESPACE_BEGIN
  27
  28 PropertyNames::~PropertyNames() {}
  29
  30 int32_t
  31 PropertyNames::getPropertyEnum(const char *name) const {
  32     return u_getPropertyEnum(name);
  33 }
  34
  35 int32_t
  36 PropertyNames::getPropertyValueEnum(int32_t property, const char *name) const {
  37     return u_getPropertyValueEnum((UProperty)property, name);
  38 }
  39
  40 UniProps::UniProps()
  41         : start(U_SENTINEL), end(U_SENTINEL),
  42           bmg(U_SENTINEL), bpb(U_SENTINEL),
  43           scf(U_SENTINEL), slc(U_SENTINEL), stc(U_SENTINEL), suc(U_SENTINEL),
  44           digitValue(-1), numericValue(NULL),
  45           name(NULL), nameAlias(NULL) {
  46     memset(binProps, 0, sizeof(binProps));
  47     memset(intProps, 0, sizeof(intProps));
  48     memset(age, 0, 4);
  49 }
  50
  51 UniProps::~UniProps() {}
  52
  53 const int32_t PreparsedUCD::kNumLineBuffers;
  54
  55 PreparsedUCD::PreparsedUCD(const char *filename, UErrorCode &errorCode)
  56         : icuPnames(new PropertyNames()), pnames(icuPnames),
  57           file(NULL),
  58           defaultLineIndex(-1), blockLineIndex(-1), lineIndex(0),
  59           lineNumber(0),
  60           lineType(NO_LINE),
  61           fieldLimit(NULL), lineLimit(NULL) {
  62     if(U_FAILURE(errorCode)) { return; }
  63
  64     if(filename==NULL || *filename==0 || (*filename=='-' && filename[1]==0)) {
  65         filename=NULL;
  66         file=stdin;
  67     } else {
  68         file=fopen(filename, "r");
  69     }
  70     if(file==NULL) {
  71         perror("error opening preparsed UCD");
  72         fprintf(stderr, "error opening preparsed UCD file %s\n", filename ? filename : "\"no file name given\"");
  73         errorCode=U_FILE_ACCESS_ERROR;
  74         return;
  75     }
  76
  77     memset(ucdVersion, 0, 4);
  78     lines[0][0]=0;
  79 }
  80
  81 PreparsedUCD::~PreparsedUCD() {
  82     if(file!=stdin) {
  83         fclose(file);
  84     }
  85     delete icuPnames;
  86 }
  87
  88 // Same order as the LineType values.
  89 static const char *lineTypeStrings[]={
  90     NULL,
  91     NULL,
  92     "ucd",
  93     "property",
  94     "binary",
  95     "value",
  96     "defaults",
  97     "block",
  98     "cp",
  99     "algnamesrange"
 100 };
 101
 102 PreparsedUCD::LineType
 103 PreparsedUCD::readLine(UErrorCode &errorCode) {
 104     if(U_FAILURE(errorCode)) { return NO_LINE; }
 105     // Select the next available line buffer.
 106     while(!isLineBufferAvailable(lineIndex)) {
 107         ++lineIndex;
 108         if (lineIndex == kNumLineBuffers) {
 109             lineIndex = 0;
 110         }
 111     }
 112     char *line=lines[lineIndex];
 113     *line=0;
 114     lineLimit=fieldLimit=line;
 115     lineType=NO_LINE;
 116     char *result=fgets(line, sizeof(lines[0]), file);
 117     if(result==NULL) {
 118         if(ferror(file)) {
 119             perror("error reading preparsed UCD");
 120             fprintf(stderr, "error reading preparsed UCD before line %ld\n", (long)lineNumber);
 121             errorCode=U_FILE_ACCESS_ERROR;
 122         }
 123         return NO_LINE;
 124     }
 125     ++lineNumber;
 126     if(*line=='#') {
 127         fieldLimit=strchr(line, 0);
 128         return lineType=EMPTY_LINE;
 129     }
 130     // Remove trailing /r/n.
 131     char c;
 132     char *limit=strchr(line, 0);
 133     while(line<limit && ((c=*(limit-1))=='\n' || c=='\r')) { --limit; }
 134     // Remove trailing white space.
 135     while(line<limit && ((c=*(limit-1))==' ' || c=='\t')) { --limit; }
 136     *limit=0;
 137     lineLimit=limit;
 138     if(line==limit) {
 139         fieldLimit=limit;
 140         return lineType=EMPTY_LINE;
 141     }
 142     // Split by ';'.
 143     char *semi=line;
 144     while((semi=strchr(semi, ';'))!=NULL) { *semi++=0; }
 145     fieldLimit=strchr(line, 0);
 146     // Determine the line type.
 147     int32_t type;
 148     for(type=EMPTY_LINE+1;; ++type) {
 149         if(type==LINE_TYPE_COUNT) {
 150             fprintf(stderr,
 151                     "error in preparsed UCD: unknown line type (first field) '%s' on line %ld\n",
 152                     line, (long)lineNumber);
 153             errorCode=U_PARSE_ERROR;
 154             return NO_LINE;
 155         }
 156         if(0==strcmp(line, lineTypeStrings[type])) {
 157             break;
 158         }
 159     }
 160     lineType=(LineType)type;
 161     if(lineType==UNICODE_VERSION_LINE && fieldLimit<lineLimit) {
 162         u_versionFromString(ucdVersion, fieldLimit+1);
 163     }
 164     return lineType;
 165 }
 166
 167 const char *
 168 PreparsedUCD::firstField() {
 169     char *field=lines[lineIndex];
 170     fieldLimit=strchr(field, 0);
 171     return field;
 172 }
 173
 174 const char *
 175 PreparsedUCD::nextField() {
 176     if(fieldLimit==lineLimit) { return NULL; }
 177     char *field=fieldLimit+1;
 178     fieldLimit=strchr(field, 0);
 179     return field;
 180 }
 181
 182 const UniProps *
 183 PreparsedUCD::getProps(UnicodeSet &newValues, UErrorCode &errorCode) {
 184     if(U_FAILURE(errorCode)) { return NULL; }
 185     newValues.clear();
 186     if(!lineHasPropertyValues()) {
 187         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 188         return NULL;
 189     }
 190     firstField();
 191     const char *field=nextField();
 192     if(field==NULL) {
 193         // No range field after the type.
 194         fprintf(stderr,
 195                 "error in preparsed UCD: missing default/block/cp range field "
 196                 "(no second field) on line %ld\n",
 197                 (long)lineNumber);
 198         errorCode=U_PARSE_ERROR;
 199         return NULL;
 200     }
 201     UChar32 start, end;
 202     if(!parseCodePointRange(field, start, end, errorCode)) { return NULL; }
 203     UniProps *props;
 204     switch(lineType) {
 205     case DEFAULTS_LINE:
 206         if(defaultLineIndex>=0) {
 207             fprintf(stderr,
 208                     "error in preparsed UCD: second line with default properties on line %ld\n",
 209                     (long)lineNumber);
 210             errorCode=U_PARSE_ERROR;
 211             return NULL;
 212         }
 213         if(start!=0 || end!=0x10ffff) {
 214             fprintf(stderr,
 215                     "error in preparsed UCD: default range must be 0..10FFFF, not '%s' on line %ld\n",
 216                     field, (long)lineNumber);
 217             errorCode=U_PARSE_ERROR;
 218             return NULL;
 219         }
 220         props=&defaultProps;
 221         defaultLineIndex=lineIndex;
 222         break;
 223     case BLOCK_LINE:
 224         blockProps=defaultProps;  // Block inherits default properties.
 225         props=&blockProps;
 226         blockLineIndex=lineIndex;
 227         break;
 228     case CP_LINE:
 229         if(blockProps.start<=start && end<=blockProps.end) {
 230             // Code point range fully inside the last block inherits the block properties.
 231             cpProps=blockProps;
 232         } else if(start>blockProps.end || end<blockProps.start) {
 233             // Code point range fully outside the last block inherits the default properties.
 234             cpProps=defaultProps;
 235         } else {
 236             // Code point range partially overlapping with the last block is illegal.
 237             fprintf(stderr,
 238                     "error in preparsed UCD: cp range %s on line %ld only "
 239                     "partially overlaps with block range %04lX..%04lX\n",
 240                     field, (long)lineNumber, (long)blockProps.start, (long)blockProps.end);
 241             errorCode=U_PARSE_ERROR;
 242             return NULL;
 243         }
 244         props=&cpProps;
 245         break;
 246     default:
 247         // Will not occur because of the range check above.
 248         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 249         return NULL;
 250     }
 251     props->start=start;
 252     props->end=end;
 253     while((field=nextField())!=NULL) {
 254         if(!parseProperty(*props, field, newValues, errorCode)) { return NULL; }
 255     }
 256     return props;
 257 }
 258
 259 static const struct {
 260     const char *name;
 261     int32_t prop;
 262 } ppucdProperties[]={
 263     { "Name_Alias", PPUCD_NAME_ALIAS },
 264     { "Conditional_Case_Mappings", PPUCD_CONDITIONAL_CASE_MAPPINGS },
 265     { "Turkic_Case_Folding", PPUCD_TURKIC_CASE_FOLDING }
 266 };
 267
 268 // Returns TRUE for "ok to continue parsing fields".
 269 UBool
 270 PreparsedUCD::parseProperty(UniProps &props, const char *field, UnicodeSet &newValues,
 271                             UErrorCode &errorCode) {
 272     CharString pBuffer;
 273     const char *p=field;
 274     const char *v=strchr(p, '=');
 275     int binaryValue;
 276     if(*p=='-') {
 277         if(v!=NULL) {
 278             fprintf(stderr,
 279                     "error in preparsed UCD: mix of binary-property-no and "
 280                     "enum-property syntax '%s' on line %ld\n",
 281                     field, (long)lineNumber);
 282             errorCode=U_PARSE_ERROR;
 283             return FALSE;
 284         }
 285         binaryValue=0;
 286         ++p;
 287     } else if(v==NULL) {
 288         binaryValue=1;
 289     } else {
 290         binaryValue=-1;
 291         // Copy out the property name rather than modifying the field (writing a NUL).
 292         pBuffer.append(p, (int32_t)(v-p), errorCode);
 293         p=pBuffer.data();
 294         ++v;
 295     }
 296     int32_t prop=pnames->getPropertyEnum(p);
 297     if(prop<0) {
 298         for(int32_t i=0;; ++i) {
 299             if(i==UPRV_LENGTHOF(ppucdProperties)) {
 300                 // Ignore unknown property names.
 301                 return TRUE;
 302             }
 303             if(0==uprv_stricmp(p, ppucdProperties[i].name)) {
 304                 prop=ppucdProperties[i].prop;
 305                 U_ASSERT(prop>=0);
 306                 break;
 307             }
 308         }
 309     }
 310     if(prop<UCHAR_BINARY_LIMIT) {
 311         if(binaryValue>=0) {
 312             props.binProps[prop]=(UBool)binaryValue;
 313         } else {
 314             // No binary value for a binary property.
 315             fprintf(stderr,
 316                     "error in preparsed UCD: enum-property syntax '%s' "
 317                     "for binary property on line %ld\n",
 318                     field, (long)lineNumber);
 319             errorCode=U_PARSE_ERROR;
 320         }
 321     } else if(binaryValue>=0) {
 322         // Binary value for a non-binary property.
 323         fprintf(stderr,
 324                 "error in preparsed UCD: binary-property syntax '%s' "
 325                 "for non-binary property on line %ld\n",
 326                 field, (long)lineNumber);
 327         errorCode=U_PARSE_ERROR;
 328     } else if (prop < UCHAR_INT_START) {
 329         fprintf(stderr,
 330                 "error in preparsed UCD: prop value is invalid: '%d' for line %ld\n",
 331                 prop, (long)lineNumber);
 332         errorCode=U_PARSE_ERROR;
 333     } else if(prop<UCHAR_INT_LIMIT) {
 334         int32_t value=pnames->getPropertyValueEnum(prop, v);
 335         if(value==UCHAR_INVALID_CODE && prop==UCHAR_CANONICAL_COMBINING_CLASS) {
 336             // TODO: Make getPropertyValueEnum(UCHAR_CANONICAL_COMBINING_CLASS, v) work.
 337             char *end;
 338             unsigned long ccc=uprv_strtoul(v, &end, 10);
 339             if(v<end && *end==0 && ccc<=254) {
 340                 value=(int32_t)ccc;
 341             }
 342         }
 343         if(value==UCHAR_INVALID_CODE) {
 344             fprintf(stderr,
 345                     "error in preparsed UCD: '%s' is not a valid value on line %ld\n",
 346                     field, (long)lineNumber);
 347             errorCode=U_PARSE_ERROR;
 348         } else {
 349             props.intProps[prop-UCHAR_INT_START]=value;
 350         }
 351     } else if(*v=='<') {
 352         // Do not parse default values like <code point>, just set null values.
 353         switch(prop) {
 354         case UCHAR_BIDI_MIRRORING_GLYPH:
 355             props.bmg=U_SENTINEL;
 356             break;
 357         case UCHAR_BIDI_PAIRED_BRACKET:
 358             props.bpb=U_SENTINEL;
 359             break;
 360         case UCHAR_SIMPLE_CASE_FOLDING:
 361             props.scf=U_SENTINEL;
 362             break;
 363         case UCHAR_SIMPLE_LOWERCASE_MAPPING:
 364             props.slc=U_SENTINEL;
 365             break;
 366         case UCHAR_SIMPLE_TITLECASE_MAPPING:
 367             props.stc=U_SENTINEL;
 368             break;
 369         case UCHAR_SIMPLE_UPPERCASE_MAPPING:
 370             props.suc=U_SENTINEL;
 371             break;
 372         case UCHAR_CASE_FOLDING:
 373             props.cf.remove();
 374             break;
 375         case UCHAR_LOWERCASE_MAPPING:
 376             props.lc.remove();
 377             break;
 378         case UCHAR_TITLECASE_MAPPING:
 379             props.tc.remove();
 380             break;
 381         case UCHAR_UPPERCASE_MAPPING:
 382             props.uc.remove();
 383             break;
 384         case UCHAR_SCRIPT_EXTENSIONS:
 385             props.scx.clear();
 386             break;
 387         default:
 388             fprintf(stderr,
 389                     "error in preparsed UCD: '%s' is not a valid default value on line %ld\n",
 390                     field, (long)lineNumber);
 391             errorCode=U_PARSE_ERROR;
 392         }
 393     } else {
 394         char c;
 395         switch(prop) {
 396         case UCHAR_NUMERIC_VALUE:
 397             props.numericValue=v;
 398             c=*v;
 399             if('0'<=c && c<='9' && v[1]==0) {
 400                 props.digitValue=c-'0';
 401             } else {
 402                 props.digitValue=-1;
 403             }
 404             break;
 405         case UCHAR_NAME:
 406             props.name=v;
 407             break;
 408         case UCHAR_AGE:
 409             u_versionFromString(props.age, v);  // Writes 0.0.0.0 if v is not numeric.
 410             break;
 411         case UCHAR_BIDI_MIRRORING_GLYPH:
 412             props.bmg=parseCodePoint(v, errorCode);
 413             break;
 414         case UCHAR_BIDI_PAIRED_BRACKET:
 415             props.bpb=parseCodePoint(v, errorCode);
 416             break;
 417         case UCHAR_SIMPLE_CASE_FOLDING:
 418             props.scf=parseCodePoint(v, errorCode);
 419             break;
 420         case UCHAR_SIMPLE_LOWERCASE_MAPPING:
 421             props.slc=parseCodePoint(v, errorCode);
 422             break;
 423         case UCHAR_SIMPLE_TITLECASE_MAPPING:
 424             props.stc=parseCodePoint(v, errorCode);
 425             break;
 426         case UCHAR_SIMPLE_UPPERCASE_MAPPING:
 427             props.suc=parseCodePoint(v, errorCode);
 428             break;
 429         case UCHAR_CASE_FOLDING:
 430             parseString(v, props.cf, errorCode);
 431             break;
 432         case UCHAR_LOWERCASE_MAPPING:
 433             parseString(v, props.lc, errorCode);
 434             break;
 435         case UCHAR_TITLECASE_MAPPING:
 436             parseString(v, props.tc, errorCode);
 437             break;
 438         case UCHAR_UPPERCASE_MAPPING:
 439             parseString(v, props.uc, errorCode);
 440             break;
 441         case PPUCD_NAME_ALIAS:
 442             props.nameAlias=v;
 443             break;
 444         case PPUCD_CONDITIONAL_CASE_MAPPINGS:
 445         case PPUCD_TURKIC_CASE_FOLDING:
 446             // No need to parse their values: They are hardcoded in the runtime library.
 447             break;
 448         case UCHAR_SCRIPT_EXTENSIONS:
 449             parseScriptExtensions(v, props.scx, errorCode);
 450             break;
 451         default:
 452             // Ignore unhandled properties.
 453             return TRUE;
 454         }
 455     }
 456     if(U_SUCCESS(errorCode)) {
 457         newValues.add((UChar32)prop);
 458         return TRUE;
 459     } else {
 460         return FALSE;
 461     }
 462 }
 463
 464 UBool
 465 PreparsedUCD::getRangeForAlgNames(UChar32 &start, UChar32 &end, UErrorCode &errorCode) {
 466     if(U_FAILURE(errorCode)) { return FALSE; }
 467     if(lineType!=ALG_NAMES_RANGE_LINE) {
 468         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 469         return FALSE;
 470     }
 471     firstField();
 472     const char *field=nextField();
 473     if(field==NULL) {
 474         // No range field after the type.
 475         fprintf(stderr,
 476                 "error in preparsed UCD: missing algnamesrange range field "
 477                 "(no second field) on line %ld\n",
 478                 (long)lineNumber);
 479         errorCode=U_PARSE_ERROR;
 480         return FALSE;
 481     }
 482     return parseCodePointRange(field, start, end, errorCode);
 483 }
 484
 485 UChar32
 486 PreparsedUCD::parseCodePoint(const char *s, UErrorCode &errorCode) {
 487     char *end;
 488     uint32_t value=(uint32_t)uprv_strtoul(s, &end, 16);
 489     if(end<=s || *end!=0 || value>=0x110000) {
 490         fprintf(stderr,
 491                 "error in preparsed UCD: '%s' is not a valid code point on line %ld\n",
 492                 s, (long)lineNumber);
 493         errorCode=U_PARSE_ERROR;
 494         return U_SENTINEL;
 495     }
 496     return (UChar32)value;
 497 }
 498
 499 UBool
 500 PreparsedUCD::parseCodePointRange(const char *s, UChar32 &start, UChar32 &end, UErrorCode &errorCode) {
 501     uint32_t st, e;
 502     u_parseCodePointRange(s, &st, &e, &errorCode);
 503     if(U_FAILURE(errorCode)) {
 504         fprintf(stderr,
 505                 "error in preparsed UCD: '%s' is not a valid code point range on line %ld\n",
 506                 s, (long)lineNumber);
 507         return FALSE;
 508     }
 509     start=(UChar32)st;
 510     end=(UChar32)e;
 511     return TRUE;
 512 }
 513
 514 void
 515 PreparsedUCD::parseString(const char *s, UnicodeString &uni, UErrorCode &errorCode) {
 516     UChar *buffer=uni.getBuffer(-1);
 517     int32_t length=u_parseString(s, buffer, uni.getCapacity(), NULL, &errorCode);
 518     if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
 519         errorCode=U_ZERO_ERROR;
 520         uni.releaseBuffer(0);
 521         buffer=uni.getBuffer(length);
 522         length=u_parseString(s, buffer, uni.getCapacity(), NULL, &errorCode);
 523     }
 524     uni.releaseBuffer(length);
 525     if(U_FAILURE(errorCode)) {
 526         fprintf(stderr,
 527                 "error in preparsed UCD: '%s' is not a valid Unicode string on line %ld\n",
 528                 s, (long)lineNumber);
 529     }
 530 }
 531
 532 void
 533 PreparsedUCD::parseScriptExtensions(const char *s, UnicodeSet &scx, UErrorCode &errorCode) {
 534     if(U_FAILURE(errorCode)) { return; }
 535     scx.clear();
 536     CharString scString;
 537     for(;;) {
 538         const char *scs;
 539         const char *scLimit=strchr(s, ' ');
 540         if(scLimit!=NULL) {
 541             scs=scString.clear().append(s, (int32_t)(scLimit-s), errorCode).data();
 542             if(U_FAILURE(errorCode)) { return; }
 543         } else {
 544             scs=s;
 545         }
 546         int32_t script=pnames->getPropertyValueEnum(UCHAR_SCRIPT, scs);
 547         if(script==UCHAR_INVALID_CODE) {
 548             fprintf(stderr,
 549                     "error in preparsed UCD: '%s' is not a valid script code on line %ld\n",
 550                     scs, (long)lineNumber);
 551             errorCode=U_PARSE_ERROR;
 552             return;
 553         } else if(scx.contains(script)) {
 554             fprintf(stderr,
 555                     "error in preparsed UCD: scx has duplicate '%s' codes on line %ld\n",
 556                     scs, (long)lineNumber);
 557             errorCode=U_PARSE_ERROR;
 558             return;
 559         } else {
 560             scx.add(script);
 561         }
 562         if(scLimit!=NULL) {
 563             s=scLimit+1;
 564         } else {
 565             break;
 566         }
 567     }
 568     if(scx.isEmpty()) {
 569         fprintf(stderr, "error in preparsed UCD: empty scx= on line %ld\n", (long)lineNumber);
 570         errorCode=U_PARSE_ERROR;
 571     }
 572 }
 573
 574 U_NAMESPACE_END