icuSources/tools/genrb/read.c

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4 *******************************************************************************
   5 *
   6 *   Copyright (C) 1998-2012, International Business Machines
   7 *   Corporation and others.  All Rights Reserved.
   8 *
   9 *******************************************************************************
  10 *
  11 * File read.c
  12 *
  13 * Modification History:
  14 *
  15 *   Date        Name        Description
  16 *   05/26/99    stephen     Creation.
  17 *   5/10/01     Ram         removed ustdio dependency
  18 *******************************************************************************
  19 */
  20
  21 #include "read.h"
  22 #include "errmsg.h"
  23 #include "unicode/ustring.h"
  24 #include "unicode/utf16.h"
  25
  26 #define OPENBRACE    0x007B
  27 #define CLOSEBRACE   0x007D
  28 #define COMMA        0x002C
  29 #define QUOTE        0x0022
  30 #define ESCAPE       0x005C
  31 #define SLASH        0x002F
  32 #define ASTERISK     0x002A
  33 #define SPACE        0x0020
  34 #define COLON        0x003A
  35 #define BADBOM       0xFFFE
  36 #define CR           0x000D
  37 #define LF           0x000A
  38
  39 static int32_t lineCount;
  40
  41 /* Protos */
  42 static enum ETokenType getStringToken(UCHARBUF *buf,
  43                                       UChar32 initialChar,
  44                                       struct UString *token,
  45                                       UErrorCode *status);
  46
  47 static UChar32 getNextChar           (UCHARBUF *buf, UBool skipwhite, struct UString *token, UErrorCode *status);
  48 static void    seekUntilNewline      (UCHARBUF *buf, struct UString *token, UErrorCode *status);
  49 static void    seekUntilEndOfComment (UCHARBUF *buf, struct UString *token, UErrorCode *status);
  50 static UBool   isWhitespace          (UChar32 c);
  51 static UBool   isNewline             (UChar32 c);
  52
  53 U_CFUNC void resetLineNumber() {
  54     lineCount = 1;
  55 }
  56
  57 /* Read and return the next token from the stream.  If the token is of
  58    type eString, fill in the token parameter with the token.  If the
  59    token is eError, then the status parameter will contain the
  60    specific error.  This will be eItemNotFound at the end of file,
  61    indicating that all tokens have been returned.  This method will
  62    never return eString twice in a row; instead, multiple adjacent
  63    string tokens will be merged into one, with no intervening
  64    space. */
  65 U_CFUNC enum ETokenType
  66 getNextToken(UCHARBUF* buf,
  67              struct UString *token,
  68              uint32_t *linenumber, /* out: linenumber of token */
  69              struct UString *comment,
  70              UErrorCode *status) {
  71     enum ETokenType result;
  72     UChar32         c;
  73
  74     if (U_FAILURE(*status)) {
  75         return TOK_ERROR;
  76     }
  77
  78     /* Skip whitespace */
  79     c = getNextChar(buf, TRUE, comment, status);
  80
  81     if (U_FAILURE(*status)) {
  82         return TOK_ERROR;
  83     }
  84
  85     *linenumber = lineCount;
  86
  87     switch(c) {
  88     case BADBOM:
  89         return TOK_ERROR;
  90     case OPENBRACE:
  91         return TOK_OPEN_BRACE;
  92     case CLOSEBRACE:
  93         return TOK_CLOSE_BRACE;
  94     case COMMA:
  95         return TOK_COMMA;
  96     case U_EOF:
  97         return TOK_EOF;
  98     case COLON:
  99         return TOK_COLON;
 100
 101     default:
 102         result = getStringToken(buf, c, token, status);
 103     }
 104
 105     *linenumber = lineCount;
 106     return result;
 107 }
 108
 109 /* Copy a string token into the given UnicodeString.  Upon entry, we
 110    have already read the first character of the string token, which is
 111    not a whitespace character (but may be a QUOTE or ESCAPE). This
 112    function reads all subsequent characters that belong with this
 113    string, and copy them into the token parameter. The other
 114    important, and slightly convoluted purpose of this function is to
 115    merge adjacent strings.  It looks forward a bit, and if the next
 116    non comment, non whitespace item is a string, it reads it in as
 117    well.  If two adjacent strings are quoted, they are merged without
 118    intervening space.  Otherwise a single SPACE character is
 119    inserted. */
 120 static enum ETokenType getStringToken(UCHARBUF* buf,
 121                                       UChar32 initialChar,
 122                                       struct UString *token,
 123                                       UErrorCode *status) {
 124     UBool    lastStringWasQuoted;
 125     UChar32  c;
 126     UChar    target[3] = { '\0' };
 127     UChar    *pTarget   = target;
 128     int      len=0;
 129     UBool    isFollowingCharEscaped=FALSE;
 130     UBool    isNLUnescaped = FALSE;
 131     UChar32  prevC=0;
 132
 133     /* We are guaranteed on entry that initialChar is not a whitespace
 134        character. If we are at the EOF, or have some other problem, it
 135        doesn't matter; we still want to validly return the initialChar
 136        (if nothing else) as a string token. */
 137
 138     if (U_FAILURE(*status)) {
 139         return TOK_ERROR;
 140     }
 141
 142     /* setup */
 143     lastStringWasQuoted = FALSE;
 144     c = initialChar;
 145     ustr_setlen(token, 0, status);
 146
 147     if (U_FAILURE(*status)) {
 148         return TOK_ERROR;
 149     }
 150
 151     for (;;) {
 152         if (c == QUOTE) {
 153             if (!lastStringWasQuoted && token->fLength > 0) {
 154                 ustr_ucat(token, SPACE, status);
 155
 156                 if (U_FAILURE(*status)) {
 157                     return TOK_ERROR;
 158                 }
 159             }
 160
 161             lastStringWasQuoted = TRUE;
 162
 163             for (;;) {
 164                 c = ucbuf_getc(buf,status);
 165
 166                 /* EOF reached */
 167                 if (c == U_EOF) {
 168                     return TOK_EOF;
 169                 }
 170
 171                 /* Unterminated quoted strings */
 172                 if (U_FAILURE(*status)) {
 173                     return TOK_ERROR;
 174                 }
 175
 176                 if (c == QUOTE && !isFollowingCharEscaped) {
 177                     break;
 178                 }
 179
 180                 if (c == ESCAPE  && !isFollowingCharEscaped) {
 181                     pTarget = target;
 182                     c       = unescape(buf, status);
 183
 184                     if (c == U_ERR) {
 185                         return TOK_ERROR;
 186                     }
 187                     if(c == CR || c == LF){
 188                         isNLUnescaped = TRUE;
 189                     }
 190                 }
 191
 192                 if(c==ESCAPE && !isFollowingCharEscaped){
 193                     isFollowingCharEscaped = TRUE;
 194                 }else{
 195                     U_APPEND_CHAR32(c, pTarget,len);
 196                     pTarget = target;
 197                     ustr_uscat(token, pTarget,len, status);
 198                     isFollowingCharEscaped = FALSE;
 199                     len=0;
 200                     if(c == CR || c == LF){
 201                         if(isNLUnescaped == FALSE && prevC!=CR){
 202                             lineCount++;
 203                         }
 204                         isNLUnescaped = FALSE;
 205                     }
 206                 }
 207
 208                 if (U_FAILURE(*status)) {
 209                     return TOK_ERROR;
 210                 }
 211                 prevC = c;
 212             }
 213         } else {
 214             if (token->fLength > 0) {
 215                 ustr_ucat(token, SPACE, status);
 216
 217                 if (U_FAILURE(*status)) {
 218                     return TOK_ERROR;
 219                 }
 220             }
 221
 222             if(lastStringWasQuoted){
 223                 if(getShowWarning()){
 224                     warning(lineCount, "Mixing quoted and unquoted strings");
 225                 }
 226                 if(isStrict()){
 227                     return TOK_ERROR;
 228                 }
 229
 230             }
 231
 232             lastStringWasQuoted = FALSE;
 233
 234             /* if we reach here we are mixing
 235              * quoted and unquoted strings
 236              * warn in normal mode and error in
 237              * pedantic mode
 238              */
 239
 240             if (c == ESCAPE) {
 241                 pTarget = target;
 242                 c       = unescape(buf, status);
 243
 244                 /* EOF reached */
 245                 if (c == U_EOF) {
 246                     return TOK_ERROR;
 247                 }
 248             }
 249
 250             U_APPEND_CHAR32(c, pTarget,len);
 251             pTarget = target;
 252             ustr_uscat(token, pTarget,len, status);
 253             len=0;
 254
 255             if (U_FAILURE(*status)) {
 256                 return TOK_ERROR;
 257             }
 258
 259             for (;;) {
 260                 /* DON'T skip whitespace */
 261                 c = getNextChar(buf, FALSE, NULL, status);
 262
 263                 /* EOF reached */
 264                 if (c == U_EOF) {
 265                     ucbuf_ungetc(c, buf);
 266                     return TOK_STRING;
 267                 }
 268
 269                 if (U_FAILURE(*status)) {
 270                     return TOK_STRING;
 271                 }
 272
 273                 if (c == QUOTE
 274                         || c == OPENBRACE
 275                         || c == CLOSEBRACE
 276                         || c == COMMA
 277                         || c == COLON) {
 278                     ucbuf_ungetc(c, buf);
 279                     break;
 280                 }
 281
 282                 if (isWhitespace(c)) {
 283                     break;
 284                 }
 285
 286                 if (c == ESCAPE) {
 287                     pTarget = target;
 288                     c       = unescape(buf, status);
 289
 290                     if (c == U_ERR) {
 291                         return TOK_ERROR;
 292                     }
 293                 }
 294
 295                 U_APPEND_CHAR32(c, pTarget,len);
 296                 pTarget = target;
 297                 ustr_uscat(token, pTarget,len, status);
 298                 len=0;
 299                 if (U_FAILURE(*status)) {
 300                     return TOK_ERROR;
 301                 }
 302             }
 303         }
 304
 305         /* DO skip whitespace */
 306         c = getNextChar(buf, TRUE, NULL, status);
 307
 308         if (U_FAILURE(*status)) {
 309             return TOK_STRING;
 310         }
 311
 312         if (c == OPENBRACE || c == CLOSEBRACE || c == COMMA || c == COLON) {
 313             ucbuf_ungetc(c, buf);
 314             return TOK_STRING;
 315         }
 316     }
 317 }
 318
 319 /* Retrieve the next character.  If skipwhite is
 320    true, whitespace is skipped as well. */
 321 static UChar32 getNextChar(UCHARBUF* buf,
 322                            UBool skipwhite,
 323                            struct UString *token,
 324                            UErrorCode *status) {
 325     UChar32 c, c2;
 326
 327     if (U_FAILURE(*status)) {
 328         return U_EOF;
 329     }
 330
 331     for (;;) {
 332         c = ucbuf_getc(buf,status);
 333
 334         if (c == U_EOF) {
 335             return U_EOF;
 336         }
 337
 338         if (skipwhite && isWhitespace(c)) {
 339             continue;
 340         }
 341
 342         /* This also handles the get() failing case */
 343         if (c != SLASH) {
 344             return c;
 345         }
 346
 347         c = ucbuf_getc(buf,status); /* "/c" */
 348
 349         if (c == U_EOF) {
 350             return U_EOF;
 351         }
 352
 353         switch (c) {
 354         case SLASH:  /* "//" */
 355             seekUntilNewline(buf, NULL, status);
 356             break;
 357
 358         case ASTERISK:  /* " / * " */
 359             c2 = ucbuf_getc(buf, status); /* "/ * c" */
 360             if(c2 == ASTERISK){  /* "/ * *" */
 361                 /* parse multi-line comment and store it in token*/
 362                 seekUntilEndOfComment(buf, token, status);
 363             } else {
 364                 ucbuf_ungetc(c2, buf); /* c2 is the non-asterisk following "/ *".  Include c2  back in buffer.  */
 365                 seekUntilEndOfComment(buf, NULL, status);
 366             }
 367             break;
 368
 369         default:
 370             ucbuf_ungetc(c, buf); /* "/c" - put back the c */
 371             /* If get() failed this is a NOP */
 372             return SLASH;
 373         }
 374
 375     }
 376 }
 377
 378 static void seekUntilNewline(UCHARBUF* buf,
 379                              struct UString *token,
 380                              UErrorCode *status) {
 381     UChar32 c;
 382
 383     if (U_FAILURE(*status)) {
 384         return;
 385     }
 386
 387     do {
 388         c = ucbuf_getc(buf,status);
 389         /* add the char to token */
 390         if(token!=NULL){
 391             ustr_u32cat(token, c, status);
 392         }
 393     } while (!isNewline(c) && c != U_EOF && *status == U_ZERO_ERROR);
 394 }
 395
 396 static void seekUntilEndOfComment(UCHARBUF *buf,
 397                                   struct UString *token,
 398                                   UErrorCode *status) {
 399     UChar32  c, d;
 400     uint32_t line;
 401
 402     if (U_FAILURE(*status)) {
 403         return;
 404     }
 405
 406     line = lineCount;
 407
 408     do {
 409         c = ucbuf_getc(buf, status);
 410
 411         if (c == ASTERISK) {
 412             d = ucbuf_getc(buf, status);
 413
 414             if (d != SLASH) {
 415                 ucbuf_ungetc(d, buf);
 416             } else {
 417                 break;
 418             }
 419         }
 420         /* add the char to token */
 421         if(token!=NULL){
 422             ustr_u32cat(token, c, status);
 423         }
 424         /* increment the lineCount */
 425         isNewline(c);
 426
 427     } while (c != U_EOF && *status == U_ZERO_ERROR);
 428
 429     if (c == U_EOF) {
 430         *status = U_INVALID_FORMAT_ERROR;
 431         error(line, "unterminated comment detected");
 432     }
 433 }
 434
 435 U_CFUNC UChar32 unescape(UCHARBUF *buf, UErrorCode *status) {
 436     if (U_FAILURE(*status)) {
 437         return U_EOF;
 438     }
 439
 440     /* We expect to be called after the ESCAPE has been seen, but
 441      * u_fgetcx needs an ESCAPE to do its magic. */
 442     ucbuf_ungetc(ESCAPE, buf);
 443
 444     return ucbuf_getcx32(buf, status);
 445 }
 446
 447 static UBool isWhitespace(UChar32 c) {
 448     switch (c) {
 449         /* ' ', '\t', '\n', '\r', 0x2029, 0xFEFF */
 450     case 0x000A:
 451     case 0x2029:
 452         lineCount++;
 453     case 0x000D:
 454     case 0x0020:
 455     case 0x0009:
 456     case 0xFEFF:
 457         return TRUE;
 458
 459     default:
 460         return FALSE;
 461     }
 462 }
 463
 464 static UBool isNewline(UChar32 c) {
 465     switch (c) {
 466         /* '\n', '\r', 0x2029 */
 467     case 0x000A:
 468     case 0x2029:
 469         lineCount++;
 470     case 0x000D:
 471         return TRUE;
 472
 473     default:
 474         return FALSE;
 475     }
 476 }