X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/b75a7d8f3b4adbae880cab104ce2c6a50eee4db2..d5d484b0fbe924d3663b177965538d517ee412c1:/icuSources/tools/genprops/props2.c

diff --git a/icuSources/tools/genprops/props2.c b/icuSources/tools/genprops/props2.c
index 8dc53441..9f18a11b 100644
--- a/icuSources/tools/genprops/props2.c
+++ b/icuSources/tools/genprops/props2.c
@@ -1,7 +1,7 @@
 /*
 *******************************************************************************
 *
-*   Copyright (C) 2002-2003, International Business Machines
+*   Copyright (C) 2002-2006, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 *
 *******************************************************************************
@@ -27,6 +27,7 @@
 #include "uprops.h"
 #include "propsvec.h"
 #include "uparse.h"
+#include "writesrc.h"
 #include "genprops.h"
 
 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
@@ -102,11 +103,6 @@ numericLineFn(void *context,
               char *fields[][2], int32_t fieldCount,
               UErrorCode *pErrorCode);
 
-static void U_CALLCONV
-bidiClassLineFn(void *context,
-                char *fields[][2], int32_t fieldCount,
-                UErrorCode *pErrorCode);
-
 /* parse files with single enumerated properties ---------------------------- */
 
 struct SingleEnum {
@@ -134,6 +130,24 @@ static const SingleEnum blockSingleEnum={
     0, UPROPS_BLOCK_SHIFT, UPROPS_BLOCK_MASK
 };
 
+static const SingleEnum graphemeClusterBreakSingleEnum={
+    "GraphemeBreakProperty", "Grapheme_Cluster_Break",
+    UCHAR_GRAPHEME_CLUSTER_BREAK,
+    2, UPROPS_GCB_SHIFT, UPROPS_GCB_MASK
+};
+
+static const SingleEnum wordBreakSingleEnum={
+    "WordBreakProperty", "Word_Break",
+    UCHAR_WORD_BREAK,
+    2, UPROPS_WB_SHIFT, UPROPS_WB_MASK
+};
+
+static const SingleEnum sentenceBreakSingleEnum={
+    "SentenceBreakProperty", "Sentence_Break",
+    UCHAR_SENTENCE_BREAK,
+    2, UPROPS_SB_SHIFT, UPROPS_SB_MASK
+};
+
 static const SingleEnum lineBreakSingleEnum={
     "LineBreak", "line break",
     UCHAR_LINE_BREAK,
@@ -146,18 +160,6 @@ static const SingleEnum eawSingleEnum={
     0, UPROPS_EA_SHIFT, UPROPS_EA_MASK
 };
 
-static const SingleEnum jtSingleEnum={
-    "DerivedJoiningType", "joining type",
-    UCHAR_JOINING_TYPE,
-    2, UPROPS_JT_SHIFT, UPROPS_JT_MASK
-};
-
-static const SingleEnum jgSingleEnum={
-    "DerivedJoiningGroup", "joining group",
-    UCHAR_JOINING_GROUP,
-    2, UPROPS_JG_SHIFT, UPROPS_JG_MASK
-};
-
 static void U_CALLCONV
 singleEnumLineFn(void *context,
                  char *fields[][2], int32_t fieldCount,
@@ -199,7 +201,7 @@ singleEnumLineFn(void *context,
     uv=(uint32_t)(value<<sen->vecShift);
     if((uv&sen->vecMask)!=uv) {
         fprintf(stderr, "genprops error: %s value overflow (0x%x) at %s\n",
-                        sen->propName, uv, s);
+                        sen->propName, (int)uv, s);
         exit(U_INTERNAL_PROGRAM_ERROR);
     }
 
@@ -246,8 +248,6 @@ typedef struct Binaries Binaries;
 static const Binary
 propListNames[]={
     { "White_Space",                        1, UPROPS_WHITE_SPACE },
-    { "Bidi_Control",                       1, UPROPS_BIDI_CONTROL },
-    { "Join_Control",                       1, UPROPS_JOIN_CONTROL },
     { "Dash",                               1, UPROPS_DASH },
     { "Hyphen",                             1, UPROPS_HYPHEN },
     { "Quotation_Mark",                     1, UPROPS_QUOTATION_MARK },
@@ -264,8 +264,15 @@ propListNames[]={
     { "Radical",                            1, UPROPS_RADICAL },
     { "Unified_Ideograph",                  1, UPROPS_UNIFIED_IDEOGRAPH },
     { "Deprecated",                         1, UPROPS_DEPRECATED },
-    { "Soft_Dotted",                        1, UPROPS_SOFT_DOTTED },
-    { "Logical_Order_Exception",            1, UPROPS_LOGICAL_ORDER_EXCEPTION }
+    { "Logical_Order_Exception",            1, UPROPS_LOGICAL_ORDER_EXCEPTION },
+
+    /* new properties in Unicode 4.0.1 */
+    { "STerm",                              2, UPROPS_V2_S_TERM },
+    { "Variation_Selector",                 2, UPROPS_V2_VARIATION_SELECTOR },
+
+    /* new properties in Unicode 4.1 */
+    { "Pattern_Syntax",                     2, UPROPS_V2_PATTERN_SYNTAX },
+    { "Pattern_White_Space",                2, UPROPS_V2_PATTERN_WHITE_SPACE }
 };
 
 static const Binaries
@@ -281,15 +288,19 @@ derCorePropsNames[]={
     /* before Unicode 4/ICU 2.6/format version 3.2, these used to be Other_XYZ from PropList.txt */
     { "Math",                               1, UPROPS_MATH },
     { "Alphabetic",                         1, UPROPS_ALPHABETIC },
-    { "Lowercase",                          1, UPROPS_LOWERCASE },
-    { "Uppercase",                          1, UPROPS_UPPERCASE },
     { "Grapheme_Extend",                    1, UPROPS_GRAPHEME_EXTEND },
     { "Default_Ignorable_Code_Point",       1, UPROPS_DEFAULT_IGNORABLE_CODE_POINT },
 
     /* new properties bits in ICU 2.6/format version 3.2 */
     { "ID_Start",                           1, UPROPS_ID_START },
     { "ID_Continue",                        1, UPROPS_ID_CONTINUE },
-    { "Grapheme_Base",                      1, UPROPS_GRAPHEME_BASE }
+    { "Grapheme_Base",                      1, UPROPS_GRAPHEME_BASE },
+
+    /*
+     * Unicode 5/ICU 3.6 moves Grapheme_Link from PropList.txt
+     * to DerivedCoreProperties.txt and deprecates it.
+     */
+    { "Grapheme_Link",                      1, UPROPS_GRAPHEME_LINK }
 };
 
 static const Binaries
@@ -336,7 +347,9 @@ binariesLineFn(void *context,
     for(i=0;; ++i) {
         if(i==bin->binariesCount) {
             /* ignore unrecognized properties */
-            addIgnoredProp(s, fields[1][1]);
+            if(beVerbose) {
+                addIgnoredProp(s, fields[1][1]);
+            }
             return;
         }
         if(isToken(bin->binaries[i].propName, s)) {
@@ -346,7 +359,7 @@ binariesLineFn(void *context,
 
     if(bin->binaries[i].vecShift>=32) {
         fprintf(stderr, "genprops error: shift value %d>=32 for %s %s\n",
-                        bin->binaries[i].vecShift, bin->ucdFile, bin->binaries[i].propName);
+                        (int)bin->binaries[i].vecShift, bin->ucdFile, bin->binaries[i].propName);
         exit(U_INTERNAL_PROGRAM_ERROR);
     }
     uv=U_MASK(bin->binaries[i].vecShift);
@@ -378,8 +391,10 @@ parseBinariesFile(char *filename, char *basename, const char *suffix,
         fprintf(stderr, "error parsing %s.txt: %s\n", bin->ucdFile, u_errorName(*pErrorCode));
     }
 
-    for(i=0; i<ignoredPropsCount; ++i) {
-        printf("genprops: ignoring property %s in %s.txt\n", ignoredProps[i], bin->ucdFile);
+    if(beVerbose) {
+        for(i=0; i<ignoredPropsCount; ++i) {
+            printf("genprops: ignoring property %s in %s.txt\n", ignoredProps[i], bin->ucdFile);
+        }
     }
 }
 
@@ -390,6 +405,12 @@ initAdditionalProperties() {
     pv=upvec_open(UPROPS_VECTOR_WORDS, 20000);
 }
 
+U_CFUNC void
+exitAdditionalProperties() {
+    utrie_close(trie);
+    upvec_close(pv);
+}
+
 U_CFUNC void
 generateAdditionalProperties(char *filename, const char *suffix, UErrorCode *pErrorCode) {
     char *basename;
@@ -399,10 +420,7 @@ generateAdditionalProperties(char *filename, const char *suffix, UErrorCode *pEr
     /* process various UCD .txt files */
 
     /* add Han numeric types & values */
-    parseMultiFieldFile(filename, basename, "DerivedNumericValues", suffix, 3, numericLineFn, pErrorCode);
-
-    /* set proper bidi class for unassigned code points (Cn) */
-    parseTwoFieldFile(filename, basename, "DerivedBidiClass", suffix, bidiClassLineFn, pErrorCode);
+    parseMultiFieldFile(filename, basename, "DerivedNumericValues", suffix, 2, numericLineFn, pErrorCode);
 
     parseTwoFieldFile(filename, basename, "DerivedAge", suffix, ageLineFn, pErrorCode);
 
@@ -428,6 +446,12 @@ generateAdditionalProperties(char *filename, const char *suffix, UErrorCode *pEr
 
     parseBinariesFile(filename, basename, suffix, &derCorePropsBinaries, pErrorCode);
 
+    parseSingleEnumFile(filename, basename, suffix, &graphemeClusterBreakSingleEnum, pErrorCode);
+
+    parseSingleEnumFile(filename, basename, suffix, &wordBreakSingleEnum, pErrorCode);
+
+    parseSingleEnumFile(filename, basename, suffix, &sentenceBreakSingleEnum, pErrorCode);
+
     /*
      * LineBreak-4.0.0.txt:
      *  - All code points, assigned and unassigned, that are not listed 
@@ -437,10 +461,6 @@ generateAdditionalProperties(char *filename, const char *suffix, UErrorCode *pEr
      */
     parseSingleEnumFile(filename, basename, suffix, &lineBreakSingleEnum, pErrorCode);
 
-    parseSingleEnumFile(filename, basename, suffix, &jtSingleEnum, pErrorCode);
-
-    parseSingleEnumFile(filename, basename, suffix, &jgSingleEnum, pErrorCode);
-
     /*
      * Preset East Asian Width defaults:
      *
@@ -470,14 +490,14 @@ generateAdditionalProperties(char *filename, const char *suffix, UErrorCode *pEr
     /* parse EastAsianWidth.txt */
     parseSingleEnumFile(filename, basename, suffix, &eawSingleEnum, pErrorCode);
 
-    trie=utrie_open(NULL, NULL, 50000, 0, TRUE);
+    trie=utrie_open(NULL, NULL, 50000, 0, 0, TRUE);
     if(trie==NULL) {
         *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
         upvec_close(pv);
         return;
     }
 
-    pvCount=upvec_toTrie(pv, trie, pErrorCode);
+    pvCount=upvec_compact(pv, upvec_compactToTrieHandler, trie, pErrorCode);
     if(U_FAILURE(*pErrorCode)) {
         fprintf(stderr, "genprops error: unable to build trie for additional properties: %s\n", u_errorName(*pErrorCode));
         exit(*pErrorCode);
@@ -500,8 +520,13 @@ ageLineFn(void *context,
     }
     ++limit;
 
-    /* parse version number */
+    /* ignore "unassigned" (the default is already set to 0.0) */
     s=(char *)u_skipWhitespace(fields[1][0]);
+    if(0==uprv_strncmp(s, "unassigned", 10)) {
+        return;
+    }
+
+    /* parse version number */
     value=(uint32_t)uprv_strtoul(s, &end, 10);
     if(s==end || value==0 || value>15 || (*end!='.' && *end!=' ' && *end!='\t' && *end!=0)) {
         fprintf(stderr, "genprops: syntax error in DerivedAge.txt field 1 at %s\n", fields[1][0]);
@@ -534,10 +559,10 @@ static void U_CALLCONV
 numericLineFn(void *context,
               char *fields[][2], int32_t fieldCount,
               UErrorCode *pErrorCode) {
-    Props newProps;
+    Props newProps={ 0 };
     char *s, *end;
     uint32_t start, limit, value, oldProps32;
-    int32_t type, oldType;
+    int32_t oldType;
     char c;
     UBool isFraction;
 
@@ -571,11 +596,14 @@ numericLineFn(void *context,
         /* try large powers of 10 first, may otherwise overflow strtoul() */
         if(0==uprv_strncmp(s, "10000000000", 11)) {
             /* large powers of 10 are encoded in a special way, see store.c */
-            value=0x7fffff00;
+            uint8_t exp=0;
+
             end=s;
             while(*(++end)=='0') {
-                ++value;
+                ++exp;
             }
+            value=1;
+            newProps.exponent=exp;
         } else {
             /* normal number parsing */
             value=(uint32_t)uprv_strtoul(s, &end, 10);
@@ -586,144 +614,103 @@ numericLineFn(void *context,
         }
     }
 
-    /* parse numeric type */
-    s=trimTerminateField(fields[2][0], fields[2][1]);
-    type=u_getPropertyValueEnum(UCHAR_NUMERIC_TYPE, s);
-    if(type<=0) {
-        fprintf(stderr, "genprops error: unknown numeric type in DerivedNumericValues.txt field 1 at %s\n", s);
-        exit(U_PARSE_ERROR);
-    }
+    /*
+     * Unicode 4.0.1 removes the third column that used to list the numeric type.
+     * Assume that either the data is the same as in UnicodeData.txt,
+     * or else that the numeric type is "numeric".
+     * This should work because we only expect to add numeric values for
+     * Han characters; for those, UnicodeData.txt lists only ranges without
+     * specific properties for single characters.
+     */
+
+    /* set the new numeric type and value */
+    newProps.numericType=(uint8_t)U_NT_NUMERIC; /* assumed numeric type, see Unicode 4.0.1 comment */
+    newProps.numericValue=(int32_t)value;       /* newly parsed numeric value */
+    /* the exponent may have been set above */
+    value=makeProps(&newProps);
 
     for(; start<limit; ++start) {
         oldProps32=getProps(start);
         oldType=(int32_t)GET_NUMERIC_TYPE(oldProps32);
-        if(oldType==type) {
-            /* this code point was already listed with its numeric value in UnicodeData.txt */
-            continue;
-        }
-        if(oldType!=0) {
-            /* the numeric type differs from what we got from UnicodeData.txt */
-            fprintf(stderr, "genprops error: new numeric value for an already numeric character in DerivedNumericValues.txt at %s\n", fields[0][0]);
-            exit(U_PARSE_ERROR);
+
+        if(isFraction) {
+            if(oldType!=0) {
+                /* this code point was already listed with its numeric value in UnicodeData.txt */
+                continue;
+            } else {
+                fprintf(stderr, "genprops: not prepared for new fractions in DerivedNumericValues.txt field 1 at %s\n", fields[1][0]);
+                exit(U_PARSE_ERROR);
+            }
         }
 
         /*
-         * Do not set a numeric value for code points that have other
-         * values or exceptions because the code below is not prepared
-         * to maintain such values and exceptions.
-         *
-         * Check store.c (e.g., file format description and makeProps())
-         * for details of what code points get their value field interpreted.
-         * For example, case mappings for Ll/Lt/Lu and mirror mappings for mirrored characters.
-         *
          * For simplicity, and because we only expect to set numeric values for Han characters,
          * for now we only allow to set these values for Lo characters.
          */
-        if(GET_UNSIGNED_VALUE(oldProps32)!=0 || PROPS_VALUE_IS_EXCEPTION(oldProps32) || GET_CATEGORY(oldProps32)!=U_OTHER_LETTER) {
-            fprintf(stderr, "genprops error: new numeric value for a character with some other value in DerivedNumericValues.txt at %s\n", fields[0][0]);
+        if(oldType==0 && GET_CATEGORY(oldProps32)!=U_OTHER_LETTER) {
+            fprintf(stderr, "genprops error: new numeric value for a character other than Lo in DerivedNumericValues.txt at %s\n", fields[0][0]);
             exit(U_PARSE_ERROR);
         }
 
-        if(isFraction) {
-            fprintf(stderr, "genprops: not prepared for new fractions in DerivedNumericValues.txt field 1 at %s\n", fields[1][0]);
-            exit(U_PARSE_ERROR);
-        }
-
-        if(beVerbose) {
-            printf("adding U+%04x numeric type %d value %u\n", start, type, value);
-        }
-
-        /* reconstruct the properties and set the new numeric type and value */
-        uprv_memset(&newProps, 0, sizeof(newProps));
-        newProps.code=start;
-        newProps.generalCategory=(uint8_t)GET_CATEGORY(oldProps32);
-        newProps.bidi=(uint8_t)GET_BIDI_CLASS(oldProps32);
-        newProps.isMirrored=(uint8_t)(oldProps32&(1UL<<UPROPS_MIRROR_SHIFT) ? TRUE : FALSE);
-        newProps.numericType=(uint8_t)type;     /* newly parsed numeric type */
-        newProps.numericValue=(int32_t)value;   /* newly parsed numeric value */
-        addProps(start, makeProps(&newProps));
-    }
-}
-
-/* DerivedBidiClass.txt ----------------------------------------------------- */
-
-static void U_CALLCONV
-bidiClassLineFn(void *context,
-                char *fields[][2], int32_t fieldCount,
-                UErrorCode *pErrorCode) {
-    char *s;
-    uint32_t oldStart, start, limit, value, props32;
-    UBool didSet;
-
-    /* get the code point range */
-    u_parseCodePointRange(fields[0][0], &start, &limit, pErrorCode);
-    if(U_FAILURE(*pErrorCode)) {
-        fprintf(stderr, "genprops: syntax error in DerivedBidiClass.txt field 0 at %s\n", fields[0][0]);
-        exit(*pErrorCode);
-    }
-    ++limit;
-
-    /* parse bidi class */
-    s=trimTerminateField(fields[1][0], fields[1][1]);
-    value=u_getPropertyValueEnum(UCHAR_BIDI_CLASS, s);
-    if(value<0) {
-        fprintf(stderr, "genprops error: unknown bidi class in DerivedBidiClass.txt field 1 at %s\n", s);
-        exit(U_PARSE_ERROR);
-    }
-
-    didSet=FALSE;
-    oldStart=start;
-    for(; start<limit; ++start) {
-        props32=getProps(start);
-
-        /* ignore if this bidi class is already set */
-        if(value==GET_BIDI_CLASS(props32)) {
-            continue;
-        }
+        /* verify that we do not change an existing value (fractions were excluded above) */
+        if(oldType!=0) {
+            /* the code point already has a value stored */
+            if((oldProps32&0xff00)!=(value&0xff00)) {
+                fprintf(stderr, "genprops error: new numeric value differs from old one for U+%04lx\n", (long)start);
+                exit(U_PARSE_ERROR);
+            }
+            /* same value, continue */
+        } else {
+            /* the code point is getting a new numeric value */
+            if(beVerbose) {
+                printf("adding U+%04x numeric type %d value 0x%04x from %s\n", (int)start, U_NT_NUMERIC, (int)value, fields[0][0]);
+            }
 
-        /* ignore old bidi class, set only for unassigned code points (Cn) */
-        if(GET_CATEGORY(props32)!=0) {
-            /* error if this one contradicts what we parsed from UnicodeData.txt */
-            fprintf(stderr, "genprops error: different bidi class in DerivedBidiClass.txt field 1 at %s\n", s);
-            exit(U_PARSE_ERROR);
+            addProps(start, value|GET_CATEGORY(oldProps32));
         }
-
-        /* remove whatever bidi class was set before */
-        props32&=~(0x1f<<UPROPS_BIDI_SHIFT);
-
-        /* set bidi class for Cn according to DerivedBidiClass.txt */
-        props32|=value<<UPROPS_BIDI_SHIFT;
-
-        /* set the modified properties */
-        addProps(start, props32);
-        didSet=TRUE;
-    }
-
-    if(didSet && beVerbose) {
-        printf("setting U+%04x..U+%04x bidi class %d\n", oldStart, limit-1, value);
     }
 }
 
 /* data serialization ------------------------------------------------------- */
 
 U_CFUNC int32_t
-writeAdditionalData(uint8_t *p, int32_t capacity, int32_t indexes[UPROPS_INDEX_COUNT]) {
+writeAdditionalData(FILE *f, uint8_t *p, int32_t capacity, int32_t indexes[UPROPS_INDEX_COUNT]) {
     int32_t length;
     UErrorCode errorCode;
 
     errorCode=U_ZERO_ERROR;
-    length=utrie_serialize(trie, p, capacity, getFoldedPropsValue, TRUE, &errorCode);
+    length=utrie_serialize(trie, p, capacity, NULL, TRUE, &errorCode);
     if(U_FAILURE(errorCode)) {
         fprintf(stderr, "genprops error: unable to serialize trie for additional properties: %s\n", u_errorName(errorCode));
         exit(errorCode);
     }
     if(p!=NULL) {
-        p+=length;
-        capacity-=length;
         if(beVerbose) {
-            printf("size in bytes of additional props trie:%5u\n", length);
+            printf("size in bytes of additional props trie:%5u\n", (int)length);
+        }
+        if(f!=NULL) {
+            UTrie trie2={ NULL };
+            utrie_unserialize(&trie2, p, length, &errorCode);
+            if(U_FAILURE(errorCode)) {
+                fprintf(
+                    stderr,
+                    "genprops error: failed to utrie_unserialize(trie for additional properties) - %s\n",
+                    u_errorName(errorCode));
+                exit(errorCode);
+            }
+            usrc_writeUTrieArrays(f,
+                "static const uint16_t propsVectorsTrie_index[%ld]={\n", NULL,
+                &trie2,
+                "\n};\n\n");
+            usrc_writeUTrieStruct(f,
+                "static const UTrie propsVectorsTrie={\n",
+                &trie2, "propsVectorsTrie_index", NULL, NULL,
+                "};\n\n");
         }
 
+        p+=length;
+        capacity-=length;
+
         /* set indexes */
         indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]=
             indexes[UPROPS_ADDITIONAL_TRIE_INDEX]+length/4;
@@ -737,23 +724,29 @@ writeAdditionalData(uint8_t *p, int32_t capacity, int32_t indexes[UPROPS_INDEX_C
             (((int32_t)UBLOCK_COUNT-1)<<UPROPS_BLOCK_SHIFT)|
             ((int32_t)USCRIPT_CODE_LIMIT-1);
         indexes[UPROPS_MAX_VALUES_2_INDEX]=
-            (((int32_t)U_JT_COUNT-1)<<UPROPS_JT_SHIFT)|
-            (((int32_t)U_JG_COUNT-1)<<UPROPS_JG_SHIFT)|
+            (((int32_t)U_SB_COUNT-1)<<UPROPS_SB_SHIFT)|
+            (((int32_t)U_WB_COUNT-1)<<UPROPS_WB_SHIFT)|
+            (((int32_t)U_GCB_COUNT-1)<<UPROPS_GCB_SHIFT)|
             ((int32_t)U_DT_COUNT-1);
     }
 
     if(p!=NULL && (pvCount*4)<=capacity) {
-        uprv_memcpy(p, pv, pvCount*4);
+        if(f!=NULL) {
+            usrc_writeArray(f,
+                "static const uint32_t propsVectors[%ld]={\n",
+                pv, 32, pvCount,
+                "};\n\n");
+            fprintf(f, "static const int32_t countPropsVectors=%ld;\n", (long)pvCount);
+            fprintf(f, "static const int32_t propsVectorsColumns=%ld;\n", (long)indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]);
+        } else {
+            uprv_memcpy(p, pv, pvCount*4);
+        }
         if(beVerbose) {
-            printf("number of additional props vectors:    %5u\n", pvCount/UPROPS_VECTOR_WORDS);
+            printf("number of additional props vectors:    %5u\n", (int)pvCount/UPROPS_VECTOR_WORDS);
             printf("number of 32-bit words per vector:     %5u\n", UPROPS_VECTOR_WORDS);
         }
     }
     length+=pvCount*4;
 
-    if(p!=NULL) {
-        utrie_close(trie);
-        upvec_close(pv);
-    }
     return length;
 }