ICU-461.17.tar.gz

[apple/icu.git] / icuSources / common / rbbicst.pl

diff --git a/icuSources/common/rbbicst.pl b/icuSources/common/rbbicst.pl
index e31b35ef154aee92ae691ca9aa8d1e635ca6778e..98b06cbc27d0b786f3f159300155d90eee1ae776 100644 (file)
--- a/icuSources/common/rbbicst.pl
+++ b/icuSources/common/rbbicst.pl
@@ -1,19 +1,20 @@
 #**************************************************************************
 #**************************************************************************

-#   Copyright (C) 2002-2003 International Business Machines Corporation   *
+#   Copyright (C) 2002-2005 International Business Machines Corporation   *

 #   and others. All rights reserved.                                      *
 #**************************************************************************
 #
 #  rbbicst   Compile the RBBI rule paser state table data into initialized C data.
 #            Usage:
 #                   cd icu/source/common
 #   and others. All rights reserved.                                      *
 #**************************************************************************
 #
 #  rbbicst   Compile the RBBI rule paser state table data into initialized C data.
 #            Usage:
 #                   cd icu/source/common

-#                   perl rbbicst.pl [-j] < rbbirpt.txt > rbbirpt.h
+#                   perl rbbicst.pl    < rbbirpt.txt > rbbirpt.h
+#                   perl rbbicst.pl -j < rbbirpt.txt > RBBIRuleParseTable.java

 #
 #             The output file, rbbrpt.h, is included by some of the .cpp rbbi
 #             implementation files.   This perl script is NOT run as part
 #             of a normal ICU build.  It is run by hand when needed, and the
 #             rbbirpt.h generated file is put back into cvs.
 #
 #
 #             The output file, rbbrpt.h, is included by some of the .cpp rbbi
 #             implementation files.   This perl script is NOT run as part
 #             of a normal ICU build.  It is run by hand when needed, and the
 #             rbbirpt.h generated file is put back into cvs.
 #

-#             See rbbirpt.h for a description of the input format for this script.
+#             See rbbirpt.txt for a description of the input format for this script.

 #
 
 if ($ARGV[0] eq "-j") {
 #
 
 if ($ARGV[0] eq "-j") {


@@ -106,9 +107,9 @@ line_loop: while (<>) {
     #
     # do the 'n' flag
     #
     #
     # do the 'n' flag
     #

-    $state_flag[$num_states] = "FALSE";
+    $state_flag[$num_states] = $javaOutput? "false" : "FALSE";

     if ($fields[0] eq "n") {
     if ($fields[0] eq "n") {

-        $state_flag[$num_states] = "TRUE";
+        $state_flag[$num_states] = $javaOutput? "true": "TRUE";

         shift @fields;
     }
 
         shift @fields;
     }
 


@@ -197,18 +198,42 @@ for ($state=1; $state<$num_states; $state++) {
 
 die if ($errors>0);
 
 
 die if ($errors>0);
 

+#
+# Assign numbers to each of the character classes classes  used.
+#   Sets are numbered from 128 - 250
+#   The values 0-127 in the state table are used for matching
+#     individual ASCII characters (the only thing that can appear in the rules.)
+#   The "set" names appearing in the code below (default, etc.)  need special
+#     handling because they do not correspond to a normal set of characters,
+#     but trigger special handling by code in the state machine.
+#
+$i = 128;
+foreach $setName (sort keys %charClasses) {
+    if ($setName eq "default") {
+        $charClasses{$setName} = 255;}
+    elsif ($setName eq "escaped") {
+        $charClasses{$setName} = 254;}
+    elsif ($setName eq "escapedP") {
+        $charClasses{$setName} = 253;}
+    elsif ($setName eq "eof") {
+        $charClasses{$setName} = 252;}
+    else {
+        # Normal (single) character class.  Number them.
+        $charClasses{$setName} = $i;
+        $i++;
+    }
+}
+
+
+my ($sec, $min, $hour, , $day, $mon, $year, $wday, $yday, $isdst) = localtime;
+$year += 1900;
+

 if ($javaOutput) {
     print "/*\n";
     print " *******************************************************************************\n";
 if ($javaOutput) {
     print "/*\n";
     print " *******************************************************************************\n";

-    print " * Copyright (C) 2003,\n";
+    print " * Copyright (C) 2003-$year,\n";

     print " * International Business Machines Corporation and others. All Rights Reserved.\n";
     print " *******************************************************************************\n";
     print " * International Business Machines Corporation and others. All Rights Reserved.\n";
     print " *******************************************************************************\n";

-    print " *\n";
-    print " * \$Source: /cvs/root/ICU/icuSources/common/rbbicst.pl,v $\n";
-    print " * \$Date: 2003/07/03 18:13:31 $\n";
-    print " * \$Revision: 1.1.1.2 $\n";
-    print " *\n";
-    print " *******************************************************************************\n";

     print " */\n";
     print " \n";
     print "package com.ibm.icu.text;\n";
     print " */\n";
     print " \n";
     print "package com.ibm.icu.text;\n";


@@ -219,59 +244,87 @@ if ($javaOutput) {
     print " * rule parser.\n";
     print " * It is generated by the Perl script \"rbbicst.pl\" from\n";
     print " * the rule parser state definitions file \"rbbirpt.txt\".\n";
     print " * rule parser.\n";
     print " * It is generated by the Perl script \"rbbicst.pl\" from\n";
     print " * the rule parser state definitions file \"rbbirpt.txt\".\n";

+    print " * \@internal \n";

     print " *\n";
     print " */\n";
 
     print " *\n";
     print " */\n";
 

-    print "public class RuleBasedBreakIteratorStateTable\n";
+    print "class RBBIRuleParseTable\n";

     print "{\n";
 
     print "{\n";
 

-    #
+     #

     # Emit the constants for the actions to be performed.
     #
     $n = 1;
     # Emit the constants for the actions to be performed.
     #
     $n = 1;

-    foreach $act (keys %actions) {
-        print "    public static final int $act = $n;\n";
+    foreach $act (sort keys %actions) {
+        print "     static final short $act = $n;\n";

         $n++;
     }
     print " \n";
         $n++;
     }
     print " \n";

+    

     #
     #

-    # emit the state transition table
+    # Emit constants for char class names

     #
     #

-    print "    public static final String[] gRuleParseStateTable = {\n";
-    printf("        \"\\u%04.4x\\u%04.4x\\u%04.4x\\u%04.4x\\u%04.4x\"\n", doNOP, 0, 0, 0, 1);
+    foreach $setName (sort keys %charClasses) {
+       print "     static final short kRuleSet_$setName = $charClasses{$setName};\n";
+    }
+    print "\n\n";
+    
+    
+    print "   static class RBBIRuleTableElement { \n";
+    print "      short      fAction; \n";
+    print "      short      fCharClass; \n";
+    print "      short      fNextState; \n";
+    print "      short      fPushState; \n";
+    print "      boolean    fNextChar;  \n";
+    print "      String     fStateName; \n";
+    print "      RBBIRuleTableElement(short a, int cc, int ns, int ps, boolean nc, String sn) {  \n";
+    print "      fAction = a; \n";
+    print "      fCharClass = (short)cc; \n";
+    print "      fNextState = (short)ns; \n";
+    print "      fPushState = (short)ps; \n";
+    print "      fNextChar  = nc; \n";
+    print "      fStateName = sn; \n";
+    print "   } \n";
+    print "   }; \n";
+    print "  \n";
+    
+    
+    print "    static RBBIRuleTableElement[] gRuleParseStateTable = { \n ";
+    print "      new RBBIRuleTableElement(doNOP, 0, 0,0,  true,   null )     //  0 \n";  #output the unused state 0. 

     for ($state=1; $state < $num_states; $state++) {
     for ($state=1; $state < $num_states; $state++) {

-        printf("        , \"\\u%04.4x", $state_func_name[$state]);
-        # print "  , {$state_func_name[$state],";
+        print "     , new RBBIRuleTableElement($state_func_name[$state],";

         if ($state_literal_chars[$state] ne "") {
         if ($state_literal_chars[$state] ne "") {

-            printf("\\u%04.4x", $state_func_name[$state]);
+            $c = $state_literal_chars[$state];
+            print("'$c', "); 

         }else {
         }else {

-            printf("\\u%04.4x", $charClasses{$state_char_class[$state]});
+            print " $charClasses{$state_char_class[$state]},";

         }
         }

-        printf("\\u%04.4x", $states{$state_dest_state[$state]});
-
+        print " $states{$state_dest_state[$state]},";
+ 

         # The push-state field is optional.  If omitted, fill field with a zero, which flags
         #   the state machine that there is no push state.
         if ($state_push_state[$state] eq "") {
         # The push-state field is optional.  If omitted, fill field with a zero, which flags
         #   the state machine that there is no push state.
         if ($state_push_state[$state] eq "") {

-            print "\\u0000";
+            print "0, ";

         } else {
         } else {

-           printf("\\u%04.4x", $states{$state_push_state[$state]});
+            print " $states{$state_push_state[$state]},";

         }
         }

-        printf("\\u%04.4x", $state_flag[$state]);
-
-        # For the first row of each state, append the state name.
-        #   Used for debugging only.
+        print " $state_flag[$state], ";
+ 
+        # if this is the first row of the table for this state, put out the state name.

         if ($stateNames[$state] ne "") {
         if ($stateNames[$state] ne "") {

-            printf("%-20s", $stateNames[$state]."\"");
+            print "  \"$stateNames[$state]\") ";

         } else {
         } else {

-            printf("%-20s", "\"");
+            print "  null ) ";

         }
         }

-
-        # Put out a C++ comment showing the number (index) of this state row,
-        print "        //  $state ";
+            
+        # Put out a comment showing the number (index) of this state row,
+        print "    //  $state ";

         print "\n";
         print "\n";

-    };
-    print "    };\n";
-    print "}\n";
+    }
+    print " };\n";
+
+    print "}; \n";
+    

 }
 else
 {
 }
 else
 {


@@ -288,7 +341,7 @@ else
     print "//    It is generated by the Perl script \"rbbicst.pl\" from\n";
     print "//    the rule parser state definitions file \"rbbirpt.txt\".\n";
     print "//\n";
     print "//    It is generated by the Perl script \"rbbicst.pl\" from\n";
     print "//    the rule parser state definitions file \"rbbirpt.txt\".\n";
     print "//\n";

-    print "//   Copyright (C) 2002 International Business Machines Corporation \n";
+    print "//   Copyright (C) 2002-$year International Business Machines Corporation \n";

     print "//   and others. All rights reserved.  \n";
     print "//\n";
     print "//---------------------------------------------------------------------------------\n";
     print "//   and others. All rights reserved.  \n";
     print "//\n";
     print "//---------------------------------------------------------------------------------\n";


@@ -305,25 +358,10 @@ else
     print "//\n";
     print "// Character classes for RBBI rule scanning.\n";
     print "//\n";
     print "//\n";
     print "// Character classes for RBBI rule scanning.\n";
     print "//\n";

-    $i = 128;                   # State Table values for Unicode char sets range from 128-250.
-                                # Sets "default", "escaped", etc. get special handling.
-                                #  They have no corresponding UnicodeSet object in the state machine,
-                                #    but are handled by special case code.  So we emit no reference
-                                #    to a UnicodeSet object to them here.
-    foreach $setName (keys %charClasses) {
-        if ($setName eq "default") {
-            $charClasses{$setName} = 255;}
-        elsif ($setName eq "escaped") {
-            $charClasses{$setName} = 254;}
-        elsif ($setName eq "escapedP") {
-            $charClasses{$setName} = 253;}
-        elsif ($setName eq "eof") {
-            $charClasses{$setName} = 252;}
-        else {
-            # Normal character class.  Fill in array with a ptr to the corresponding UnicodeSet in the state machine.
-           print "    static const uint8_t kRuleSet_$setName = $i;\n";
-            $charClasses{$setName} = $i;
-            $i++;
+    foreach $setName (sort keys %charClasses) {
+        if ($charClasses{$setName} < 250) {
+           # Normal character class.
+           print "    static const uint8_t kRuleSet_$setName = $charClasses{$setName};\n";

         }
     }
     print "\n\n";
         }
     }
     print "\n\n";


@@ -332,7 +370,7 @@ else
     # Emit the enum for the actions to be performed.
     #
     print "enum RBBI_RuleParseAction {\n";
     # Emit the enum for the actions to be performed.
     #
     print "enum RBBI_RuleParseAction {\n";

-    foreach $act (keys %actions) {
+    foreach $act (sort keys %actions) {

         print "    $act,\n";
     }
     print "    rbbiLastAction};\n\n";
         print "    $act,\n";
     }
     print "    rbbiLastAction};\n\n";


@@ -395,6 +433,7 @@ else
     #
     #    This array is used for producing debugging output from the rule parser.
     #
     #
     #    This array is used for producing debugging output from the rule parser.
     #

+    print "#ifdef RBBI_DEBUG\n";

     print "static const char * const RBBIRuleStateNames[] = {";
     for ($state=0; $state<$num_states; $state++) {
         if ($stateNames[$state] ne "") {
     print "static const char * const RBBIRuleStateNames[] = {";
     for ($state=0; $state<$num_states; $state++) {
         if ($stateNames[$state] ne "") {


@@ -403,7 +442,8 @@ else
             print "    0,\n";
         }
     }
             print "    0,\n";
         }
     }

-    print "    0};\n\n";
+    print "    0};\n";
+    print "#endif\n\n";

 
     print "U_NAMESPACE_END\n";
     print "#endif\n";
 
     print "U_NAMESPACE_END\n";
     print "#endif\n";