icuSources/common/rbbicst.pl

   1 #**************************************************************************
   2 #   Copyright (C) 2002-2004 International Business Machines Corporation   *
   3 #   and others. All rights reserved.                                      *
   4 #**************************************************************************
   5 #
   6 #  rbbicst   Compile the RBBI rule paser state table data into initialized C data.
   7 #            Usage:
   8 #                   cd icu/source/common
   9 #                   perl rbbicst.pl [-j] < rbbirpt.txt > rbbirpt.h
  10 #
  11 #             The output file, rbbrpt.h, is included by some of the .cpp rbbi
  12 #             implementation files.   This perl script is NOT run as part
  13 #             of a normal ICU build.  It is run by hand when needed, and the
  14 #             rbbirpt.h generated file is put back into cvs.
  15 #
  16 #             See rbbirpt.h for a description of the input format for this script.
  17 #
  18
  19 if ($ARGV[0] eq "-j") {
  20     $javaOutput = 1;
  21     shift @ARGV;
  22 }
  23
  24
  25 $num_states = 1;     # Always the state number for the line being compiled.
  26 $line_num  = 0;      # The line number in the input file.
  27
  28 $states{"pop"} = 255;    # Add the "pop"  to the list of defined state names.
  29                          # This prevents any state from being labelled with "pop",
  30                          #  and resolves references to "pop" in the next state field.
  31
  32 line_loop: while (<>) {
  33     chomp();
  34     $line = $_;
  35     @fields = split();
  36     $line_num++;
  37
  38     # Remove # comments, which are any fields beginning with a #, plus all
  39     #  that follow on the line.
  40     for ($i=0; $i<@fields; $i++) {
  41         if ($fields[$i] =~ /^#/) {
  42             @fields = @fields[0 .. $i-1];
  43             last;
  44         }
  45     }
  46     # ignore blank lines, and those with no fields left after stripping comments..
  47     if (@fields == 0) {
  48         next;
  49     }
  50
  51     #
  52     # State Label:  handling.
  53     #    Does the first token end with a ":"?  If so, it's the name  of a state.
  54     #    Put in a hash, together with the current state number,
  55     #        so that we can later look up the number from the name.
  56     #
  57     if (@fields[0] =~ /.*:$/) {
  58         $state_name = @fields[0];
  59         $state_name =~ s/://;        # strip off the colon from the state name.
  60
  61         if ($states{$state_name} != 0) {
  62             print "  rbbicst: at line $line-num duplicate definition of state $state_name\n";
  63         }
  64         $states{$state_name} = $num_states;
  65         $stateNames[$num_states] = $state_name;
  66
  67         # if the label was the only thing on this line, go on to the next line,
  68         # otherwise assume that a state definition is on the same line and fall through.
  69         if (@fields == 1) {
  70             next line_loop;
  71         }
  72         shift @fields;                       # shift off label field in preparation
  73                                              #  for handling the rest of the line.
  74     }
  75
  76     #
  77     # State Transition line.
  78     #   syntax is this,
  79     #       character   [n]  target-state  [^push-state]  [function-name]
  80     #   where
  81     #      [something]   is an optional something
  82     #      character     is either a single quoted character e.g. '['
  83     #                       or a name of a character class, e.g. white_space
  84     #
  85
  86     $state_line_num[$num_states] = $line_num;   # remember line number with each state
  87                                                 #  so we can make better error messages later.
  88     #
  89     # First field, character class or literal character for this transition.
  90     #
  91     if ($fields[0] =~ /^'.'$/) {
  92         # We've got a quoted literal character.
  93         $state_literal_chars[$num_states] = $fields[0];
  94         $state_literal_chars[$num_states] =~ s/'//g;
  95     } else {
  96         # We've got the name of a character class.
  97         $state_char_class[$num_states] = $fields[0];
  98         if ($fields[0] =~ /[\W]/) {
  99             print "  rbbicsts:  at line $line_num, bad character literal or character class name.\n";
 100             print "     scanning $fields[0]\n";
 101             exit(-1);
 102         }
 103     }
 104     shift @fields;
 105
 106     #
 107     # do the 'n' flag
 108     #
 109     $state_flag[$num_states] = "FALSE";
 110     if ($fields[0] eq "n") {
 111         $state_flag[$num_states] = "TRUE";
 112         shift @fields;
 113     }
 114
 115     #
 116     # do the destination state.
 117     #
 118     $state_dest_state[$num_states] = $fields[0];
 119     if ($fields[0] eq "") {
 120         print "  rbbicsts:  at line $line_num, destination state missing.\n";
 121         exit(-1);
 122     }
 123     shift @fields;
 124
 125     #
 126     # do the push state, if present.
 127     #
 128     if ($fields[0] =~ /^\^/) {
 129         $fields[0] =~ s/^\^//;
 130         $state_push_state[$num_states] = $fields[0];
 131         if ($fields[0] eq "" ) {
 132             print "  rbbicsts:  at line $line_num, expected state after ^ (no spaces).\n";
 133             exit(-1);
 134         }
 135         shift @fields;
 136     }
 137
 138     #
 139     # Lastly, do the optional action name.
 140     #
 141     if ($fields[0] ne "") {
 142         $state_func_name[$num_states] = $fields[0];
 143         shift @fields;
 144     }
 145
 146     #
 147     #  There should be no fields left on the line at this point.
 148     #
 149     if (@fields > 0) {
 150        print "  rbbicsts:  at line $line_num, unexpected extra stuff on input line.\n";
 151        print "     scanning $fields[0]\n";
 152    }
 153    $num_states++;
 154 }
 155
 156 #
 157 # We've read in the whole file, now go back and output the
 158 #   C source code for the state transition table.
 159 #
 160 # We read all states first, before writing anything,  so that the state numbers
 161 # for the destination states are all available to be written.
 162 #
 163
 164 #
 165 # Make hashes for the names of the character classes and
 166 #      for the names of the actions that appeared.
 167 #
 168 for ($state=1; $state < $num_states; $state++) {
 169     if ($state_char_class[$state] ne "") {
 170         if ($charClasses{$state_char_class[$state]} == 0) {
 171             $charClasses{$state_char_class[$state]} = 1;
 172         }
 173     }
 174     if ($state_func_name[$state] eq "") {
 175         $state_func_name[$state] = "doNOP";
 176     }
 177     if ($actions{$state_action_name[$state]} == 0) {
 178         $actions{$state_func_name[$state]} = 1;
 179     }
 180 }
 181
 182 #
 183 # Check that all of the destination states have been defined
 184 #
 185 #
 186 $states{"exit"} = 0;              # Predefined state name, terminates state machine.
 187 for ($state=1; $state<$num_states; $state++) {
 188    if ($states{$state_dest_state[$state]} == 0 && $state_dest_state[$state] ne "exit") {
 189        print "Error at line $state_line_num[$state]: target state \"$state_dest_state[$state]\" is not defined.\n";
 190        $errors++;
 191    }
 192    if ($state_push_state[$state] ne "" && $states{$state_push_state[$state]} == 0) {
 193        print "Error at line $state_line_num[$state]: target state \"$state_push_state[$state]\" is not defined.\n";
 194        $errors++;
 195    }
 196 }
 197
 198 die if ($errors>0);
 199
 200 my ($sec, $min, $hour, , $day, $mon, $year, $wday, $yday, $isdst) = localtime;
 201 $year += 1900;
 202
 203 if ($javaOutput) {
 204     print "/*\n";
 205     print " *******************************************************************************\n";
 206     print " * Copyright (C) 2003-$year,\n";
 207     print " * International Business Machines Corporation and others. All Rights Reserved.\n";
 208     print " *******************************************************************************\n";
 209     print " */\n";
 210     print " \n";
 211     print "package com.ibm.icu.text;\n";
 212     print " \n";
 213     print "/**\n";
 214     print " * Generated Java File.  Do not edit by hand.\n";
 215     print " * This file contains the state table for the ICU Rule Based Break Iterator\n";
 216     print " * rule parser.\n";
 217     print " * It is generated by the Perl script \"rbbicst.pl\" from\n";
 218     print " * the rule parser state definitions file \"rbbirpt.txt\".\n";
 219     print " *\n";
 220     print " */\n";
 221
 222     print "public class RuleBasedBreakIteratorStateTable\n";
 223     print "{\n";
 224
 225     #
 226     # Emit the constants for the actions to be performed.
 227     #
 228     $n = 1;
 229     foreach $act (keys %actions) {
 230         print "    public static final int $act = $n;\n";
 231         $n++;
 232     }
 233     print " \n";
 234     #
 235     # emit the state transition table
 236     #
 237     print "    public static final String[] gRuleParseStateTable = {\n";
 238     printf("        \"\\u%04.4x\\u%04.4x\\u%04.4x\\u%04.4x\\u%04.4x\"\n", doNOP, 0, 0, 0, 1);
 239     for ($state=1; $state < $num_states; $state++) {
 240         printf("        , \"\\u%04.4x", $state_func_name[$state]);
 241         # print "  , {$state_func_name[$state],";
 242         if ($state_literal_chars[$state] ne "") {
 243             printf("\\u%04.4x", $state_func_name[$state]);
 244         }else {
 245             printf("\\u%04.4x", $charClasses{$state_char_class[$state]});
 246         }
 247         printf("\\u%04.4x", $states{$state_dest_state[$state]});
 248
 249         # The push-state field is optional.  If omitted, fill field with a zero, which flags
 250         #   the state machine that there is no push state.
 251         if ($state_push_state[$state] eq "") {
 252             print "\\u0000";
 253         } else {
 254            printf("\\u%04.4x", $states{$state_push_state[$state]});
 255         }
 256         printf("\\u%04.4x", $state_flag[$state]);
 257
 258         # For the first row of each state, append the state name.
 259         #   Used for debugging only.
 260         if ($stateNames[$state] ne "") {
 261             printf("%-20s", $stateNames[$state]."\"");
 262         } else {
 263             printf("%-20s", "\"");
 264         }
 265
 266         # Put out a C++ comment showing the number (index) of this state row,
 267         print "        //  $state ";
 268         print "\n";
 269     };
 270     print "    };\n";
 271     print "}\n";
 272 }
 273 else
 274 {
 275     #
 276     #  C++ Output ...
 277     #
 278
 279
 280     print "//---------------------------------------------------------------------------------\n";
 281     print "//\n";
 282     print "// Generated Header File.  Do not edit by hand.\n";
 283     print "//    This file contains the state table for the ICU Rule Based Break Iterator\n";
 284     print "//    rule parser.\n";
 285     print "//    It is generated by the Perl script \"rbbicst.pl\" from\n";
 286     print "//    the rule parser state definitions file \"rbbirpt.txt\".\n";
 287     print "//\n";
 288     print "//   Copyright (C) 2002-$year International Business Machines Corporation \n";
 289     print "//   and others. All rights reserved.  \n";
 290     print "//\n";
 291     print "//---------------------------------------------------------------------------------\n";
 292     print "#ifndef RBBIRPT_H\n";
 293     print "#define RBBIRPT_H\n";
 294     print "\n";
 295     print "U_NAMESPACE_BEGIN\n";
 296
 297     #
 298     # Emit the constants for indicies of Unicode Sets
 299     #   Define one constant for each of the character classes encountered.
 300     #   At the same time, store the index corresponding to the set name back into hash.
 301     #
 302     print "//\n";
 303     print "// Character classes for RBBI rule scanning.\n";
 304     print "//\n";
 305     $i = 128;                   # State Table values for Unicode char sets range from 128-250.
 306                                 # Sets "default", "escaped", etc. get special handling.
 307                                 #  They have no corresponding UnicodeSet object in the state machine,
 308                                 #    but are handled by special case code.  So we emit no reference
 309                                 #    to a UnicodeSet object to them here.
 310     foreach $setName (keys %charClasses) {
 311         if ($setName eq "default") {
 312             $charClasses{$setName} = 255;}
 313         elsif ($setName eq "escaped") {
 314             $charClasses{$setName} = 254;}
 315         elsif ($setName eq "escapedP") {
 316             $charClasses{$setName} = 253;}
 317         elsif ($setName eq "eof") {
 318             $charClasses{$setName} = 252;}
 319         else {
 320             # Normal character class.  Fill in array with a ptr to the corresponding UnicodeSet in the state machine.
 321            print "    static const uint8_t kRuleSet_$setName = $i;\n";
 322             $charClasses{$setName} = $i;
 323             $i++;
 324         }
 325     }
 326     print "\n\n";
 327
 328     #
 329     # Emit the enum for the actions to be performed.
 330     #
 331     print "enum RBBI_RuleParseAction {\n";
 332     foreach $act (keys %actions) {
 333         print "    $act,\n";
 334     }
 335     print "    rbbiLastAction};\n\n";
 336
 337     #
 338     # Emit the struct definition for transtion table elements.
 339     #
 340     print "//-------------------------------------------------------------------------------\n";
 341     print "//\n";
 342     print "//  RBBIRuleTableEl    represents the structure of a row in the transition table\n";
 343     print "//                     for the rule parser state machine.\n";
 344     print "//-------------------------------------------------------------------------------\n";
 345     print "struct RBBIRuleTableEl {\n";
 346     print "    RBBI_RuleParseAction          fAction;\n";
 347     print "    uint8_t                       fCharClass;       // 0-127:    an individual ASCII character\n";
 348     print "                                                    // 128-255:  character class index\n";
 349     print "    uint8_t                       fNextState;       // 0-250:    normal next-stat numbers\n";
 350     print "                                                    // 255:      pop next-state from stack.\n";
 351     print "    uint8_t                       fPushState;\n";
 352     print "    UBool                         fNextChar;\n";
 353     print "};\n\n";
 354
 355     #
 356     # emit the state transition table
 357     #
 358     print "static const struct RBBIRuleTableEl gRuleParseStateTable[] = {\n";
 359     print "    {doNOP, 0, 0, 0, TRUE}\n";    # State 0 is a dummy.  Real states start with index = 1.
 360     for ($state=1; $state < $num_states; $state++) {
 361         print "    , {$state_func_name[$state],";
 362         if ($state_literal_chars[$state] ne "") {
 363             $c = $state_literal_chars[$state];
 364             printf(" %d /* $c */,", ord($c));   #  use numeric value, so EBCDIC machines are ok.
 365         }else {
 366             print " $charClasses{$state_char_class[$state]},";
 367         }
 368         print " $states{$state_dest_state[$state]},";
 369
 370         # The push-state field is optional.  If omitted, fill field with a zero, which flags
 371         #   the state machine that there is no push state.
 372         if ($state_push_state[$state] eq "") {
 373             print "0, ";
 374         } else {
 375             print " $states{$state_push_state[$state]},";
 376         }
 377         print " $state_flag[$state]} ";
 378
 379         # Put out a C++ comment showing the number (index) of this state row,
 380         #   and, if this is the first row of the table for this state, the state name.
 381         print "    //  $state ";
 382         if ($stateNames[$state] ne "") {
 383             print "     $stateNames[$state]";
 384         }
 385         print "\n";
 386     };
 387     print " };\n";
 388
 389
 390     #
 391     # emit a mapping array from state numbers to state names.
 392     #
 393     #    This array is used for producing debugging output from the rule parser.
 394     #
 395     print "#ifdef RBBI_DEBUG\n";
 396     print "static const char * const RBBIRuleStateNames[] = {";
 397     for ($state=0; $state<$num_states; $state++) {
 398         if ($stateNames[$state] ne "") {
 399             print "     \"$stateNames[$state]\",\n";
 400         } else {
 401             print "    0,\n";
 402         }
 403     }
 404     print "    0};\n";
 405     print "#endif\n\n";
 406
 407     print "U_NAMESPACE_END\n";
 408     print "#endif\n";
 409 }
 410
 411
 412