]> git.saurik.com Git - apple/icu.git/blob - icuSources/common/rbbicst.pl
ICU-6.2.8.tar.gz
[apple/icu.git] / icuSources / common / rbbicst.pl
1 #**************************************************************************
2 # Copyright (C) 2002-2004 International Business Machines Corporation *
3 # and others. All rights reserved. *
4 #**************************************************************************
5 #
6 # rbbicst Compile the RBBI rule paser state table data into initialized C data.
7 # Usage:
8 # cd icu/source/common
9 # perl rbbicst.pl [-j] < rbbirpt.txt > rbbirpt.h
10 #
11 # The output file, rbbrpt.h, is included by some of the .cpp rbbi
12 # implementation files. This perl script is NOT run as part
13 # of a normal ICU build. It is run by hand when needed, and the
14 # rbbirpt.h generated file is put back into cvs.
15 #
16 # See rbbirpt.h for a description of the input format for this script.
17 #
18
19 if ($ARGV[0] eq "-j") {
20 $javaOutput = 1;
21 shift @ARGV;
22 }
23
24
25 $num_states = 1; # Always the state number for the line being compiled.
26 $line_num = 0; # The line number in the input file.
27
28 $states{"pop"} = 255; # Add the "pop" to the list of defined state names.
29 # This prevents any state from being labelled with "pop",
30 # and resolves references to "pop" in the next state field.
31
32 line_loop: while (<>) {
33 chomp();
34 $line = $_;
35 @fields = split();
36 $line_num++;
37
38 # Remove # comments, which are any fields beginning with a #, plus all
39 # that follow on the line.
40 for ($i=0; $i<@fields; $i++) {
41 if ($fields[$i] =~ /^#/) {
42 @fields = @fields[0 .. $i-1];
43 last;
44 }
45 }
46 # ignore blank lines, and those with no fields left after stripping comments..
47 if (@fields == 0) {
48 next;
49 }
50
51 #
52 # State Label: handling.
53 # Does the first token end with a ":"? If so, it's the name of a state.
54 # Put in a hash, together with the current state number,
55 # so that we can later look up the number from the name.
56 #
57 if (@fields[0] =~ /.*:$/) {
58 $state_name = @fields[0];
59 $state_name =~ s/://; # strip off the colon from the state name.
60
61 if ($states{$state_name} != 0) {
62 print " rbbicst: at line $line-num duplicate definition of state $state_name\n";
63 }
64 $states{$state_name} = $num_states;
65 $stateNames[$num_states] = $state_name;
66
67 # if the label was the only thing on this line, go on to the next line,
68 # otherwise assume that a state definition is on the same line and fall through.
69 if (@fields == 1) {
70 next line_loop;
71 }
72 shift @fields; # shift off label field in preparation
73 # for handling the rest of the line.
74 }
75
76 #
77 # State Transition line.
78 # syntax is this,
79 # character [n] target-state [^push-state] [function-name]
80 # where
81 # [something] is an optional something
82 # character is either a single quoted character e.g. '['
83 # or a name of a character class, e.g. white_space
84 #
85
86 $state_line_num[$num_states] = $line_num; # remember line number with each state
87 # so we can make better error messages later.
88 #
89 # First field, character class or literal character for this transition.
90 #
91 if ($fields[0] =~ /^'.'$/) {
92 # We've got a quoted literal character.
93 $state_literal_chars[$num_states] = $fields[0];
94 $state_literal_chars[$num_states] =~ s/'//g;
95 } else {
96 # We've got the name of a character class.
97 $state_char_class[$num_states] = $fields[0];
98 if ($fields[0] =~ /[\W]/) {
99 print " rbbicsts: at line $line_num, bad character literal or character class name.\n";
100 print " scanning $fields[0]\n";
101 exit(-1);
102 }
103 }
104 shift @fields;
105
106 #
107 # do the 'n' flag
108 #
109 $state_flag[$num_states] = "FALSE";
110 if ($fields[0] eq "n") {
111 $state_flag[$num_states] = "TRUE";
112 shift @fields;
113 }
114
115 #
116 # do the destination state.
117 #
118 $state_dest_state[$num_states] = $fields[0];
119 if ($fields[0] eq "") {
120 print " rbbicsts: at line $line_num, destination state missing.\n";
121 exit(-1);
122 }
123 shift @fields;
124
125 #
126 # do the push state, if present.
127 #
128 if ($fields[0] =~ /^\^/) {
129 $fields[0] =~ s/^\^//;
130 $state_push_state[$num_states] = $fields[0];
131 if ($fields[0] eq "" ) {
132 print " rbbicsts: at line $line_num, expected state after ^ (no spaces).\n";
133 exit(-1);
134 }
135 shift @fields;
136 }
137
138 #
139 # Lastly, do the optional action name.
140 #
141 if ($fields[0] ne "") {
142 $state_func_name[$num_states] = $fields[0];
143 shift @fields;
144 }
145
146 #
147 # There should be no fields left on the line at this point.
148 #
149 if (@fields > 0) {
150 print " rbbicsts: at line $line_num, unexpected extra stuff on input line.\n";
151 print " scanning $fields[0]\n";
152 }
153 $num_states++;
154 }
155
156 #
157 # We've read in the whole file, now go back and output the
158 # C source code for the state transition table.
159 #
160 # We read all states first, before writing anything, so that the state numbers
161 # for the destination states are all available to be written.
162 #
163
164 #
165 # Make hashes for the names of the character classes and
166 # for the names of the actions that appeared.
167 #
168 for ($state=1; $state < $num_states; $state++) {
169 if ($state_char_class[$state] ne "") {
170 if ($charClasses{$state_char_class[$state]} == 0) {
171 $charClasses{$state_char_class[$state]} = 1;
172 }
173 }
174 if ($state_func_name[$state] eq "") {
175 $state_func_name[$state] = "doNOP";
176 }
177 if ($actions{$state_action_name[$state]} == 0) {
178 $actions{$state_func_name[$state]} = 1;
179 }
180 }
181
182 #
183 # Check that all of the destination states have been defined
184 #
185 #
186 $states{"exit"} = 0; # Predefined state name, terminates state machine.
187 for ($state=1; $state<$num_states; $state++) {
188 if ($states{$state_dest_state[$state]} == 0 && $state_dest_state[$state] ne "exit") {
189 print "Error at line $state_line_num[$state]: target state \"$state_dest_state[$state]\" is not defined.\n";
190 $errors++;
191 }
192 if ($state_push_state[$state] ne "" && $states{$state_push_state[$state]} == 0) {
193 print "Error at line $state_line_num[$state]: target state \"$state_push_state[$state]\" is not defined.\n";
194 $errors++;
195 }
196 }
197
198 die if ($errors>0);
199
200 my ($sec, $min, $hour, , $day, $mon, $year, $wday, $yday, $isdst) = localtime;
201 $year += 1900;
202
203 if ($javaOutput) {
204 print "/*\n";
205 print " *******************************************************************************\n";
206 print " * Copyright (C) 2003-$year,\n";
207 print " * International Business Machines Corporation and others. All Rights Reserved.\n";
208 print " *******************************************************************************\n";
209 print " */\n";
210 print " \n";
211 print "package com.ibm.icu.text;\n";
212 print " \n";
213 print "/**\n";
214 print " * Generated Java File. Do not edit by hand.\n";
215 print " * This file contains the state table for the ICU Rule Based Break Iterator\n";
216 print " * rule parser.\n";
217 print " * It is generated by the Perl script \"rbbicst.pl\" from\n";
218 print " * the rule parser state definitions file \"rbbirpt.txt\".\n";
219 print " *\n";
220 print " */\n";
221
222 print "public class RuleBasedBreakIteratorStateTable\n";
223 print "{\n";
224
225 #
226 # Emit the constants for the actions to be performed.
227 #
228 $n = 1;
229 foreach $act (keys %actions) {
230 print " public static final int $act = $n;\n";
231 $n++;
232 }
233 print " \n";
234 #
235 # emit the state transition table
236 #
237 print " public static final String[] gRuleParseStateTable = {\n";
238 printf(" \"\\u%04.4x\\u%04.4x\\u%04.4x\\u%04.4x\\u%04.4x\"\n", doNOP, 0, 0, 0, 1);
239 for ($state=1; $state < $num_states; $state++) {
240 printf(" , \"\\u%04.4x", $state_func_name[$state]);
241 # print " , {$state_func_name[$state],";
242 if ($state_literal_chars[$state] ne "") {
243 printf("\\u%04.4x", $state_func_name[$state]);
244 }else {
245 printf("\\u%04.4x", $charClasses{$state_char_class[$state]});
246 }
247 printf("\\u%04.4x", $states{$state_dest_state[$state]});
248
249 # The push-state field is optional. If omitted, fill field with a zero, which flags
250 # the state machine that there is no push state.
251 if ($state_push_state[$state] eq "") {
252 print "\\u0000";
253 } else {
254 printf("\\u%04.4x", $states{$state_push_state[$state]});
255 }
256 printf("\\u%04.4x", $state_flag[$state]);
257
258 # For the first row of each state, append the state name.
259 # Used for debugging only.
260 if ($stateNames[$state] ne "") {
261 printf("%-20s", $stateNames[$state]."\"");
262 } else {
263 printf("%-20s", "\"");
264 }
265
266 # Put out a C++ comment showing the number (index) of this state row,
267 print " // $state ";
268 print "\n";
269 };
270 print " };\n";
271 print "}\n";
272 }
273 else
274 {
275 #
276 # C++ Output ...
277 #
278
279
280 print "//---------------------------------------------------------------------------------\n";
281 print "//\n";
282 print "// Generated Header File. Do not edit by hand.\n";
283 print "// This file contains the state table for the ICU Rule Based Break Iterator\n";
284 print "// rule parser.\n";
285 print "// It is generated by the Perl script \"rbbicst.pl\" from\n";
286 print "// the rule parser state definitions file \"rbbirpt.txt\".\n";
287 print "//\n";
288 print "// Copyright (C) 2002-$year International Business Machines Corporation \n";
289 print "// and others. All rights reserved. \n";
290 print "//\n";
291 print "//---------------------------------------------------------------------------------\n";
292 print "#ifndef RBBIRPT_H\n";
293 print "#define RBBIRPT_H\n";
294 print "\n";
295 print "U_NAMESPACE_BEGIN\n";
296
297 #
298 # Emit the constants for indicies of Unicode Sets
299 # Define one constant for each of the character classes encountered.
300 # At the same time, store the index corresponding to the set name back into hash.
301 #
302 print "//\n";
303 print "// Character classes for RBBI rule scanning.\n";
304 print "//\n";
305 $i = 128; # State Table values for Unicode char sets range from 128-250.
306 # Sets "default", "escaped", etc. get special handling.
307 # They have no corresponding UnicodeSet object in the state machine,
308 # but are handled by special case code. So we emit no reference
309 # to a UnicodeSet object to them here.
310 foreach $setName (keys %charClasses) {
311 if ($setName eq "default") {
312 $charClasses{$setName} = 255;}
313 elsif ($setName eq "escaped") {
314 $charClasses{$setName} = 254;}
315 elsif ($setName eq "escapedP") {
316 $charClasses{$setName} = 253;}
317 elsif ($setName eq "eof") {
318 $charClasses{$setName} = 252;}
319 else {
320 # Normal character class. Fill in array with a ptr to the corresponding UnicodeSet in the state machine.
321 print " static const uint8_t kRuleSet_$setName = $i;\n";
322 $charClasses{$setName} = $i;
323 $i++;
324 }
325 }
326 print "\n\n";
327
328 #
329 # Emit the enum for the actions to be performed.
330 #
331 print "enum RBBI_RuleParseAction {\n";
332 foreach $act (keys %actions) {
333 print " $act,\n";
334 }
335 print " rbbiLastAction};\n\n";
336
337 #
338 # Emit the struct definition for transtion table elements.
339 #
340 print "//-------------------------------------------------------------------------------\n";
341 print "//\n";
342 print "// RBBIRuleTableEl represents the structure of a row in the transition table\n";
343 print "// for the rule parser state machine.\n";
344 print "//-------------------------------------------------------------------------------\n";
345 print "struct RBBIRuleTableEl {\n";
346 print " RBBI_RuleParseAction fAction;\n";
347 print " uint8_t fCharClass; // 0-127: an individual ASCII character\n";
348 print " // 128-255: character class index\n";
349 print " uint8_t fNextState; // 0-250: normal next-stat numbers\n";
350 print " // 255: pop next-state from stack.\n";
351 print " uint8_t fPushState;\n";
352 print " UBool fNextChar;\n";
353 print "};\n\n";
354
355 #
356 # emit the state transition table
357 #
358 print "static const struct RBBIRuleTableEl gRuleParseStateTable[] = {\n";
359 print " {doNOP, 0, 0, 0, TRUE}\n"; # State 0 is a dummy. Real states start with index = 1.
360 for ($state=1; $state < $num_states; $state++) {
361 print " , {$state_func_name[$state],";
362 if ($state_literal_chars[$state] ne "") {
363 $c = $state_literal_chars[$state];
364 printf(" %d /* $c */,", ord($c)); # use numeric value, so EBCDIC machines are ok.
365 }else {
366 print " $charClasses{$state_char_class[$state]},";
367 }
368 print " $states{$state_dest_state[$state]},";
369
370 # The push-state field is optional. If omitted, fill field with a zero, which flags
371 # the state machine that there is no push state.
372 if ($state_push_state[$state] eq "") {
373 print "0, ";
374 } else {
375 print " $states{$state_push_state[$state]},";
376 }
377 print " $state_flag[$state]} ";
378
379 # Put out a C++ comment showing the number (index) of this state row,
380 # and, if this is the first row of the table for this state, the state name.
381 print " // $state ";
382 if ($stateNames[$state] ne "") {
383 print " $stateNames[$state]";
384 }
385 print "\n";
386 };
387 print " };\n";
388
389
390 #
391 # emit a mapping array from state numbers to state names.
392 #
393 # This array is used for producing debugging output from the rule parser.
394 #
395 print "#ifdef RBBI_DEBUG\n";
396 print "static const char * const RBBIRuleStateNames[] = {";
397 for ($state=0; $state<$num_states; $state++) {
398 if ($stateNames[$state] ne "") {
399 print " \"$stateNames[$state]\",\n";
400 } else {
401 print " 0,\n";
402 }
403 }
404 print " 0};\n";
405 print "#endif\n\n";
406
407 print "U_NAMESPACE_END\n";
408 print "#endif\n";
409 }
410
411
412