ICU-57149.0.1.tar.gz

[apple/icu.git] / icuSources / i18n / regexcst.txt

diff --git a/icuSources/i18n/regexcst.txt b/icuSources/i18n/regexcst.txt
index fec788c14172575d6e8c14d9268daca4e595ef15..1b88e44654021cc4f0ab541550bd343be26e6ef4 100644 (file)
--- a/icuSources/i18n/regexcst.txt
+++ b/icuSources/i18n/regexcst.txt
@@ -1,7 +1,7 @@
 
 #*****************************************************************************
 #
 
 #*****************************************************************************
 #

-#   Copyright (C) 2002-2003, International Business Machines Corporation and others.
+#   Copyright (C) 2002-2015, International Business Machines Corporation and others.

 #   All Rights Reserved.
 #
 #*****************************************************************************
 #   All Rights Reserved.
 #
 #*****************************************************************************


@@ -25,8 +25,8 @@
 #
 #
 #StateName:
 #
 #
 #StateName:

-#   input-char           n next-state           ^push-state     action    
-#   input-char           n next-state           ^push-state     action    
+#   input-char           n next-state           ^push-state     action
+#   input-char           n next-state           ^push-state     action

 #       |                |   |                      |             |
 #       |                |   |                      |             |--- action to be performed by state machine
 #       |                |   |                      |                  See function RBBIRuleScanner::doParseActions()
 #       |                |   |                      |             |
 #       |                |   |                      |             |--- action to be performed by state machine
 #       |                |   |                      |                  See function RBBIRuleScanner::doParseActions()


@@ -46,7 +46,7 @@
 #            matches, peform the actions and go to the state specified on this line.
 #            The input character is tested sequentally, in the order written.  The characters and
 #            character classes tested for do not need to be mutually exclusive.  The first match wins.
 #            matches, peform the actions and go to the state specified on this line.
 #            The input character is tested sequentally, in the order written.  The characters and
 #            character classes tested for do not need to be mutually exclusive.  The first match wins.

-#            
+#

 
 
 
 
 
 


@@ -56,27 +56,27 @@
 #
 start:
    default                 term                                     doPatStart
 #
 start:
    default                 term                                     doPatStart

-    

 
 

-    
-    
+
+
+

 #
 #  term.  At a position where we can accept the start most items in a pattern.
 #
 term:
     quoted               n expr-quant                               doLiteralChar
     rule_char            n expr-quant                               doLiteralChar
 #
 #  term.  At a position where we can accept the start most items in a pattern.
 #
 term:
     quoted               n expr-quant                               doLiteralChar
     rule_char            n expr-quant                               doLiteralChar

-    '['                  n expr-quant                               doScanUnicodeSet
-    '('                  n open-paren                     
+    '['                  n set-open       ^set-finish               doSetBegin
+    '('                  n open-paren

     '.'                  n expr-quant                               doDotAny
     '.'                  n expr-quant                               doDotAny

-    '^'                  n term                                     doCaret
-    '$'                  n term                                     doDollar
+    '^'                  n expr-quant                               doCaret
+    '$'                  n expr-quant                               doDollar

     '\'                  n backslash
     '|'                  n  term                                    doOrOperator
     ')'                  n  pop                                     doCloseParen
     eof                           term                                     doPatFinish
     default                errorDeath                               doRuleError
     '\'                  n backslash
     '|'                  n  term                                    doOrOperator
     ')'                  n  pop                                     doCloseParen
     eof                           term                                     doPatFinish
     default                errorDeath                               doRuleError

-    
+

 
 
 #
 
 
 #


@@ -84,14 +84,14 @@ term:
 #                 trailing quantifier - *, +, ?, *?,  etc.
 #
 expr-quant:
 #                 trailing quantifier - *, +, ?, *?,  etc.
 #
 expr-quant:

-    '*'                  n  quant-star                       
-    '+'                  n  quant-plus                              
-    '?'                  n  quant-opt     
+    '*'                  n  quant-star
+    '+'                  n  quant-plus
+    '?'                  n  quant-opt

     '{'                  n  interval-open                          doIntervalInit
     '('                  n  open-paren-quant
     '{'                  n  interval-open                          doIntervalInit
     '('                  n  open-paren-quant

-    default                 expr-cont 
-    
-    
+    default                 expr-cont
+
+

 #
 #  expr-cont      Expression, continuation.  At a point where additional terms are
 #                                            allowed, but not required.  No Quantifiers
 #
 #  expr-cont      Expression, continuation.  At a point where additional terms are
 #                                            allowed, but not required.  No Quantifiers


@@ -99,8 +99,8 @@ expr-quant:
 expr-cont:
     '|'                  n  term                                    doOrOperator
     ')'                  n  pop                                     doCloseParen
 expr-cont:
     '|'                  n  term                                    doOrOperator
     ')'                  n  pop                                     doCloseParen

-    default                 term                                    
-    
+    default                 term
+

 
 #
 #   open-paren-quant   Special case handling for comments appearing before a quantifier,
 
 #
 #   open-paren-quant   Special case handling for comments appearing before a quantifier,


@@ -111,12 +111,12 @@ expr-cont:
 open-paren-quant:
     '?'                  n  open-paren-quant2                      doSuppressComments
     default                 open-paren
 open-paren-quant:
     '?'                  n  open-paren-quant2                      doSuppressComments
     default                 open-paren

-    
+

 open-paren-quant2:
     '#'                  n  paren-comment   ^expr-quant
     default                 open-paren-extended
 open-paren-quant2:
     '#'                  n  paren-comment   ^expr-quant
     default                 open-paren-extended

-    
- 
+
+

 #
 #   open-paren    We've got an open paren.  We need to scan further to
 #                 determine what kind of quantifier it is - plain (, (?:, (?>, or whatever.
 #
 #   open-paren    We've got an open paren.  We need to scan further to
 #                 determine what kind of quantifier it is - plain (, (?:, (?>, or whatever.


@@ -124,7 +124,7 @@ open-paren-quant2:
 open-paren:
     '?'                  n  open-paren-extended                     doSuppressComments
     default                 term            ^expr-quant             doOpenCaptureParen
 open-paren:
     '?'                  n  open-paren-extended                     doSuppressComments
     default                 term            ^expr-quant             doOpenCaptureParen

-    
+

 open-paren-extended:
     ':'                  n  term            ^expr-quant             doOpenNonCaptureParen  #  (?:
     '>'                  n  term            ^expr-quant             doOpenAtomicParen      #  (?>
 open-paren-extended:
     ':'                  n  term            ^expr-quant             doOpenNonCaptureParen  #  (?:
     '>'                  n  term            ^expr-quant             doOpenAtomicParen      #  (?>


@@ -133,24 +133,26 @@ open-paren-extended:
     '<'                  n  open-paren-lookbehind
     '#'                  n  paren-comment   ^term
     'i'                     paren-flag                              doBeginMatchMode
     '<'                  n  open-paren-lookbehind
     '#'                  n  paren-comment   ^term
     'i'                     paren-flag                              doBeginMatchMode

+    'd'                     paren-flag                              doBeginMatchMode

     'm'                     paren-flag                              doBeginMatchMode
     's'                     paren-flag                              doBeginMatchMode
     'm'                     paren-flag                              doBeginMatchMode
     's'                     paren-flag                              doBeginMatchMode

+    'u'                     paren-flag                              doBeginMatchMode

     'w'                     paren-flag                              doBeginMatchMode
     'x'                     paren-flag                              doBeginMatchMode
     '-'                     paren-flag                              doBeginMatchMode
     '('                  n  errorDeath                              doConditionalExpr
     '{'                  n  errorDeath                              doPerlInline
     default                 errorDeath                              doBadOpenParenType
     'w'                     paren-flag                              doBeginMatchMode
     'x'                     paren-flag                              doBeginMatchMode
     '-'                     paren-flag                              doBeginMatchMode
     '('                  n  errorDeath                              doConditionalExpr
     '{'                  n  errorDeath                              doPerlInline
     default                 errorDeath                              doBadOpenParenType

-    
+

 open-paren-lookbehind:
     '='                  n  term            ^expr-cont              doOpenLookBehind       #  (?<=
     '!'                  n  term            ^expr-cont              doOpenLookBehindNeg    #  (?<!
 open-paren-lookbehind:
     '='                  n  term            ^expr-cont              doOpenLookBehind       #  (?<=
     '!'                  n  term            ^expr-cont              doOpenLookBehindNeg    #  (?<!

+    ascii_letter            named-capture                           doBeginNamedCapture    #  (?<name

     default                 errorDeath                              doBadOpenParenType
     default                 errorDeath                              doBadOpenParenType

-    
+

 
 #
 #   paren-comment    We've got a (?# ... )  style comment.  Eat pattern text till we get to the ')'
 
 #
 #   paren-comment    We've got a (?# ... )  style comment.  Eat pattern text till we get to the ')'

-#                    TODO:  should parens nest here?  Check what perl does.

 #
 paren-comment:
     ')'                  n  pop
 #
 paren-comment:
     ')'                  n  pop


@@ -158,20 +160,30 @@ paren-comment:
     default              n  paren-comment
 
 #
     default              n  paren-comment
 
 #

-#  paren-flag    Scanned a (?ismx-ismx  flag setting 
-#                 
+#  paren-flag    Scanned a (?ismx-ismx  flag setting
+#

 paren-flag:
     'i'                  n  paren-flag                              doMatchMode
 paren-flag:
     'i'                  n  paren-flag                              doMatchMode

+    'd'                  n  paren-flag                              doMatchMode

     'm'                  n  paren-flag                              doMatchMode
     's'                  n  paren-flag                              doMatchMode
     'm'                  n  paren-flag                              doMatchMode
     's'                  n  paren-flag                              doMatchMode

+    'u'                  n  paren-flag                              doMatchMode

     'w'                  n  paren-flag                              doMatchMode
     'x'                  n  paren-flag                              doMatchMode
     '-'                  n  paren-flag                              doMatchMode
     ')'                  n  term                                    doSetMatchMode
     ':'                  n  term              ^expr-quant           doMatchModeParen
     default                 errorDeath                              doBadModeFlag
     'w'                  n  paren-flag                              doMatchMode
     'x'                  n  paren-flag                              doMatchMode
     '-'                  n  paren-flag                              doMatchMode
     ')'                  n  term                                    doSetMatchMode
     ':'                  n  term              ^expr-quant           doMatchModeParen
     default                 errorDeath                              doBadModeFlag

-    
-    
+
+#
+#  named-capture    (?<name> ... ), position currently on the name.
+#
+named-capture:
+    ascii_letter         n  named-capture                           doContinueNamedCapture
+    digit_char           n  named-capture                           doContinueNamedCapture
+    '>'                  n  term               ^expr-quant          doOpenCaptureParen      # common w non-named capture.
+    default                 errorDeath                              doBadNamedCapture
+

 #
 #  quant-star     Scanning a '*' quantifier.  Need to look ahead to decide
 #                 between plain '*', '*?', '*+'
 #
 #  quant-star     Scanning a '*' quantifier.  Need to look ahead to decide
 #                 between plain '*', '*?', '*+'


@@ -204,13 +216,12 @@ quant-opt:
 
 #
 #   Interval         scanning a '{', the opening delimiter for an interval specification
 
 #
 #   Interval         scanning a '{', the opening delimiter for an interval specification

-#                                   {number} or {min, max} or {min, }
+#                                   {number} or {min, max} or {min,}

 #
 interval-open:
 #
 interval-open:

-    white_space          n  interval-open                                  # TODO:  is white space allowed here in non-free mode?
-    digit_char              interval-lower                          
+    digit_char              interval-lower

     default                 errorDeath                              doIntervalError
     default                 errorDeath                              doIntervalError

-    
+

 interval-lower:
     digit_char           n  interval-lower                          doIntevalLowerDigit
     ','                                 n  interval-upper
 interval-lower:
     digit_char           n  interval-lower                          doIntevalLowerDigit
     ','                                 n  interval-upper


@@ -221,13 +232,13 @@ interval-upper:
     digit_char           n  interval-upper                          doIntervalUpperDigit
     '}'                  n  interval-type
     default                 errorDeath                              doIntervalError
     digit_char           n  interval-upper                          doIntervalUpperDigit
     '}'                  n  interval-type
     default                 errorDeath                              doIntervalError

-    
+

 interval-type:
     '?'                  n  expr-cont                               doNGInterval                # {n,m}?
     '+'                  n  expr-cont                               doPossessiveInterval        # {n,m}+
     default                 expr-cont                               doInterval                  # {m,n}
 interval-type:
     '?'                  n  expr-cont                               doNGInterval                # {n,m}?
     '+'                  n  expr-cont                               doPossessiveInterval        # {n,m}+
     default                 expr-cont                               doInterval                  # {m,n}

-    
-    
+
+

 #
 #  backslash        #  Backslash.  Figure out which of the \thingies we have encountered.
 #                                  The low level next-char function will have preprocessed
 #
 #  backslash        #  Backslash.  Figure out which of the \thingies we have encountered.
 #                                  The low level next-char function will have preprocessed


@@ -239,22 +250,249 @@ backslash:
    'd'                   n  expr-quant                              doBackslashd
    'D'                   n  expr-quant                              doBackslashD
    'G'                   n  term                                    doBackslashG
    'd'                   n  expr-quant                              doBackslashd
    'D'                   n  expr-quant                              doBackslashD
    'G'                   n  term                                    doBackslashG

-   'N'                      expr-quant                              doProperty       #   \N{NAME}  named char
+   'h'                   n  expr-quant                              doBackslashh
+   'H'                   n  expr-quant                              doBackslashH
+   'k'                   n  named-backref
+   'N'                      expr-quant                              doNamedChar      #   \N{NAME}  named char

    'p'                      expr-quant                              doProperty       #   \p{Lu}  style property
    'P'                      expr-quant                              doProperty
    'p'                      expr-quant                              doProperty       #   \p{Lu}  style property
    'P'                      expr-quant                              doProperty

+   'R'                   n  expr-quant                              doBackslashR

    'Q'                   n  term                                    doEnterQuoteMode
    'S'                   n  expr-quant                              doBackslashS
    's'                   n  expr-quant                              doBackslashs
    'Q'                   n  term                                    doEnterQuoteMode
    'S'                   n  expr-quant                              doBackslashS
    's'                   n  expr-quant                              doBackslashs

+   'v'                   n  expr-quant                              doBackslashv
+   'V'                   n  expr-quant                              doBackslashV

    'W'                   n  expr-quant                              doBackslashW
    'w'                   n  expr-quant                              doBackslashw
    'X'                   n  expr-quant                              doBackslashX
    'Z'                   n  term                                    doBackslashZ
    'z'                   n  term                                    doBackslashz
    'W'                   n  expr-quant                              doBackslashW
    'w'                   n  expr-quant                              doBackslashw
    'X'                   n  expr-quant                              doBackslashX
    'Z'                   n  term                                    doBackslashZ
    'z'                   n  term                                    doBackslashz

-   digit_char           n  expr-quant                              doBackRef         #  Will scan multiple digits
+   digit_char            n  expr-quant                              doBackRef         #  Will scan multiple digits

    eof                      errorDeath                              doEscapeError
    eof                      errorDeath                              doEscapeError

-   default               n  expr-quant                             doLiteralChar     #  Escaped literal char.                 
+   default               n  expr-quant                              doEscapedLiteralChar
+
+
+# named-backref   Scanned \k
+#                 Leading to \k<captureName>
+#                 Failure to get the full sequence is an error.
+#
+named-backref:
+    '<'                  n  named-backref-2                         doBeginNamedBackRef
+    default                 errorDeath                              doBadNamedCapture
+
+named-backref-2:
+    ascii_letter         n  named-backref-3                         doContinueNamedBackRef
+    default                 errorDeath                              doBadNamedCapture
+
+named-backref-3:
+    ascii_letter         n  named-backref-3                         doContinueNamedBackRef
+    digit_char           n  named-backref-3                         doContinueNamedBackRef
+    '>'                  n  expr-quant                              doCompleteNamedBackRef
+    default                 errorDeath                              doBadNamedCapture
+
+
+#
+# [set expression] parsing,
+#    All states involved in parsing set expressions have names beginning with "set-"
+#
+
+set-open:
+   '^'                   n  set-open2                               doSetNegate
+   ':'                      set-posix                               doSetPosixProp
+   default                  set-open2
+
+set-open2:
+   ']'                   n  set-after-lit                           doSetLiteral
+   default                  set-start
+
+#  set-posix:
+#                  scanned a '[:'  If it really is a [:property:], doSetPosixProp will have
+#                  moved the scan to the closing ']'.  If it wasn't a property
+#                  expression, the scan will still be at the opening ':', which should
+#                  be interpreted as a normal set expression.
+set-posix:
+    ']'                  n   pop                                    doSetEnd
+    ':'                      set-start
+    default                  errorDeath                             doRuleError  # should not be possible.
+
+#
+#   set-start   after the [ and special case leading characters (^ and/or ]) but before
+#               everything else.   A '-' is literal at this point.
+#
+set-start:
+    ']'                  n  pop                                     doSetEnd
+    '['                  n  set-open      ^set-after-set            doSetBeginUnion
+    '\'                  n  set-escape
+    '-'                  n  set-start-dash
+    '&'                  n  set-start-amp
+    default              n  set-after-lit                           doSetLiteral
+
+#    set-start-dash    Turn "[--" into a syntax error.
+#                           "[-x" is good, - and x are literals.
+#
+set-start-dash:
+    '-'                     errorDeath                              doRuleError
+    default                 set-after-lit                           doSetAddDash
+
+#    set-start-amp     Turn "[&&" into a syntax error.
+#                           "[&x" is good, & and x are literals.
+#
+set-start-amp:
+    '&'                     errorDeath                              doRuleError
+    default                 set-after-lit                           doSetAddAmp

 
 

+#
+#   set-after-lit    The last thing scanned was a literal character within a set.
+#                    Can be followed by anything.  Single '-' or '&' are
+#                    literals in this context, not operators.
+set-after-lit:
+    ']'                  n  pop                                     doSetEnd
+    '['                  n  set-open      ^set-after-set            doSetBeginUnion
+    '-'                  n  set-lit-dash
+    '&'                  n  set-lit-amp
+    '\'                  n  set-escape
+    eof                     errorDeath                              doSetNoCloseError
+    default              n  set-after-lit                           doSetLiteral
+
+set-after-set:
+    ']'                  n  pop                                     doSetEnd
+    '['                  n  set-open      ^set-after-set            doSetBeginUnion
+    '-'                  n  set-set-dash
+    '&'                  n  set-set-amp
+    '\'                  n  set-escape
+    eof                     errorDeath                              doSetNoCloseError
+    default              n  set-after-lit                           doSetLiteral
+
+set-after-range:
+    ']'                  n  pop                                     doSetEnd
+    '['                  n  set-open      ^set-after-set            doSetBeginUnion
+    '-'                  n  set-range-dash
+    '&'                  n  set-range-amp
+    '\'                  n  set-escape
+    eof                     errorDeath                              doSetNoCloseError
+    default              n  set-after-lit                           doSetLiteral

     
     

+
+# set-after-op
+#     After a --  or &&
+#     It is an error to close a set at this point.
+#
+set-after-op:
+    '['                  n  set-open         ^set-after-set         doSetBeginUnion
+    ']'                     errorDeath                              doSetOpError
+    '\'                  n  set-escape
+    default              n  set-after-lit                           doSetLiteral
+
+#
+#   set-set-amp
+#      Have scanned [[set]&
+#      Could be a '&' intersection operator, if a set follows.
+#      Could be the start of a '&&' operator.
+#      Otherewise is a literal.
+set-set-amp:
+    '['                  n  set-open      ^set-after-set           doSetBeginIntersection1
+    '&'                  n  set-after-op                           doSetIntersection2
+    default                 set-after-lit                          doSetAddAmp
+
+
+# set-lit-amp   Have scanned "[literals&"
+#               Could be a start of "&&" operator or a literal
+#               In [abc&[def]],   the '&' is a literal
+#
+set-lit-amp:
+    '&'                  n  set-after-op                            doSetIntersection2
+    default                 set-after-lit                           doSetAddAmp
+
+
+#
+#  set-set-dash
+#      Have scanned [set]-
+#      Could be a '-' difference operator, if a [set] follows.
+#      Could be the start of a '--' operator.
+#      Otherewise is a literal.
+set-set-dash:
+    '['                  n  set-open      ^set-after-set           doSetBeginDifference1
+    '-'                  n  set-after-op                           doSetDifference2
+    default                 set-after-lit                          doSetAddDash
+
+
+#
+#  set-range-dash
+#      scanned  a-b-  or \w-
+#         any set or range like item where the trailing single '-' should
+#         be literal, not a set difference operation.
+#         A trailing "--" is still a difference operator.
+set-range-dash:
+    '-'                  n  set-after-op                           doSetDifference2
+    default                 set-after-lit                          doSetAddDash
+
+
+set-range-amp:
+    '&'                  n  set-after-op                           doSetIntersection2
+    default                 set-after-lit                          doSetAddAmp
+
+
+#  set-lit-dash
+#     Have scanned "[literals-" Could be a range or a -- operator or a literal
+#     In [abc-[def]], the '-' is a literal (confirmed with a Java test)
+#        [abc-\p{xx}  the '-' is an error
+#        [abc-]       the '-' is a literal
+#        [ab-xy]      the '-' is a range
+#
+set-lit-dash:
+    '-'                  n  set-after-op                            doSetDifference2
+    '['                     set-after-lit                           doSetAddDash
+    ']'                     set-after-lit                           doSetAddDash
+    '\'                  n  set-lit-dash-escape
+    default              n  set-after-range                         doSetRange
+
+# set-lit-dash-escape
+#
+#    scanned "[literal-\"
+#    Could be a range, if the \ introduces an escaped literal char or a named char.
+#    Otherwise it is an error.
+#
+set-lit-dash-escape:
+   's'                      errorDeath                             doSetOpError
+   'S'                      errorDeath                             doSetOpError
+   'w'                      errorDeath                             doSetOpError
+   'W'                      errorDeath                             doSetOpError
+   'd'                      errorDeath                             doSetOpError
+   'D'                      errorDeath                             doSetOpError
+   'N'                      set-after-range                        doSetNamedRange
+   default               n  set-after-range                        doSetRange
+
+   
+#
+#  set-escape
+#       Common back-slash escape processing within set expressions
+#
+set-escape:
+   'p'                      set-after-set                           doSetProp
+   'P'                      set-after-set                           doSetProp
+   'N'                      set-after-lit                           doSetNamedChar
+   's'                   n  set-after-range                         doSetBackslash_s
+   'S'                   n  set-after-range                         doSetBackslash_S
+   'w'                   n  set-after-range                         doSetBackslash_w
+   'W'                   n  set-after-range                         doSetBackslash_W
+   'd'                   n  set-after-range                         doSetBackslash_d
+   'D'                   n  set-after-range                         doSetBackslash_D
+   'h'                   n  set-after-range                         doSetBackslash_h
+   'H'                   n  set-after-range                         doSetBackslash_H
+   'v'                   n  set-after-range                         doSetBackslash_v
+   'V'                   n  set-after-range                         doSetBackslash_V
+   default               n  set-after-lit                           doSetLiteralEscaped 
+
+#
+# set-finish
+#     Have just encountered the final ']' that completes a [set], and
+#     arrived here via a pop.  From here, we exit the set parsing world, and go
+#     back to generic regular expression parsing.
+#
+set-finish:
+    default                 expr-quant                              doSetFinish
+
+

 #
 # errorDeath.   This state is specified as the next state whenever a syntax error
 #               in the source rules is detected.  Barring bugs, the state machine will never
 #
 # errorDeath.   This state is specified as the next state whenever a syntax error
 #               in the source rules is detected.  Barring bugs, the state machine will never