+++ /dev/null
-#----------------------------------------------------------------------------
-# Name: parser.py
-# Purpose: parsing utilities
-#
-# Author: Jeff Norton
-#
-# Created: 8/9/05
-# CVS-ID: $Id$
-# Copyright: (c) 2004-2005 ActiveGrid, Inc.
-# License: wxWindows License
-#----------------------------------------------------------------------------
-
-import re
-from activegrid.util.lang import *
-ifDefPy()
-import string
-import array
-endIfDef()
-
-XPATH_ROOT_VAR = '__rootObj__'
-GETOBJECTPARTNAMES = ["primaryRef", "ref", "orderings", "limit"]
-
-class Tokenizer(object):
-
- TOKEN_IDENT = 1
- TOKEN_STRING = 2
- TOKEN_OP = 3
- TOKEN_WS = 4
-## TOKEN_PLACEHOLDER = 5
-
- def __init__(self, text, identStart=None, tokenSep=None, ignoreWhitespace=True):
- """
-Turn a string into individual tokens. Three types of tokens are recognized:
- TOKEN_IDENT: identifiers (those that start with the identStart pattern)
- TOKEN_STRING: quoted string
- TOKEN_OP: everything else
-Tokens are separated by white space or the tokenSep pattern.
-Constructor parameters:
- text: The string to tokenize
- identStart: A regular expression describing characters which start an identifier
- The default expression accepts letters, "_", and "/".
- tokenSep: A regular expression describing the characters which end a token
- (in addition to whitespace). The default expression accepts
- anything except alpha-numerics, "_", "/", and ":".
-Usage:
- Invoke getNextToken (or next) to get the next token. The instance variables
- token, and tokenVal will be populated with the current token type (TOKEN_IDENT,
- TOKEN_STRING, or TOEKN_OP) and value respectively. nextToken and nextTokenVal
- will also be available for lookahead. The next method is similar to
- getNextToken but also returns the token value. A value of None signals end
- of stream.
- """
- self.ignoreWhitespace=ignoreWhitespace
- ifDefPy()
- if (isinstance(text, array.array)):
- text = text.tostring()
- endIfDef()
- self.text = asString(text)
- self.textIndex = 0
- self.textLen = len(self.text)
- self.token = None
- self.tokenVal = None
- self.nextToken = None
- self.nextTokenVal = None
- if (identStart == None):
- identStart = "[a-zA-Z_/]"
- if (tokenSep == None):
- tokenSep = "[^a-zA-Z0-9_/:]"
- self.identStart = re.compile(identStart)
- self.tokenSep = re.compile(tokenSep)
- self.getNextToken() # Prime the pump
-
- def isEscaped(text, index):
- if ((index > 0) and (text[index-1] == '\\') and ((index < 2) or (text[index-2] != '\\'))):
- return True
- return False
- isEscaped = staticmethod(isEscaped)
-
- def findClosingQuote(text, index, char):
- index = index + 1
- while True:
- endIndex = text.find(char, index)
- if (endIndex < 1):
- return -1
- if (Tokenizer.isEscaped(text, endIndex)):
- index = endIndex+1
- else:
- break
- return endIndex + 1
- findClosingQuote = staticmethod(findClosingQuote)
-
- def _findClosing(self, char):
- if (self.textIndex >= self.textLen):
- raise Exception("The text \"%s\" has an unmatched string starting at %d" % (self.text, self.textIndex))
- index = Tokenizer.findClosingQuote(self.text, self.textIndex, char)
- if (index < 0):
- raise Exception("The text \"%s\" has an unmatched string starting at %d" % (self.text, self.textIndex-1))
- return index
-
- def next(self):
- self.getNextToken()
- if (self.token == None):
- raise StopIteration()
- return self.tokenVal
-
- def getNextToken(self):
- self.token = self.nextToken
- self.tokenVal = self.nextTokenVal
- while (self.textIndex < self.textLen):
- c = self.text[self.textIndex]
- if (c not in string.whitespace):
- if (c == '"' or c == "'" or c == '`'):
- endIndex = self._findClosing(c)
- self.nextToken = self.TOKEN_STRING
- self.nextTokenVal = self.text[self.textIndex:endIndex]
- self.textIndex = endIndex
- return
- elif (self.identStart.search(c)):
- endMatch = self.tokenSep.search(self.text, self.textIndex+1)
- if (endMatch):
- endIndex = endMatch.start()
- else:
- endIndex = self.textLen
- self.nextToken = self.TOKEN_IDENT
- self.nextTokenVal = self.text[self.textIndex:endIndex]
- self.textIndex = endIndex
- return
- else:
- self.nextToken = self.TOKEN_OP
- endIndex = self.textIndex + 1
- if (c == '<' or c == '>' or c == '!' or c == '='):
- if ((endIndex < self.textLen) and (self.text[endIndex] == '=')):
- endIndex += 1
- elif ((c == '%') and (endIndex < self.textLen)):
- c = self.text[endIndex]
- if (c in ['d', 'i', 'o', 'u', 'x', 'X', 'e', 'E', 'f', 'F', 'g', 'G', 'c', 'r', 's', '%']):
- endIndex += 1
-## self.nextToken = self.TOKEN_PLACEHOLDER # Should really be this but no one can handle it yet
- self.nextTokenVal = self.text[self.textIndex:endIndex]
- self.textIndex = endIndex
- return
- elif not self.ignoreWhitespace:
- self.nextToken=self.TOKEN_WS
- self.nextTokenVal=""
- while c in string.whitespace:
- self.nextTokenVal+=c
- self.textIndex+=1
- if self.textIndex==len(self.text):
- break
- c=self.text[self.textIndex]
- return
- self.textIndex += 1
- self.nextToken = None
- self.nextTokenVal = None
-
-def isXPathNonVar(var):
- """Returns true iff var is a string ("foo" or 'foo') or a number."""
- if (var.startswith("'") and var.endswith("'")) or \
- (var.startswith('"') and var.endswith('"')):
- return True
-
- # list from XPathToCode, below
- if var.lower() in ["count", "empty", "true", "false", "null", "and", "or", \
- "like", "not"]:
- return True
-
- try:
- t=int(var)
- return True
- except TypeError, e:
- pass
- except ValueError, e:
- pass
-
- return False
-
-def xpathToCode(xpaths, convertBracket=True):
- if ((xpaths == None) or (len(xpaths) < 1)):
- return "True"
- if (not isinstance(xpaths, (list, tuple))):
- xpaths = [xpaths]
- result = []
- for xpath in xpaths:
- t = Tokenizer(xpath, "[a-zA-Z0-9_/:\.]", "[^a-zA-Z0-9_/:\.]", ignoreWhitespace=False)
- expr = []
- lastToken=None
- while t.nextToken != None:
- t.getNextToken()
- if (t.token == Tokenizer.TOKEN_WS):
- expr.append(" ")
- elif (t.token == Tokenizer.TOKEN_OP):
- if (t.tokenVal == "="):
- expr.append("==")
- elif (t.tokenVal == "[" and convertBracket):
- expr.append("(")
- elif (t.tokenVal == "]" and convertBracket):
- expr.append(")")
- else:
- expr.append(t.tokenVal)
- elif (t.token == Tokenizer.TOKEN_IDENT):
- if (t.tokenVal == "and"):
- expr.append(" and ")
- elif (t.tokenVal == "or"):
- expr.append(" or ")
- elif (t.tokenVal == "not"):
- expr.append(" not ")
- elif (t.tokenVal == "like"):
- # REVIEW stoens@activegrid.com 02-Nov-05 --
- # This is very limited support for like:
- # typically like queries look like this: "foo like 'blah%'".
- # So translate this into "foo.startswith(blah)".
- # We should use a regular expression to support '%'s in
- # arbitrary places in the string. After 1.1.
- if t.nextToken and t.nextTokenVal.endswith("%'"):
- t.getNextToken() # throw away the "like" token
- last = len(expr) - 1
- expr[last] = "%s.startswith(%s')"\
- % (expr[last], t.tokenVal[:-2])
- else:
- # old behavior
- expr.append(t.tokenVal)
-
- elif (t.tokenVal == "count"):
- expr.append("len")
- elif (t.tokenVal == 'empty'):
- expr.append('ctx.isEmptyPath')
- elif (t.tokenVal == 'true'):
- expr.append(_parseConstantFunction(t, 'True'))
- elif (t.tokenVal == 'false'):
- expr.append(_parseConstantFunction(t, 'False'))
- elif (t.tokenVal == 'null'):
- expr.append(_parseConstantFunction(t, 'None'))
- elif (-1!=t.tokenVal.find(':')):
- serviceDef, args=_parseServiceFunction(t)
-
- # XXX handle serviceDef, args being None
-
- for i in range(len(args)):
- args[i]=xpathToCode(args[i], False)
- jargs="[%s]" % (",".join(args))
-
- # XXX should be processmodel.DATASERVICE_PROCESS_NAME, not "dataservice"
- if serviceDef[0]=='dataservice':
- expr.append("runtimesupport.invokeDataServiceWrapper(%s, %s, ctx, locals())" % \
- (serviceDef, jargs))
- else:
- expr.append("runtimesupport.invokeServiceWrapper(%s, %s, ctx)" % \
- (serviceDef, jargs))
- else:
- if (lastToken==')' or lastToken==']'):
- wasFunc=True
- else:
- wasFunc=False
- if (t.tokenVal.startswith('/')) and not wasFunc:
- expr.append(XPATH_ROOT_VAR)
- expr.append(t.tokenVal.replace('/','.'))
- lastToken=t.tokenVal
- else:
- expr.append(t.tokenVal)
-
-
- if (len(expr) == 2 and expr[0]==" "):
- expr = "".join(expr)
- result.append(expr)
- elif (len(expr) > 1):
- expr = "".join(expr)
- result.append("(%s)" % expr)
- elif (len(expr) > 0):
- result.append(expr[0])
-
- return " and ".join(result)
-
-def _parseArgs(t):
- args=[]
- argcon=""
-
- if t.tokenVal!='(':
- return []
- if t.nextTokenVal==')':
- t.getNextToken()
- return []
-
- depth=1
-
- while(depth!=0):
- if not t.nextToken:
- raise Exception("parameters list with no closing ) after token: %s" % t.tokenVal)
- t.getNextToken()
-
- if t.tokenVal=='(':
- depth+=1
- if t.tokenVal==')':
- depth-=1
-
- if depth==0 or (depth==1 and t.tokenVal==','):
- args.append(argcon)
- argcon=""
- else:
- argcon+=t.tokenVal
- return args
-
-def _parseServiceFunction(t):
- """Parses what appears to be a service function call into serviceDefs and args lists.
-
- Returns None, None if the serviceFunction appears to be invalid.
- """
- if t.nextTokenVal!='(':
- return t.tokenVal, None
-
- serviceDef=t.tokenVal.split(':')
- t.getNextToken()
- args=_parseArgs(t)
-
- return serviceDef, args
-
-def _parseConstantFunction(t, outputValue):
- firstVal = t.tokenVal
- if t.nextTokenVal != '(':
- return firstVal
- t.getNextToken()
- if t.nextTokenVal != ')':
- return "%s%s" % (firstVal, '(')
- t.getNextToken()
- return outputValue
-
-def parseDSPredicate(ctx, str, vars, valueList=None):
- from activegrid.util.utillang import evalCode
- from activegrid.util.utillang import ObjAsDict
-
- if valueList == None:
- valueList = []
- indexVar=0
- oldIndexVar=0
- sourceStr=str
- inlinedPredicate=[]
- qualifications=[]
- while True:
- oldIndexVar = indexVar
- dollarCurlForm = False
- quoted = False
- indexVar = sourceStr.find("bpws:getVariableData", indexVar)
- if indexVar == -1:
- indexVar = sourceStr.find("${", oldIndexVar)
- if indexVar == -1:
- break
- dollarCurlForm = True
- if indexVar > 0 and sourceStr[indexVar-1] in ('"',"'"):
- quoted = True
- if not dollarCurlForm:
- openParen = sourceStr.find("(", indexVar)
- if openParen == -1:
- break
- closeParen = sourceStr.find(")", openParen)
- if closeParen == -1:
- break
- else:
- openParen = indexVar+1
- closeParen = sourceStr.find("}", openParen)
- if closeParen == -1:
- break
- varRef = sourceStr[openParen+1: closeParen]
- if varRef.startswith('"') or varRef.startswith("'"):
- varRef = varRef[1:]
- if varRef.endswith('"') or varRef.endswith("'"):
- varRef = varRef[:-1]
- if isinstance(vars, dict) or isinstance(vars, ObjAsDict):
- varRefCode = xpathToCode(varRef)
- value = evalCode(varRefCode, vars)
- else:
- value = ctx.evalPath(vars, varRef)
- inlinedPredicate.append(sourceStr[oldIndexVar:indexVar])
- if quoted:
- inlinedPredicate.append("%s" % value)
- else:
- inlinedPredicate.append('%s')
- valueList.append(value)
- indexVar = closeParen+1
- inlinedPredicate.append(sourceStr[oldIndexVar:])
- qualifications.append(''.join(inlinedPredicate))
- return qualifications, valueList
projects / wxWidgets.git / blobdiff