Added wxEditor from Dirc Holtwic

[wxWidgets.git] / utils / wxPython / lib / editor / tokenizer.py

diff --git a/utils/wxPython/lib/editor/tokenizer.py b/utils/wxPython/lib/editor/tokenizer.py
new file mode 100644 (file)
index 0000000..aafe0aa
--- /dev/null
+++ b/utils/wxPython/lib/editor/tokenizer.py
@@ -0,0 +1,60 @@
+from tokenize import *
+from keyword import *
+from string import *
+
+class Tokenizer:
+    """
+    Simple class to create a list of token-tuples like:
+
+       (type, string, first, last)
+
+    Example:
+    t = Tokenizer('def hallo(du): # juchee')
+    print t.tokens()
+    """
+
+    def __init__(self, text):
+       self.text = text
+       self.toks = []
+       try:
+           tokenize(self.readline, self.get)
+       except TokenError:
+           pass
+
+    def tokens(self):
+       return self.toks
+
+    def get(self, type, string, begin, end, l):
+       #print begin,end
+       h1, b = begin
+       h2, e = end
+       tname = tok_name[type]
+       if iskeyword(string):
+           tname = "KEY"
+       self.toks.append(tname, string, b, e)
+
+    def readline(self):
+       t = self.text
+       self.text = ""
+       return t
+
+    def line(self):
+       pre = ""
+       out = ""
+       for type, string, begin, end in self.toks:
+           if (pre in ["NAME","KEY"]) and (not string in [".",",","("]):
+               out = out + " "
+
+           if type in ["NAME","KEY"]:
+               out = out + string
+           elif type=="OP":
+               if string in [",",":"]:
+                   out = out + string + " "
+               else:
+                           out = out + string
+           else:
+               out = out + string
+           pre = type
+       return out
+
+