implemented lexer (with tokens and symbolTable)

todo: beautify code, implement token classes for parser
implemented test function with testcode
moved token class to single file (token.py)

1 files changed, 66 insertions, 0 deletions

diff --git a/src/front/token.py b/src/front/token.py
new file mode 100644
index 0000000..def0236
--- /dev/null
+++ b/src/front/token.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# date: 20 Mai 2009
+
+class Tag:
+    # lexer tokens
+    NUMBER           = 1
+    TRUE             = 2
+    FALSE            = 3
+    IDENT            = 4
+    WHILE            = 5
+    IF               = 6
+    ELSE             = 7
+    END              = 8
+    LBRAK            = 9
+    RBRAK            = 10
+    LPAREN           = 11
+    RPAREN           = 12
+    NEWLINE          = 13
+    COMMA            = 14
+    FUN              = 15
+    ASSIGNMENT       = 16
+    RETURN           = 17
+    OPERATOR         = 18
+
+    # parser tokens
+    BOOL             = 19
+    JOIN             = 20
+    EQUALITY         = 21
+    RELATION         = 22
+    EXPRESSION       = 23
+    TERM             = 24
+    UNARY            = 25
+    FACTOR           = 26
+    IDENT_LIST       = 27
+    EXPRESSION_LIST  = 28
+    PROGRAM          = 29
+    FUNCTION         = 30
+    STATEMENT        = 31
+    STATEMENTS       = 32
+    IF_STATEMENT     = 33
+    WHILE_STATEMENT  = 34
+    RETURN_STATEMENT = 35
+    ASSIGN_STATEMENT = 36
+    FUNCTION_CALL    = 37
+
+class Token:
+    tag = None
+
+    def __init__(self, tag):
+        self.tag = tag
+        return
+
+    def __repr__(self):
+        return "<Token: %d>" % self.tag
+
+class ValueToken(Token):
+    value = None
+
+    def __init__(self, tag, value):
+        Token.__init__(self, tag)
+        self.value = value
+        return
+
+    def __repr__(self):
+        return "<Token: %d Value: %s>" % (self.tag, self.value.__str__())
+