From fb9e6a20ac7393b6cc27949ad2d2af7a305c96ba Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 20 May 2009 04:19:52 +0200 Subject: implemented lexer (with tokens and symbolTable) todo: beautify code, implement token classes for parser implemented test function with testcode moved token class to single file (token.py) --- src/front/token.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/front/token.py (limited to 'src/front/token.py') diff --git a/src/front/token.py b/src/front/token.py new file mode 100644 index 0000000..def0236 --- /dev/null +++ b/src/front/token.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# date: 20 Mai 2009 + +class Tag: + # lexer tokens + NUMBER = 1 + TRUE = 2 + FALSE = 3 + IDENT = 4 + WHILE = 5 + IF = 6 + ELSE = 7 + END = 8 + LBRAK = 9 + RBRAK = 10 + LPAREN = 11 + RPAREN = 12 + NEWLINE = 13 + COMMA = 14 + FUN = 15 + ASSIGNMENT = 16 + RETURN = 17 + OPERATOR = 18 + + # parser tokens + BOOL = 19 + JOIN = 20 + EQUALITY = 21 + RELATION = 22 + EXPRESSION = 23 + TERM = 24 + UNARY = 25 + FACTOR = 26 + IDENT_LIST = 27 + EXPRESSION_LIST = 28 + PROGRAM = 29 + FUNCTION = 30 + STATEMENT = 31 + STATEMENTS = 32 + IF_STATEMENT = 33 + WHILE_STATEMENT = 34 + RETURN_STATEMENT = 35 + ASSIGN_STATEMENT = 36 + FUNCTION_CALL = 37 + +class Token: + tag = None + + def __init__(self, tag): + self.tag = tag + return + + def __repr__(self): + return "" % self.tag + +class ValueToken(Token): + value = None + + def __init__(self, tag, value): + Token.__init__(self, tag) + self.value = value + return + + def __repr__(self): + return "" % (self.tag, self.value.__str__()) + -- cgit v1.2.3