symbol tables: correct handling of variables scope

Author: ssloy

analyzer.py

@@ -0,0 +1,71 @@
+from syntree import *
+from symtable import *
+
+def build_symtable(ast):
+    if not isinstance(ast, Function) or ast.name != 'main' or ast.deco['type'] != Type.VOID or len(ast.args)>0:
+        raise Exception('Cannot find a valid entry point')
+    symtable = SymbolTable()
+    process_scope(ast, symtable)
+
+def process_scope(fun, symtable):
+    fun.deco['nonlocal'] = set() # set of nonlocal variable names in the function body, used in "readable" python transpilation only
+    symtable.push_scope(fun.deco)
+    for v in fun.args: # process function arguments
+        symtable.add_var(*v)
+    for v in fun.var:  # process local variables
+        symtable.add_var(*v)
+    for f in fun.fun:  # then process nested function bodies
+        process_scope(f, symtable)
+    for s in fun.body: # process the list of statements
+        process_stat(s, symtable)
+    symtable.pop_scope()
+
+def process_stat(n, symtable): # process "statement" syntax tree nodes
+    if isinstance(n, Print):
+        process_expr(n.expr, symtable)
+    elif isinstance(n, Return):
+        if n.expr is None: return
+        process_expr(n.expr, symtable)
+    elif isinstance(n, Assign):
+        process_expr(n.expr, symtable)
+        deco = symtable.find_var(n.name)
+        update_nonlocals(n.name, symtable) # used in "readable" python transpilation only
+    elif isinstance(n, FunCall): # no type checking is necessary
+        process_expr(n, symtable)
+    elif isinstance(n, While):
+        process_expr(n.expr, symtable)
+        for s in n.body:
+            process_stat(s, symtable)
+    elif isinstance(n, IfThenElse):
+        process_expr(n.expr, symtable)
+        for s in n.ibody + n.ebody:
+            process_stat(s, symtable)
+    else:
+        raise Exception('Unknown statement type')
+
+def process_expr(n, symtable): # process "expression" syntax tree nodes
+    if isinstance(n, ArithOp):
+        process_expr(n.left,  symtable)
+        process_expr(n.right, symtable)
+    elif isinstance(n, LogicOp):
+        process_expr(n.left,  symtable)
+        process_expr(n.right, symtable)
+    elif isinstance(n, Integer):
+        n.deco['type'] = Type.INT
+    elif isinstance(n, Boolean):
+        n.deco['type'] = Type.BOOL
+    elif isinstance(n, Var):
+        deco = symtable.find_var(n.name)
+        update_nonlocals(n.name, symtable) # used in "readable" python transpilation only
+    elif isinstance(n, FunCall):
+        for s in n.args:
+            process_expr(s, symtable)
+    elif isinstance(n, String):
+        pass
+    else:
+        raise Exception('Unknown expression type', n)
+
+def update_nonlocals(var, symtable):                    # add the variable name to the set of nonlocals
+    for i in reversed(range(len(symtable.variables))):  # for all the enclosing scopes until we find the instance
+        if var in symtable.variables[i]: break          # used in "readable" python transpilation only
+        symtable.ret_stack[i]['nonlocal'].add(var)

symtable.py

@@ -0,0 +1,23 @@
+class SymbolTable():
+    def __init__(self):
+        self.variables = [{}]     # stack of variable symbol tables
+        self.ret_stack = [ None ] # stack of enclosing function symbols, useful for return statements
+
+    def add_var(self, name, deco):
+        if name in self.variables[-1]:
+            raise Exception('Double declaration of the variable %s' % name)
+        self.variables[-1][name] = deco
+
+    def push_scope(self, deco):
+        self.variables.append({})
+        self.ret_stack.append(deco)
+
+    def pop_scope(self):
+        self.variables.pop()
+        self.ret_stack.pop()
+
+    def find_var(self, name):
+        for i in reversed(range(len(self.variables))):
+            if name in self.variables[i]:
+                return self.variables[i][name]
+        raise Exception('No declaration for the variable %s' % name)

tests/test_syntree.py

@@ -1,6 +1,7 @@
 import pytest
 import io, sys # for exec stdout redirection
 from syntree import *
+from analyzer import *
 from transpy_naive import *
 
 def test_syntree(tmp_path):
@@ -60,6 +61,7 @@ def test_syntree(tmp_path):
         ],
         {'type':Type.VOID})     # return type
 
+    build_symtable(main_fun)
     program = transpy(main_fun)
 
     print(program) # save the output to a temp dir

tests/test_transpy.py

@@ -3,6 +3,7 @@
 import io, sys
 from lexer import WendLexer
 from parser import WendParser
+from analyzer import *
 from transpy_naive import *
 
 @pytest.mark.parametrize('test_case', (
@@ -18,6 +19,7 @@ def test_transpy(tmp_path, test_case):
 
     tokens = WendLexer().tokenize(f.read())
     ast = WendParser().parse(tokens)
+    build_symtable(ast)
 
     program = transpy(ast)
     old_stdout = sys.stdout

transpy_naive.py

@@ -3,11 +3,13 @@
 def transpy(n):
     funname   = n.name
     funargs   = ', '.join([v for v,t in n.args])
+    nonlocals = ('nonlocal ' + ', '.join([v for v in n.deco['nonlocal'] ])) if len(n.deco['nonlocal']) else 'pass # no non-local variables'
     allocvars = ''.join(['%s = None\n' % v for v,t in n.var])
     nestedfun = ''.join([transpy(f) for f in n.fun])
     funbody   = ''.join([stat(s) for s in n.body])
     return f'def {funname}({funargs}):\n' + \
-            indent(f'{allocvars}\n'
+            indent(f'{nonlocals}\n'
+                   f'{allocvars}\n'
                    f'{nestedfun}\n'
                    f'{funbody}\n') + \
           (f'{funname}()\n' if n.name=='main' else '')