Current version of AST does not always parse all nodes

Author: cedricrupb

code_diff/__init__.py

@@ -18,6 +18,9 @@ def difference(source, target, lang = "guess", **kwargs):
     # Concretize Diff
     source_ast, target_ast = diff_search(source_ast, target_ast)
 
+    if source_ast is None:
+        raise ValueError("Source and Target AST are identical.")
+
     return ASTDiff(config, source_ast, target_ast)
 
 
@@ -79,6 +82,10 @@ def statement_diff(self):
         return ASTDiff(self.config, source_stmt, target_stmt)
 
     def sstub_pattern(self):
+        if (parent_statement(self.config.statement_types, self.source_ast) is None
+                or parent_statement(self.config.statement_types, self.target_ast) is None):
+            return SStubPattern.NO_STMT                
+
         if not self.is_single_statement:
             return SStubPattern.MULTI_STMT
 

code_diff/ast.py

@@ -98,12 +98,17 @@ def __init__(self, create_node_fn):
         self.child_count = defaultdict(int)
 
     def _create_node(self, ast_node, text = None):
+
+        if ast_node.type == "comment": return # We ignore comments
+
         node_key = _node_key(ast_node)
         children = [self.node_index[_node_key(c)] for c in ast_node.children
                      if _node_key(c) in self.node_index]
 
         position = (ast_node.start_point, ast_node.end_point)
         current_node = self.create_node_fn(ast_node.type, children, text = text, position = position)
+        current_node.backend = ast_node
+
         self.node_index[node_key] = current_node
 
         # Add parent if ready
@@ -119,21 +124,51 @@ def _create_node(self, ast_node, text = None):
             self.root_node = current_node
 
 
+    def _open_node(self, node):
+        node_key = _node_key(node)
+        if node_key in self.node_index: return False
+
+        opened = False
+        for c in node.children:
+            opened = opened or self._open_node(c)
+        
+        if not opened:
+            self.waitlist.append(node)
+            return True
+        
+        return False
+
+    def _open_root_if_not_complete(self, base_node):
+        
+        root = base_node
+        while root.parent is not None:
+            root = root.parent
+        
+        for c in root.children:
+            self._open_node(c)
+
     def __call__(self, tokens):
 
         token_nodes = ((t.text, t.ast_node) for t in tokens if hasattr(t, "ast_node"))
         for token_text, token_ast in token_nodes:
             self._create_node(token_ast, text = token_text)
 
-        while len(self.waitlist) > 0:
-            self._create_node(self.waitlist.pop(0))
-    
+        while self.root_node is None:
+            while len(self.waitlist) > 0:
+                current_node = self.waitlist.pop(0)
+                self._create_node(current_node)
+
+            self._open_root_if_not_complete(current_node)
+
+        print(self.root_node.sexp())
+        
         return self.root_node
 
 
 
 # Interface ----------------------------------------------------------------
 
+
 def parse_ast(source_code, lang = "guess", **kwargs):
 
     # Parse AST 

code_diff/gumtree/__init__.py

@@ -38,4 +38,4 @@ def __init__(self, operations):
         super().__init__(operations)
 
     def __repr__(self):
-        return serialize_script(self)
+        return serialize_script(self, indent = 2)

code_diff/gumtree/ops.py

@@ -54,7 +54,7 @@ def _serialize_node(new_node_index, node):
     return _serialize_ast_node(node)
 
 
-def serialize_script(edit_script):
+def serialize_script(edit_script, indent = 0):
 
     sedit_script = []
     new_node_index = {}
@@ -77,15 +77,19 @@ def serialize_script(edit_script):
             else: # Leaf node
                 new_node_str = "%s:%s" % new_node
 
-            sedit_script.append("%s(%s, %s, %d)" % (operation_name, target_node_str, new_node_str, operation.position))
+            sedit_script.append("%s(%s, %s, %d)" % (operation_name, new_node_str, target_node_str, operation.position))
 
         elif operation_name == "Move":
 
             new_node_str = _serialize_node(new_node_index, operation.node)
 
-            sedit_script.append("%s(%s, %s, %d)" % (operation_name, target_node_str, new_node_str, operation.position))
+            sedit_script.append("%s(%s, %s, %d)" % (operation_name, new_node_str, target_node_str, operation.position))
 
         elif operation_name == "Delete":
             sedit_script.append("%s(%s)" % (operation_name, target_node_str))
 
-    return json.dumps(sedit_script, indent=2)
+    if indent > 0:
+        sedit_script = [" "*indent + e for e in sedit_script]
+        return "[\n%s\n]" % (",\n").join(sedit_script)
+
+    return "[%s]" % ", ".join(sedit_script)