Merge pull request #229 from dimitri-yatsenko/master

insert accepts relations.  ERD looks up class names in context.

Author: fabiansinz

CHANGELOG.md

@@ -0,0 +1,5 @@
+## Release notes
+### 0.3.2.   
+* Fixed issue #223:  `insert` can insert relations without fetching
+* ERD() now takes the `context` argument, which specifies in which context to look for classes. The default is taken from the argument (schema or relation).
+* ERD.draw() no longer has the `prefix` argument: class names are shown as found in the context.

README.md

@@ -14,7 +14,6 @@ DataJoint for Python is a high-level programming interface for relational databa
 
 DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
 
-
 ## Quick start guide
 The current pip version is ancient. We will update it as soon as datajoint release 1.0 is out. 
 ~~To install datajoint using `pip` just run:~~

datajoint/__init__.py

@@ -16,8 +16,8 @@
 import os
 
 __author__ = "Dimitri Yatsenko, Edgar Walker, and Fabian Sinz at Baylor College of Medicine"
-__version__ = "0.3.0"
-__date__ = "July 14, 2016"
+__version__ = "0.3.2"
+__date__ = "July 18, 2016"
 __all__ = ['__author__', '__version__',
            'config', 'conn', 'kill', 'BaseRelation',
            'Connection', 'Heading', 'FreeRelation', 'Not', 'schema',

datajoint/base_relation.py

@@ -37,6 +37,10 @@ def heading(self):
             self._heading.init_from_database(self.connection, self.database, self.table_name)
         return self._heading
 
+    @property
+    def context(self):
+        return self._context
+
     def declare(self):
         """
         Loads the table heading. If the table is not declared, use self.definition to declare
@@ -98,16 +102,18 @@ def full_table_name(self):
     def insert1(self, row, **kwargs):
         """
         Insert one data record or one Mapping (like a dict).
-        :param row: Data record, a Mapping (like a dict), or a list or tuple with ordered values.
+        :param row: a numpy record, a dict-like object, or an ordered sequence to be inserted as one row.
+        For kwargs, see insert()
         """
         self.insert((row,), **kwargs)
 
     def insert(self, rows, replace=False, ignore_errors=False, skip_duplicates=False):
         """
-        Insert a collection of rows. Additional keyword arguments are passed to insert1.
+        Insert a collection of rows.
 
-        :param rows: An iterable where an element is a valid arguments for insert1.
-        :param replace: If True, replaces the matching data tuple in the table if it exists.
+        :param rows: An iterable where an element is a numpy record, a dict-like object, or an ordered sequence.
+        rows may also be another relation with the same heading.
+        :param replace: If True, replaces the existing tuple.
         :param ignore_errors: If True, ignore errors: e.g. constraint violations.
         :param skip_duplicates: If True, silently skip duplicate inserts.
 
@@ -116,6 +122,19 @@ def insert(self, rows, replace=False, ignore_errors=False, skip_duplicates=False
         >>>     dict(subject_id=7, species="mouse", date_of_birth="2014-09-01"),
         >>>     dict(subject_id=8, species="mouse", date_of_birth="2014-09-02")])
         """
+
+        if isinstance(rows, RelationalOperand):
+            # INSERT FROM SELECT
+            if skip_duplicates:
+                ignore_errors = True
+            query = 'INSERT{ignore} INTO {table} ({fields}) {select}'.format(
+                ignore=" IGNORE" if ignore_errors else "",
+                table=self.full_table_name,
+                fields='`'+'`,`'.join(rows.heading.names)+'`',
+                select=rows.make_sql())
+            self.connection.query(query)
+            return
+
         heading = self.heading
         field_list = None  # ensures that all rows have the same attributes in the same order as the first row.
 
@@ -336,30 +355,42 @@ def show_definition(self):
                     do_include = False
                 if attributes_thus_far.issuperset(primary_key):
                     parents.pop(parent)
-                    definition += '-> ' + self.lookup_table_name(parent) + '\n'
+                    definition += '-> ' + lookup_class_name(self.context, parent) + '\n'
             if do_include:
                 definition += '%-20s : %-28s # %s\n' % (
-                    attr.name if attr.default is None else '%s=%s' % (attr.name, attr.default),
+                    attr.name if attr.default is None else '%s = %s' % (attr.name, attr.default),
                     '%s%s' % (attr.type, 'auto_increment' if attr.autoincrement else ''), attr.comment)
+        print(definition)
         return definition
 
-    def lookup_table_name(self, name):
-        """
-        given the name of another table in the form `database`.`table_name`, find its class in the context
-        :param name: `database`.`table_name`
-        :return: class name found in the context.
-        """
-        def _lookup(context, name, level=0):
-            for member_name, member in context.items():
-                if inspect.isclass(member) and issubclass(member, BaseRelation) and member.full_table_name == name:
-                    return member_name
-                if level < 5 and inspect.ismodule(member) and member.__name__ != 'datajoint':
-                    candidate_name = _lookup(dict(inspect.getmembers(member)), name, level+1)
-                    if candidate_name != name:
-                        return member_name + '.' + candidate_name
-            return name
 
-        return _lookup(self._context, name)
+def lookup_class_name(context, name, levels=4):
+    """
+    given the name of another table in the form `database`.`table_name`, find its class in the context
+    :param context: dictionary representing the namespace
+    :param name: `database`.`table_name`
+    :param levels: the limit on how deep to go into imported modules, helps avoid infinite recursion.
+    :return: class name found in the context.
+    """
+    for member_name, member in context.items():
+        if inspect.isclass(member) and issubclass(member, BaseRelation):
+            if member.full_table_name == name:
+                return member_name
+            # check part tables
+            try:
+                parts = member._ordered_class_members
+            except AttributeError:
+                pass  # not a UserRelation
+            else:
+                for part in parts:
+                    part = getattr(member, part)
+                    if inspect.isclass(part) and issubclass(part, BaseRelation) and part.full_table_name == name:
+                        return member_name + '.' + part.__name__
+        elif levels and inspect.ismodule(member) and member.__name__ != 'datajoint':
+            candidate_name = lookup_class_name(dict(inspect.getmembers(member)), name, levels-1)
+            if candidate_name != name:
+                return member_name + '.' + candidate_name
+    return name
 
 
 class FreeRelation(BaseRelation):

datajoint/erd.py

@@ -1,22 +1,15 @@
 import networkx as nx
-import numpy as np
 import re
+import numpy as np
 from scipy.optimize import basinhopping
 import itertools
 import inspect
 from . import Manual, Imported, Computed, Lookup, Part, DataJointError
+from .base_relation import lookup_class_name
 
 user_relation_classes = (Manual, Lookup, Computed, Imported, Part)
 
 
-def _get_concrete_subclasses(class_list):
-    for cls in class_list:
-        for subclass in cls.__subclasses__():
-            if not inspect.isabstract(subclass):
-                yield subclass
-            yield from _get_concrete_subclasses([subclass])
-
-
 def _get_tier(table_name):
     try:
         return next(tier for tier in user_relation_classes
@@ -30,64 +23,71 @@ class ERD(nx.DiGraph):
     Entity relationship diagram.
 
     Usage:
-    >>>  erd = Erd(source)
-    source can be a base relation object, a base relation class, a schema, or a module that has a schema
-    or source can be a sequence of such objects.
+    >>>  erd = Erd(source, context=None)
+    source can be a base relation object, a base relation class, a schema, or a module that has a schema.
+    context is the namespace in which to search for classes. If None (default), looks
+    in source.context or source.schema.context
 
     >>> erd.draw()
     draws the diagram using pyplot
 
     erd1 + erd2  - combines the two ERDs.
     erd + n   - adds n levels of successors
-    erd - n   - adds n levens of predecessors
+    erd - n   - adds n levels of predecessors
     Thus dj.ERD(schema.Table)+1-1 defines the diagram of immediate ancestors and descendants of schema.Table
 
     Note that erd + 1 - 1  may differ from erd - 1 + 1 and so forth.
     Only those tables that are loaded in the connection object are displayed
     """
-    def __init__(self, source):
+    def __init__(self, source, context=None):
 
         if isinstance(source, ERD):
             # copy constructor
             self.nodes_to_show = set(source.nodes_to_show)
+            self._context = source._context
             super().__init__(source)
             return
 
-        # if source is not a list, make it a list
+        # find connection in the source
         try:
-            source[0]
-        except (TypeError, KeyError):
-            source = [source]
-
-        # find connection in the first item in the list
-        try:
-            connection = source[0].connection
+            connection = source.connection
         except AttributeError:
             try:
-                connection = source[0].schema.connection
+                connection = source.schema.connection
+            except AttributeError:
+                raise DataJointError('Could not find database connection in %s' % repr(source[0]))
+
+        if context is None:
+            # find context in the source
+            try:
+                context = source.context
             except AttributeError:
-                raise DataJointError('Could find database connection in %s' % repr(source[0]))
+                try:
+                    context = source.schema.context
+                except AttributeError:
+                    raise DataJointError('Could not find context in %s' % repr(source[0]))
+
+        self._context = context
 
         # initialize graph from dependencies
         connection.dependencies.load()
         super().__init__(connection.dependencies)
 
         # Enumerate nodes from all the items in the list
         self.nodes_to_show = set()
-        for source in source:
+        try:
+            self.nodes_to_show.add(source.full_table_name)
+        except AttributeError:
             try:
-                self.nodes_to_show.add(source.full_table_name)
+                database = source.database
             except AttributeError:
                 try:
-                    database = source.database
+                    database = source.schema.database
                 except AttributeError:
-                    try:
-                        database = source.schema.database
-                    except AttributeError:
-                        raise DataJointError('Cannot plot ERD for %s' % repr(source))
-                for node in self:
-                    if node.startswith('`%s`' % database):
-                        self.nodes_to_show.add(node)
+                    raise DataJointError('Cannot plot ERD for %s' % repr(source))
+            for node in self:
+                if node.startswith('`%s`' % database):
+                    self.nodes_to_show.add(node)
 
     def __add__(self, arg):
         """
@@ -131,31 +131,25 @@ def __mul__(self, arg):
         self.nodes_to_show.intersection_update(arg.nodes_to_show)
         return self
 
-    def _make_graph(self, prefix):
+    def _make_graph(self):
         """
         Make the self.graph - a graph object ready for drawing
         """
         graph = nx.DiGraph(self).subgraph(self.nodes_to_show)
         nx.set_node_attributes(graph, 'node_type', {n: _get_tier(n.split('`')[-2]) for n in graph})
         # relabel nodes to class names
-        class_list = list(cls for cls in _get_concrete_subclasses(user_relation_classes))
-        mapping = {
-            cls.full_table_name:
-                (cls._context['__name__'] + '.'
-                 if (prefix and cls._context['__name__'] != '__main__') else '') +
-                (cls._master.__name__+'.' if issubclass(cls, Part) else '') + cls.__name__
-                   for cls in class_list if cls.full_table_name in graph}
+        mapping = {node: lookup_class_name(self._context, node) for node in graph.nodes()}
         new_names = [mapping.values()]
         if len(new_names) > len(set(new_names)):
             raise DataJointError('Some classes have identical names. The ERD cannot be plotted.')
         nx.relabel_nodes(graph, mapping, copy=False)
         return graph
 
-    def draw(self, pos=None, layout=None, prefix=True, font_scale=1.2, **layout_options):
+    def draw(self, pos=None, layout=None, font_scale=1.2, **layout_options):
         if not self.nodes_to_show:
             print('There is nothing to plot')
             return
-        graph = self._make_graph(prefix)
+        graph = self._make_graph()
         if pos is None:
             pos = (layout if layout else self._layout)(graph, **layout_options)
         import matplotlib.pyplot as plt

datajoint/heading.py

@@ -183,6 +183,9 @@ def init_from_database(self, conn, database, table_name):
             if attr['string'] and attr['default'] is not None and attr['default'] not in sql_literals:
                 attr['default'] = '"%s"' % attr['default']
 
+            if attr['nullable']:   # nullable fields always default to null
+                attr['default'] = 'null'
+
             attr['sql_expression'] = None
             if not (attr['numeric'] or attr['string'] or attr['is_blob']):
                 raise DataJointError('Unsupported field type {field} in `{database}`.`{table_name}`'.format(

datajoint/schema.py

@@ -6,7 +6,7 @@
 from . import conn, DataJointError, config
 from .heading import Heading
 from .utils import user_choice, to_camel_case
-from .user_relations import UserRelation, Part, Computed, Imported, Manual, Lookup
+from .user_relations import Part, Computed, Imported, Manual, Lookup
 
 logger = logging.getLogger(__name__)
 
@@ -117,7 +117,6 @@ def exists(self):
         cur = self.connection.query("SHOW DATABASES LIKE '{database}'".format(database=self.database))
         return cur.rowcount > 0
 
-
     def process_relation_class(self, relation_class, context, assert_declared=False):
         """
         assign schema properties to the relation class and declare the table
@@ -141,7 +140,6 @@ def process_relation_class(self, relation_class, context, assert_declared=False)
                 else:
                     instance.insert(contents, skip_duplicates=True)
 
-
     def __call__(self, cls):
         """
         Binds the passed in class object to a database. This is intended to be used as a decorator.
@@ -170,3 +168,4 @@ def jobs(self):
         :return: jobs relation
         """
         return self.connection.jobs[self.database]
+

tests/test_erd.py

@@ -32,7 +32,7 @@ def test_dependencies():
     @staticmethod
     def test_erd():
         erd = dj.ERD(schema)
-        graph = erd._make_graph(prefix=False)
+        graph = erd._make_graph()
         assert_true(set(cls.__name__ for cls in (A, B, D, E, L)).issubset(graph.nodes()))
         pos = erd._layout(graph)
         assert_true(set(cls.__name__ for cls in (A, B, D, E, L)).issubset(pos.keys()))

tests/test_relation.py

@@ -84,6 +84,12 @@ def test_wrong_arguments_insert(self):
     def test_wrong_insert_type(self):
         self.user.insert1(3)
 
+    def test_insert_select(self):
+        original_length = len(self.subject)
+        self.subject.insert(self.subject.proj(
+            'real_id', 'date_of_birth', 'subject_notes', subject_id='subject_id+1000', species='"human"'))
+        assert_equal(len(self.subject), 2*original_length)
+
     def test_replace(self):
         """
         Test replacing or ignoring duplicate entries