Merge pull request #281 from dimitri-yatsenko/master

Fix issues #280 and #252

Author: eywalker

.travis.yml

@@ -1,7 +1,7 @@
 sudo: required
 language: python
 env:
-  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="datajoint" DJ_TEST_PASSWORD="datajoint" DJ_HOST="127.0.0.1" DJ_USER="datajoint" DJ_PASSWORD="datajoint"
+  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="datajoint" DJ_TEST_PASSWORD="datajoint" DJ_HOST="127.0.0.1" DJ_USER="datajoint" DJ_PASS="datajoint"
 python:
   - "3.4"
   - "3.5"

datajoint/__init__.py

@@ -16,8 +16,8 @@
 import os
 
 __author__ = "Dimitri Yatsenko, Edgar Y. Walker, and Fabian Sinz at Baylor College of Medicine"
-__version__ = "0.4.6"
-__date__ = "December 22, 2016"
+__version__ = "0.4.7"
+__date__ = "January 24, 2017"
 __all__ = ['__author__', '__version__',
            'config', 'conn', 'kill', 'BaseRelation',
            'Connection', 'Heading', 'FreeRelation', 'Not', 'schema',
@@ -47,23 +47,25 @@ class DataJointError(Exception):
 from .settings import Config, LOCALCONFIG, GLOBALCONFIG, logger, log_levels
 config = Config()
 
-
-if os.getenv('DJ_HOST') is not None and os.getenv('DJ_USER') is not None and os.getenv('DJ_PASS') is not None:  # pragma: no cover
-    print("Loading local settings from environment variables")
-    config['database.host'] = os.getenv('DJ_HOST')
-    config['database.user'] = os.getenv('DJ_USER')
-    config['database.password'] = os.getenv('DJ_PASS')
-elif os.path.exists(LOCALCONFIG):  # pragma: no cover
-    local_config_file = os.path.expanduser(LOCALCONFIG)
-    print("Loading local settings from {0:s}".format(local_config_file))
-    logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file))
-    config.load(local_config_file)
-elif os.path.exists(os.path.expanduser('~/') + GLOBALCONFIG):  # pragma: no cover
-    local_config_file = os.path.expanduser('~/') + GLOBALCONFIG
+config_files = (os.path.expanduser(n) for n in (LOCALCONFIG, '~/' + GLOBALCONFIG))
+try:
+    local_config_file = next(n for n in config_files if os.path.exists(n))
+except StopIteration:
+    local_config_file = None
+else:
     print("Loading local settings from {0:s}".format(local_config_file))
     logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file))
     config.load(local_config_file)
-else:
+
+# override login credentials with environment variables
+mapping = {k: v for k, v in zip(
+    ('database.host', 'database.user', 'database.password'),
+    map(os.getenv, ('DJ_HOST', 'DJ_USER', 'DJ_PASS')))
+           if v is not None}
+config.update(mapping)
+
+# create local config file if neither local nor global file exists
+if not mapping and local_config_file is None:
     print("""Cannot find configuration settings. Using default configuration. To change that, either
     * modify the local copy of %s that datajoint just saved for you
     * put a file named %s with the same configuration format in your home

datajoint/base_relation.py

@@ -348,6 +348,9 @@ def size_on_disk(self):
                 database=self.database, table=self.table_name), as_dict=True).fetchone()
         return ret['Data_length'] + ret['Index_length']
 
+    def describe(self):
+        return self.show_definition()
+
     def show_definition(self):
         """
         :return:  the definition string for the relation using DataJoint DDL.

datajoint/connection.py

@@ -6,6 +6,7 @@
 from contextlib import contextmanager
 import pymysql as client
 import logging
+from getpass import getpass
 
 from . import config
 from . import DataJointError
@@ -16,7 +17,7 @@
 logger = logging.getLogger(__name__)
 
 
-def conn(host=None, user=None, passwd=None, init_fun=None, reset=False):
+def conn(host=None, user=None, password=None, init_fun=None, reset=False):
     """
     Returns a persistent connection object to be shared by multiple modules.
     If the connection is not yet established or reset=True, a new connection is set up.
@@ -26,18 +27,20 @@ def conn(host=None, user=None, passwd=None, init_fun=None, reset=False):
 
     :param host: hostname
     :param user: mysql user
-    :param passwd: mysql password
+    :param password: mysql password
     :param init_fun: initialization function
-    :param reset: whether the connection should be reseted or not
+    :param reset: whether the connection should be reset or not
     """
     if not hasattr(conn, 'connection') or reset:
         host = host if host is not None else config['database.host']
         user = user if user is not None else config['database.user']
-        passwd = passwd if passwd is not None else config['database.password']
-        if passwd is None:  # pragma: no cover
-            passwd = input("Please enter database password: ")
+        password = password if password is not None else config['database.password']
+        if user is None:  # pragma: no cover
+            user = input("Please enter DataJoint username: ")
+        if password is None:  # pragma: no cover
+            password = getpass(prompt="Please enter DataJoint password: ")
         init_fun = init_fun if init_fun is not None else config['connection.init_function']
-        conn.connection = Connection(host, user, passwd, init_fun)
+        conn.connection = Connection(host, user, password, init_fun)
     return conn.connection
 
 
@@ -50,19 +53,20 @@ class Connection:
 
     :param host: host name
     :param user: user name
-    :param passwd: password
-    :param init_fun: initialization function
+    :param password: password
+    :param init_fun: connection initialization function (SQL)
     """
 
-    def __init__(self, host, user, passwd, init_fun=None):
+    def __init__(self, host, user, password, init_fun=None):
         if ':' in host:
             host, port = host.split(':')
             port = int(port)
         else:
             port = config['database.port']
-        self.conn_info = dict(host=host, port=port, user=user, passwd=passwd)
+        self.conn_info = dict(host=host, port=port, user=user, passwd=password)
         self.init_fun = init_fun
         print("Connecting {user}@{host}:{port}".format(**self.conn_info))
+        self._conn = None
         self.connect()
         if self.is_connected:
             logger.info("Connected {user}@{host}:{port}".format(**self.conn_info))
@@ -82,11 +86,9 @@ def __repr__(self):
         return "DataJoint connection ({connected}) {user}@{host}:{port}".format(
             connected=connected, **self.conn_info)
 
-    def connect(self, init_fun=None):
+    def connect(self):
         """
         Connects to the database server.
-
-        :param init_fun: initialization function passed to pymysql
         """
         self._conn = client.connect(init_command=self.init_fun, **self.conn_info)
 
@@ -198,6 +200,3 @@ def transaction(self):
             raise
         else:
             self.commit_transaction()
-
-
-

datajoint/dependencies.py

@@ -56,8 +56,14 @@ def add_table(self, table_name):
             except pp.ParseException:
                 pass
             else:
+                referencing_attributes = [r.strip('` ') for r in result.attributes.split(',')]
+                referenced_attributes = [r.strip('` ') for r in result.referenced_attributes.split(',')]
                 self.add_edge(result.referenced_table, table_name,
-                              primary=all(r.strip('` ') in primary_key for r in result.attributes.split(',')))
+                              primary=all(a in primary_key for a in referencing_attributes),
+                              referenced_attributes=referenced_attributes,
+                              referencing_attributes=referencing_attributes,
+                              aliased=any(a != b for a, b in zip(referenced_attributes, referenced_attributes)),
+                              multi=any(a not in primary_key for a in referencing_attributes))
 
     def load(self, target=None):
         """
@@ -82,4 +88,4 @@ def descendants(self, full_table_name):
         :return: all dependent tables sorted in topological order.  Self is included.
         """
         nodes = nx.algorithms.dag.descendants(self, full_table_name)
-        return [full_table_name] + nx.algorithms.dag.topological_sort(self, nodes)
+        return [full_table_name] + nx.algorithms.dag.topological_sort(self, nodes)

datajoint/heading.py

@@ -217,23 +217,29 @@ def project(self, attribute_list, named_attributes=None, force_primary_key=None)
         """
         derive a new heading by selecting, renaming, or computing attributes.
         In relational algebra these operators are known as project, rename, and extend.
+        :param attribute_list:  the full list of existing attributes to include
+        :param force_primary_key:  attributes to force to be converted to primary
+        :param named_attributes:  dictionary of renamed attributes
         """
         try:  # check for missing attributes
             raise DataJointError('Attribute `%s` is not found' % next(a for a in attribute_list if a not in self.names))
         except StopIteration:
             if named_attributes is None:
                 named_attributes = {}
+            if force_primary_key is None:
+                force_primary_key = set()
             return Heading(
-                [dict(v.todict(), **dict(
-                    () if force_primary_key is None else [('in_key', k in force_primary_key)]))
-                 for k, v in self.attributes.items() if k in attribute_list] +
-                [dict(  # rename attribute
+                [dict(   # copied attributes
+                    self.attributes[k].todict(),
+                    in_key=self.attributes[k].in_key or k in force_primary_key)
+                 for k in attribute_list] +
+                [dict(  # renamed attributes
                     self.attributes[sql_expression].todict(),
                     name=new_name,
-                    sql_expression='`' + sql_expression + '`',
-                    **dict(() if force_primary_key is None else [('in_key', sql_expression in force_primary_key)]))
+                    sql_expression='`%s`' % sql_expression,
+                    in_key=self.attributes[sql_expression].in_key or sql_expression in force_primary_key)
                  if sql_expression in self.names else
-                 dict(  # compute attribute
+                 dict(  # computed attributes
                      default_attribute_properties,
                      name=new_name,
                      sql_expression=sql_expression)

datajoint/settings.py

@@ -37,17 +37,13 @@
 
 default = OrderedDict({
     'database.host': 'localhost',
-    'database.password': 'datajoint',
-    'database.user': 'datajoint',
+    'database.password': None,
+    'database.user': None,
     'database.port': 3306,
-    'database.reconnect': False,
-    #
     'connection.init_function': None,
-    #
+    'database.reconnect': False,
     'loglevel': 'INFO',
-    #
     'safemode': True,
-    #
     'display.limit': 7,
     'display.width': 14
 })
@@ -63,7 +59,6 @@
 }
 
 
-
 class Config(collections.MutableMapping):
 
     instance = None
@@ -98,7 +93,6 @@ def __iter__(self):
     def __len__(self):
         return len(self.instance._conf)
 
-
     @contextmanager
     def __call__(self, **kwargs):
         """
@@ -107,17 +101,15 @@ def __call__(self, **kwargs):
         double underscore '__'. The context manager yields the changed config object.
 
         Example:
-
         >>> import datajoint as dj
-        >>> with dj.config(safe__mode=False) as cfg:
+        >>> with dj.config(safemode=False, database__host="localhost") as cfg:
         >>>     # do dangerous stuff here
-
         """
 
         try:
             backup = self.instance
             self.instance = Config.__Config(self.instance._conf)
-            new = {k.replace('__','.'):v for k,v in kwargs.items()}
+            new = {k.replace('__', '.'): v for k, v in kwargs.items()}
             self.instance._conf.update(new)
             yield self
         except:
@@ -126,15 +118,13 @@ def __call__(self, **kwargs):
         else:
             self.instance = backup
 
-
     class __Config:
         """
         Stores datajoint settings. Behaves like a dictionary, but applies validator functions
         when certain keys are set.
 
         The default parameters are stored in datajoint.settings.default . If a local config file
         exists, the settings specified in this file override the default settings.
-
         """
 
         def __init__(self, *args, **kwargs):
@@ -166,8 +156,7 @@ def save(self, filename=None):
         def load(self, filename):
             """
             Updates the setting from config file in JSON format.
-
-            :param filename=None: filename of the local JSON settings file. If None, the local config file is used.
+            :param filename: filename of the local JSON settings file. If None, the local config file is used.
             """
             if filename is None:
                 filename = LOCALCONFIG

tests/__init__.py

@@ -22,7 +22,7 @@
 CONN_INFO = dict(
     host=environ.get('DJ_TEST_HOST', 'localhost'),
     user=environ.get('DJ_TEST_USER', 'datajoint'),
-    passwd=environ.get('DJ_TEST_PASSWORD', 'datajoint'))
+    password=environ.get('DJ_TEST_PASSWORD', 'datajoint'))
 
 # Prefix for all databases used during testing
 PREFIX = environ.get('DJ_TEST_DB_PREFIX', 'djtest')

tests/schema.py

@@ -13,7 +13,7 @@
 
 @schema
 class Test(dj.Computed):
-    definition="""
+    definition = """
     key   :   int     # key
     ---
     value   :   int     # value

tests/schema_simple.py

@@ -11,6 +11,24 @@
 schema = dj.schema(PREFIX + '_relational', locals(), connection=dj.conn(**CONN_INFO))
 
 
+@schema
+class IJ(dj.Manual):
+    definition = """  # tests restrictions
+    i  : int
+    j  : int
+    """
+    contents = list(dict(i=i, j=j+2) for i in range(3) for j in range(3))
+
+
+@schema
+class JI(dj.Manual):
+    definition = """  # tests restrictions by relations when attributes are reordered
+    j  : int
+    i  : int
+    """
+    contents = list(dict(i=i+1, j=j) for i in range(3) for j in range(3))
+
+
 @schema
 class A(dj.Lookup):
     definition = """