Store attachments as intact files (#659)

* attachments are saved as intact files

* reform attachments and filepath

* complete implementation of external storage except for `clean`

* refactor external storage

* complete refactoring of external storage for version 0.12
all tests pass

* rename attribute `basename` to `attachment_name` in external table

* add __repr__ to ExternalMapping

* external files are not copied if stage and store are the same

* make external tables require setting the `delete_external_files` argument

* update CHANGELOG

* fix Python 3.4 compatibility

* fix Python 3.4 and 3.5 compatibility

* dropped support for Python 3.4

* minor changes in error messages

* Update to pathlib in test init.

* Update test_blob_migrate to be compatible for WIN10.

* Fix WIN10 compatibility with KeyboardInterrupt and SystemExit exceptions.

* Fix WIN10 filepath to store as posix and fetch as user's platform.

* Fix relpath for Python3.5.

* Fix copytree for Python3.5.

* Fix typo.

* Fix for Python3.5.

* Update coveralls.

* Update coverall env vars.

* add environment variable DJ_SUPPORT_FILEPATH_MANAGEMENT to enable/disable the filepath datatype

* Update CHANGELOG.md

Co-Authored-By: guzman-raphael <38401847+guzman-raphael@users.noreply.github.com>

* Update CHANGELOG.md

Co-Authored-By: guzman-raphael <38401847+guzman-raphael@users.noreply.github.com>

Author: dimitri-yatsenko

.coveralls.yml

@@ -6,6 +6,8 @@ env:
     - COMPOSE_HTTP_TIMEOUT="300"
     - UID="2000"
     - GID="2000"
+    - COVERALLS_SERVICE_NAME="travis-ci"
+    - COVERALLS_REPO_TOKEN="fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY"
 services: 
 - docker
 main: &main
@@ -64,9 +66,3 @@ jobs:
       env:
       - PY_VER: "3.5"
       - MYSQL_VER: "5.6"
-    - stage: Coverage
-      os: linux
-      language: shell
-      script:
-        - sudo pip install python-coveralls
-        - coveralls

.travis.yml

@@ -1,11 +1,13 @@
 ## Release notes
 
 ### 0.12.0 -- October 1, 2019
+* Dropped support for Python 3.4
 * Support secure connections with TLS (aka SSL) PR #620
 * Convert numpy array from python object to appropriate data type if all elements are of the same type (#587) PR #608
 * Remove expression requirement to have additional attributes (#604) PR #604
-* Support for filepath datatype (#481) PR #603
-* Avoid creating multiple copies of attachments and return a dictionary array when specifying `as_dict=True` (#592, #595) PR #593
+* Support for filepath datatype (#481) PR #603, #659
+* Support file attachment datatype (#480, #592, #637) PR #659
+* Fetch return a dict array when specifying `as_dict=True` for specified attributes. (#595) PR #593
 * Support of ellipsis in `proj`:  `query_expression.proj(.., '-movie')` (#499) PR #578
 * Expand support of blob serialization (#572, #520, #427, #392, #244, #594) PR #577
 * Support for alter (#110) PR #573
@@ -16,10 +18,12 @@
 * `dj.ERD` is now called `dj.Diagram` (#255, #546) PR #565
 * `dj.Diagram` underlines "distinguished" classes (#378) PR #557
 * Accept alias for supported MySQL datatypes (#544) PR #545
-* Support for pandas and order by "KEY" (#459, #537, #538, #541) PR #534
-* Support file attachment datatype and configurable blob storage (#467, #475, #480, #497) PR #532
+* Support for pandas in `fetch` (#459, #537) PR #534
+* Support for ordering by "KEY" in `fetch` (#541) PR #534
+* Improved external storage - a migration script needed from version 0.11  (#467, #475, #480, #497) PR #532
 * Increase default display rows (#523) PR #526
-* Bugfixes (#521, #205, #279, #477, #570, #581, #597, #596, #618, #633, #643, #644, #647, #656)
+* Bugfixes (#521, #205, #279, #477, #570, #581, #597, #596, #618, #633, #643, #644, #647, #648, #650, #656)
+* Minor improvements (#538)
 
 ### 0.11.3 -- Jul 26, 2019
 * Fix incompatibility with pyparsing 2.4.1 (#629) PR #631

CHANGELOG.md

@@ -25,13 +25,16 @@ services:
       - PYTHON_USER=dja
       - JUPYTER_PASSWORD=datajoint
       - DISPLAY
+      - COVERALLS_SERVICE_NAME
+      - COVERALLS_REPO_TOKEN
     working_dir: /src
     command: >
       /bin/sh -c
        "
-        pip install --user nose nose-cov .;
+        pip install --user nose nose-cov coveralls .;
         pip freeze | grep datajoint;
         nosetests -vsw tests --with-coverage --cover-package=datajoint;
+        coveralls;
         # jupyter notebook;
        "
     # ports:

LNX-docker-compose.yml

@@ -1,26 +1,5 @@
 import re
-import os
-from .errors import DataJointError
-
-
-ADAPTED_TYPE_SWITCH = "DJ_SUPPORT_ADAPTED_TYPES"
-
-
-def _switch_adapated_types(on):
-    """
-    Enable (on=True) or disable (on=False) support for AttributeAdapter
-    """
-    if on:
-        os.environ[ADAPTED_TYPE_SWITCH] = "TRUE"
-    else:
-        del os.environ[ADAPTED_TYPE_SWITCH]
-
-
-def _support_adapted_types():
-    """
-    check if support for AttributeAdapter is enabled
-    """
-    return os.getenv(ADAPTED_TYPE_SWITCH, "FALSE").upper() == "TRUE"
+from .errors import DataJointError, _support_adapted_types
 
 
 class AttributeAdapter:

datajoint/attach.py

@@ -47,7 +47,6 @@ def translate_query_error(client_error, query):
         return errors.MissingTableError(client_error.args[1], query)
     if isinstance(client_error, client.err.InternalError) and client_error.args[0] == 1364:
         return errors.MissingAttributeError(*client_error.args[1:])
-    raise client_error
 
 
 logger = logging.getLogger(__name__)
@@ -187,7 +186,7 @@ def __execute_query(cursor, query, args, cursor_class, suppress_warnings):
                     warnings.simplefilter("ignore")
                 cursor.execute(query, args)
         except client_errors as err:
-            raise translate_query_error(err, query)
+            raise translate_query_error(err, query) from None
 
     def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconnect=None):
         """

datajoint/attribute_adapter.py

@@ -5,7 +5,7 @@
 import re
 import pyparsing as pp
 import logging
-from .errors import DataJointError
+from .errors import DataJointError, _support_filepath_types, FILEPATH_FEATURE_SWITCH
 from .attribute_adapter import get_adapter
 
 from .utils import OrderedDict
@@ -395,6 +395,12 @@ def substitute_special_type(match, category, foreign_key_sql, context):
     elif category == 'INTERNAL_ATTACH':
         match['type'] = 'LONGBLOB'
     elif category in EXTERNAL_TYPES:
+        if category == 'FILEPATH' and not _support_filepath_types():
+            raise DataJointError("""
+            The filepath data type is disabled until complete validation. 
+            To turn it on as experimental feature, set the environment variable 
+            {env} = TRUE or upgrade datajoint.
+            """.format(env=FILEPATH_FEATURE_SWITCH))
         match['store'] = match['type'].split('@', 1)[1]
         match['type'] = UUID_DATA_TYPE
         foreign_key_sql.append(

datajoint/connection.py

@@ -2,6 +2,8 @@
 Exception classes for the DataJoint library
 """
 
+import os
+
 
 # --- Top Level ---
 class DataJointError(Exception):
@@ -77,3 +79,43 @@ class MissingExternalFile(DataJointError):
     """
     Error raised when an external file managed by DataJoint is no longer accessible
     """
+
+
+# environment variables to control availability of experimental features
+
+ADAPTED_TYPE_SWITCH = "DJ_SUPPORT_ADAPTED_TYPES"
+FILEPATH_FEATURE_SWITCH = "DJ_SUPPORT_FILEPATH_MANAGEMENT"
+
+
+def _switch_adapted_types(on):
+    """
+    Enable (on=True) or disable (on=False) support for AttributeAdapter
+    """
+    if on:
+        os.environ[ADAPTED_TYPE_SWITCH] = "TRUE"
+    else:
+        del os.environ[ADAPTED_TYPE_SWITCH]
+
+
+def _support_adapted_types():
+    """
+    check if support for AttributeAdapter is enabled
+    """
+    return os.getenv(ADAPTED_TYPE_SWITCH, "FALSE").upper() == "TRUE"
+
+
+def _switch_filepath_types(on):
+    """
+    Enable (on=True) or disable (on=False) support for AttributeAdapter
+    """
+    if on:
+        os.environ[FILEPATH_FEATURE_SWITCH] = "TRUE"
+    else:
+        del os.environ[FILEPATH_FEATURE_SWITCH]
+
+
+def _support_filepath_types():
+    """
+    check if support for AttributeAdapter is enabled
+    """
+    return os.getenv(FILEPATH_FEATURE_SWITCH, "FALSE").upper() == "TRUE"