cut-out tests

* move tests outside package
* update Expt. cls
* fix Ray test
* update docs
* fix pandas conf.

Author: Borda

.shippable.yml

@@ -40,7 +40,7 @@ install:
   - mkdir libs
 
   - pip install -r requirements.txt
-  - pip install -r requirements-dev.txt
+  - pip install -r ./tests/requirements.txt
   - pip list
 
 script:

.travis.yml

@@ -32,7 +32,7 @@ before_install:
 install:
   - mkdir libs
   - pip install -r requirements.txt
-  - pip install -r requirements-dev.txt
+  - pip install -r ./tests/requirements.txt
   - pip list
 
 before_script:
@@ -44,7 +44,7 @@ before_script:
 script:
   - python setup.py build_ext --inplace
   # - pytest imsegm -v --doctest-modules
-  - nosetests imsegm -v --exe --with-doctest --with-xunit --with-coverage --cover-package=imsegm
+  - nosetests imsegm tests -v --exe --with-doctest --with-xunit --with-coverage --cover-package=imsegm
   - flake8 . --ignore=E402,E731 --max-line-length=100
   - python setup.py install
 

MANIFEST.in

@@ -23,6 +23,8 @@ include setup.*
 exclude *.yml
 # Exclude testing
 exclude pytest.*
+recursive-exclude tests *
+exclude tests
 
 # Exclude experiments
 exclude */*.sh

README.md

@@ -178,7 +178,7 @@ We utilise (un)supervised segmentation according to given training examples or s
     ```bash
     python experiments_segmentation/run_segm_slic_model_graphcut.py \
        -l ./data_images/langerhans_islets/list_lang-isl_imgs-annot.csv -i "" \
-       --cdf experiments_segmentation/sample_config.yml \
+       -cfg experiments_segmentation/sample_config.yml \
        -o ./results -n langIsl --nb_classes 3 --visual --nb_workers 2
     ```
     OR specified on particular path:

circle.yml

@@ -9,7 +9,7 @@ references:
         sudo apt-get update
         sudo apt-get install tk-dev pkg-config python-dev python-tk
         sudo pip install --upgrade pip setuptools
-        sudo pip install -r requirements-dev.txt
+        sudo pip install -r ./tests/requirements.txt
         sudo pip install -U backports.functools_lru_cache  # required for matplotlib @py2
         pip install -r requirements.txt --user
         pip list
@@ -19,7 +19,7 @@ references:
       name: Testing and Formating
       command: |
         unset DISPLAY
-        coverage run --source imsegm -m py.test imsegm -v --doctest-modules --junitxml=test-reports/pytest_junit.xml
+        coverage run --source imsegm -m py.test imsegm tests -v --doctest-modules --junitxml=test-reports/pytest_junit.xml
         python setup.py check -m -s
         flake8 . --ignore=E402,E731 --max-line-length=100
 
@@ -38,7 +38,7 @@ references:
      command: |
        sudo apt-get install pandoc
        sudo pip install -r docs/requirements.txt
-       # sphinx-apidoc -o ./docs/source ./imsegm **/tests/* --force --follow-links
+       # sphinx-apidoc -o ./docs/source ./imsegm ./tests/* --force --follow-links
        cd docs; make html
 
 jobs:

docs/requirements.txt

@@ -2,4 +2,5 @@ sphinx>=1.4
 recommonmark  # fails with badges
 m2r  # fails with multi-line text
 nbsphinx
-pandoc
+pandoc
+docutils<0.15  # higher version breaks py2

docs/source/conf.py

@@ -238,7 +238,7 @@
 def run_apidoc(_):
     for pkg in PACKAGES:
         argv = ['-e', '-o', PATH_HERE, os.path.join(PATH_HERE, PATH_ROOT, pkg),
-                '**/tests/*', '--force']
+                'tests/*', '--force']
         try:
             # Sphinx 1.7+
             from sphinx.ext import apidoc

experiments_ovary_centres/run_center_candidate_training.py

@@ -28,7 +28,6 @@
 import sys
 import logging
 import argparse
-import multiprocessing as mproc
 from functools import partial
 
 import tqdm
@@ -73,7 +72,7 @@
 NAME_YAML_PARAMS = 'configuration.yaml'
 NAME_DUMP_TRAIN_DATA = 'dump_training_data.npz'
 
-NB_THREADS = max(1, int(mproc.cpu_count() * 0.9))
+NB_WORKERS = tl_expt.nb_workers(0.9)
 # position is label in loaded segm and nb are out labels
 LUT_ANNOT_CENTER_RELABEL = [0, 0, -1, 1]
 CROSS_VAL_LEAVE_OUT_SEARCH = 0.2
@@ -149,7 +148,7 @@ def arg_parse_params(params):
                         help='name of the experiment', default='ovary')
     parser.add_argument('-cfg', '--path_config', type=str, required=False,
                         help='path to the configuration', default=None)
-    parser.add_argument('--nb_workers', type=int, required=False, default=NB_THREADS,
+    parser.add_argument('--nb_workers', type=int, required=False, default=NB_WORKERS,
                         help='number of processes in parallel')
     params.update(vars(parser.parse_args()))
     paths = {}
@@ -445,7 +444,7 @@ def wrapper_draw_export_slic_centers(args):
     return export_show_image_points_labels(*args)
 
 
-def dataset_load_images_segms_compute_features(params, df_paths, nb_workers=NB_THREADS):
+def dataset_load_images_segms_compute_features(params, df_paths, nb_workers=NB_WORKERS):
     """ create whole dataset composed from loading input data, computing features
     and label points by label whether its positive or negative center candidate
 
@@ -502,7 +501,7 @@ def dataset_load_images_segms_compute_features(params, df_paths, nb_workers=NB_T
 
 
 def export_dataset_visual(path_output, dict_imgs, dict_segms, dict_slics,
-                          dict_points, dict_labels, nb_workers=NB_THREADS):
+                          dict_points, dict_labels, nb_workers=NB_WORKERS):
     """ visualise complete training dataset by marking labeld points
     over image and input segmentation
 

experiments_ovary_centres/run_center_clustering.py

@@ -9,7 +9,6 @@
 import os
 import sys
 import logging
-# import multiprocessing as mproc
 from functools import partial
 
 import pandas as pd

experiments_ovary_centres/run_center_evaluation.py

@@ -18,7 +18,6 @@
 import time
 import logging
 import gc
-# import multiprocessing as mproc
 from functools import partial
 
 import pandas as pd
@@ -83,7 +82,7 @@ def estimate_eggs_from_info(row_slice, mask_shape):
     pos_ant, pos_lat, pos_post = tl_visu.parse_annot_rectangles(row_slice)
     list_masks = tl_visu.draw_eggs_rectangle(mask_shape, pos_ant, pos_lat,
                                              pos_post)
-    mask_eggs = tl_visu.merge_object_masks(list_masks, thr_overlap=0.5)
+    mask_eggs = tl_visu.merge_object_masks(list_masks, overlap_thr=0.5)
 
     return mask_eggs