update Segm (#7)

 * add binary metrics
 * add segm options
 * add config params @segm
 * extend color features @segm
 * export npz GMM @segm
 * simplify expt. @segm
 * add illustration
 * swap orient. CC
 * update CI & style - codacy

Author: Borda

.shippable.yml

@@ -3,63 +3,68 @@
 # After changing this file, check it on:
 #   http://yaml-online-parser.appspot.com/
 
+build:
+  # https://github.com/Shippable/support/issues/3873
+  pre_ci_boot:
+      # image_name: drydock/u14pytall
+      image_tag: v5.8.2
+  ci:
+    - echo $CI_REPORTS && mkdir -p $CI_REPORTS
+    - echo $COVERAGE_REPORTS && mkdir -p $COVERAGE_REPORTS
+
+env:
+  global:
+    - CI_REPORTS=shippable/testresults
+    - COVERAGE_REPORTS=shippable/codecoverage
+
 language: python
 
 sudo: true
 
 python:
   - 2.7
-  - 3.4
+  - 3.5
 
 cache: pip
 
-env:
-  - CI_REPORTS=shippable/testresults COVERAGE_REPORTS=shippable/codecoverage
-
-build:
-  ci:
-    - echo $CI_REPORTS && mkdir -p $CI_REPORTS
-    - echo $COVERAGE_REPORTS && mkdir -p $COVERAGE_REPORTS
-
 before_install:
   - apt-get update # --fix-missing
   - apt-get install freetype*
   - apt-get install python-dev python-tk python3-tk
   - apt-get install pkg-config
-  - pip install --upgrade pip
+  - pip install --upgrade pip wheel
 
 install:
   - root=$PWD
-  - gcc --version
   - mkdir libs
 #  - cd $root/libs && git clone https://github.com/Borda/pyGCO.git
-#  - cd $root/libs/pyGCO && pip install -r requirements.txt
-#  - cd $root/libs/pyGCO && python setup.py install
-#  - rm -r -f $here/libs/pyGCO
-  - cd $root
+#  - cd $root/libs/pyGCO && pip install -r requirements.txt && python setup.py install
+#  - rm -r -f $root/libs/pyGCO
+#  - cd $root
 
   - pip install -r requirements.txt
-  - pip install nose>=1.3.7 coverage codecov pytest>=3.0.5
+  - pip install "nose>=1.3.7" coverage codecov "pytest>=3.0.5"
 
 script:
   - mkdir data && mkdir output && mkdir results
+  - gcc --version ; python --version ; pip --version ; pwd ; ls -l
   - python setup.py build_ext --inplace
 
   # - nosetests -v --with-xunit --xunit-file=$CI_REPORTS/nosetests.xml
   - nosetests -v --exe --with-doctest --with-xunit --with-coverage --cover-package=imsegm --xunit-file=$CI_REPORTS/nosetests.xml
 
   # ANNOTATION section
-  - python handling_annotations/run_image_color_quantization.py -imgs "data_images/drosophila_ovary_slice/segm_rgb/*.png"
-  - python handling_annotations/run_image_convert_label_color.py -imgs "data_images/drosophila_ovary_slice/segm/*.png" -out data_images/drosophila_ovary_slice/segm_rgb
-  - python handling_annotations/run_overlap_images_segms.py  -imgs "data_images/drosophila_ovary_slice/image/*.jpg" -segs data_images/drosophila_ovary_slice/segm -out results/overlap_ovary_segment
-  - python handling_annotations/run_segm_annot_inpaint.py -imgs "data_images/drosophila_ovary_slice/segm/*.png" --label 0
-  - python handling_annotations/run_segm_annot_relabel.py -imgs "data_images/drosophila_ovary_slice/center_levels/*.png" -out results/relabel_center_levels
+  - python handling_annotations/run_image_color_quantization.py -imgs "./data_images/drosophila_ovary_slice/segm_rgb/*.png"
+  - python handling_annotations/run_image_convert_label_color.py -imgs "./data_images/drosophila_ovary_slice/segm/*.png" -out ./data_images/drosophila_ovary_slice/segm_rgb
+  - python handling_annotations/run_overlap_images_segms.py  -imgs "./data_images/drosophila_ovary_slice/image/*.jpg" -segs ./data_images/drosophila_ovary_slice/segm -out ./results/overlap_ovary_segment
+  - python handling_annotations/run_segm_annot_inpaint.py -imgs "./data_images/drosophila_ovary_slice/segm/*.png" --label 0
+  - python handling_annotations/run_segm_annot_relabel.py -imgs "./data_images/drosophila_ovary_slice/center_levels/*.png" -out ./results/relabel_center_levels
 
   # SEGMENTATION section
   - rm -r -f results && mkdir results
   - python experiments_segmentation/run_compute_stat_annot_segm.py --visual
-  - python experiments_segmentation/run_segm_slic_model_graphcut.py --path_config experiments_segmentation/sample_config.json --nb_jobs 1
-  - python experiments_segmentation/run_segm_slic_classif_graphcut.py --path_config experiments_segmentation/sample_config.json --nb_jobs 1
+  - python experiments_segmentation/run_segm_slic_model_graphcut.py -i "data_images/drosophila_disc/image/img_[5,6].jpg" -cfg ./experiments_segmentation/sample_config.json --visual
+  - python experiments_segmentation/run_segm_slic_classif_graphcut.py -l data_images/drosophila_ovary_slice/list_imgs-annot-struct_short.csv -i "data_images/drosophila_ovary_slice/image/insitu41*.jpg" -cfg ./experiments_segmentation/sample_config.json --visual
 
   # CENTER DETECT. section
   - rm -r -f results && mkdir results
@@ -89,5 +94,5 @@ after_success:
   - python setup.py install
 
   - coverage xml -o $COVERAGE_REPORTS/coverage.xml
-  - codecov -t 80efed4e-ac2b-4fea-a642-0a8b1c82e1c8
+  - codecov  # public repository on Travis CI
   - coverage report

.travis.yml

@@ -15,30 +15,31 @@ sudo: false
 
 python:
   - 2.7
-  - 3.4
+  # - 3.4  # will be deprecated for pandas
   - 3.5
   - 3.6
+  # - "3.7-dev"
 
 # See http://docs.travis-ci.com/user/caching/#pip-cache
 cache: pip
 
 before_install:
   - pip install --upgrade pip
-  - mkdir libs
 
 install:
-  - root=$PWD
+  - mkdir libs
+#  - root=$PWD
 #  - cd $root/libs && git clone https://github.com/Borda/pyGCO.git
-#  - cd $root/libs/pyGCO && pip install -r requirements.txt
-#  - cd $root/libs/pyGCO && python setup.py install
-#  - rm -r -f $here/libs/pyGCO
-  - cd $root
+#  - cd $root/libs/pyGCO && pip install -r requirements.txt && python setup.py install
+#  - rm -r -f $root/libs/pyGCO
+#  - cd $root
 
   - pip install -r requirements.txt
-  - pip install nose>=1.3.7 coverage codecov pytest>=3.0.5 codacy-coverage
+  - pip install "nose>=1.3.7" coverage codecov "pytest>=3.0.5" codacy-coverage
 
 before_script:
   - mkdir data && mkdir output && mkdir results
+  - gcc --version ; python --version ; pip --version ; pwd ; ls -l
   - python setup.py build_ext --inplace
 
 script:
@@ -47,9 +48,9 @@ script:
   - python setup.py install
 
 after_success:
-  # - codecov  # public repository on Travis CI
+  - codecov  # public repository on Travis CI
   # private repository on Travis CI
-  - codecov -t 80efed4e-ac2b-4fea-a642-0a8b1c82e1c8
+  # - codecov -t 80efed4e-ac2b-4fea-a642-0a8b1c82e1c8
   - coverage xml
   - python-codacy-coverage -r coverage.xml
   - coverage report

README.md

@@ -121,81 +121,98 @@ We introduce some useful tools for work with image annotation and segmentation.
 * **Quantization:** in case you have some smooth color labeling in your images you can remove them with following quantization script.
     ```bash
     python handling_annotations/run_image_color_quantization.py \
-        -imgs "data_images/drosophila_ovary_slice/segm_rgb/*.png" \
+        -imgs "./data_images/drosophila_ovary_slice/segm_rgb/*.png" \
         -m position -thr 0.01 --nb_jobs 2
     ```
 * **Paint labels:** concerting image labels into colour space and other way around.
     ```bash
     python handling_annotations/run_image_convert_label_color.py \
-        -imgs "data_images/drosophila_ovary_slice/segm/*.png" \
-        -out data_images/drosophila_ovary_slice/segm_rgb
+        -imgs "./data_images/drosophila_ovary_slice/segm/*.png" \
+        -out ./data_images/drosophila_ovary_slice/segm_rgb
     ```
 * **Visualisation:** having input image and its segmentation we can use simple visualisation which overlap the segmentation over input image. 
     ```bash
     python handling_annotations/run_overlap_images_segms.py \
-        -imgs "data_images/drosophila_ovary_slice/image/*.jpg" \
-        -segs data_images/drosophila_ovary_slice/segm \
-        -out results/overlap_ovary_segment
+        -imgs "./data_images/drosophila_ovary_slice/image/*.jpg" \
+        -segs ./data_images/drosophila_ovary_slice/segm \
+        -out ./results/overlap_ovary_segment
     ```
 * **Inpainting** selected labels in segmentation.
     ```bash
     python handling_annotations/run_segm_annot_inpaint.py \
-        -imgs "data_images/drosophila_ovary_slice/segm/*.png" \
+        -imgs "./data_images/drosophila_ovary_slice/segm/*.png" \
         --label 4
     ```
 * **Replace labels:** change labels in input segmentation into another set of lables in 1:1 schema.
     ```bash
     python handling_annotations/run_segm_annot_relabel.py \
-        -out results/relabel_center_levels \
+        -out ./results/relabel_center_levels \
         --label_old 2 3 --label_new 1 1 
     ```
 
 
 ### Structure segmentation
 
 We utilize (un)supervised segmentation according to given training examples or some expectations.
+![vusial debug](figures/visual_img_43_debug.jpg)
 
 * Evaluate superpixels (with given SLIC parameters) quality against given segmentation. It helps find out best SLIC configuration.
     ```bash
     python experiments_segmentation/run_eval_superpixels.py \
-        -imgs "data_images/drosophila_ovary_slice/image/*.jpg" \
-        -segm "data_images/drosophila_ovary_slice/annot_eggs/*.png" \
+        -imgs "./data_images/drosophila_ovary_slice/image/*.jpg" \
+        -segm "./data_images/drosophila_ovary_slice/annot_eggs/*.png" \
         --img_type 2d_split \
-        --slic_size 20 --slic_regul 0.25 --slico 0
+        --slic_size 20 --slic_regul 0.25 --slico
     ```
 * Perform **Unsupervised** segmentation in images given in CSV
     ```bash
     python experiments_segmentation/run_segm_slic_model_graphcut.py \
-       -l data_images/langerhans_islets/list_lang-isl_imgs-annot.csv -i "" \
-       --path_config experiments_segmentation/sample_config.json \
-       -o results -n langIsl --nb_classes 3 --visual --nb_jobs 2
+       -l ./data_images/langerhans_islets/list_lang-isl_imgs-annot.csv -i "" \
+       --cdf experiments_segmentation/sample_config.json \
+       -o ./results -n langIsl --nb_classes 3 --visual --nb_jobs 2
     ```
-    OR specified on particuler path:
+    OR specified on particular path:
     ```bash
     python experiments_segmentation/run_segm_slic_model_graphcut.py \
-       -l "" -i "data_images/langerhans_islets/image/*.jpg" \
-       --path_config experiments_segmentation/sample_config.json \
-       -o results -n langIsl --nb_classes 3 --visual --nb_jobs 2
+       -l "" -i "./data_images/langerhans_islets/image/*.jpg" \
+       -cfg ./experiments_segmentation/sample_config.json \
+       -o ./results -n langIsl --nb_classes 3 --visual --nb_jobs 2
     ```
+    ![unsupervised](figures/imag-disk-20_gmm.jpg)
 * Perform **Supervised** segmentation with afterwards evaluation.
     ```bash
     python experiments_segmentation/run_segm_slic_classif_graphcut.py \
-        -l data_images/drosophila_ovary_slice/list_imgs-annot-struct.csv \
-        -i "data_images/drosophila_ovary_slice/image/*.jpg" \
-       --path_config experiments_segmentation/sample_config.json \
-        -o results -n Ovary --img_type 2d_split --visual --nb_jobs 2
+        -l ./data_images/drosophila_ovary_slice/list_imgs-annot-struct.csv \
+        -i "./data_images/drosophila_ovary_slice/image/*.jpg" \
+        --path_config ./experiments_segmentation/sample_config.json \
+        -o ./results -n Ovary --img_type 2d_split --visual --nb_jobs 2
     ```
+    ![supervised](figures/imag-disk-20_train.jpg)
 * For both experiment you can evaluate segmentation results.
     ```bash
     python experiments_segmentation/run_compute-stat_annot-segm.py \
-        -annot "data_images/drosophila_ovary_slice/annot_struct/*.png" \
-        -segm "results/experiment_segm-supervise_ovary/*.png" \
-        -img "data_images/drosophila_ovary_slice/image/*.jpg" \
-        -out results/evaluation
+        -annot "./data_images/drosophila_ovary_slice/annot_struct/*.png" \
+        -segm "./results/experiment_segm-supervise_ovary/*.png" \
+        -img "./data_images/drosophila_ovary_slice/image/*.jpg" \
+        -out ./results/evaluation
     ```
-
-![unsupervised](figures/imag-disk-20_gmm.jpg)
-![supervised](figures/imag-disk-20_train.jpg)
+    ![vusial](figures/segm-visual_D03_sy04_100x.jpg)
+
+The previous two (un)segmentation accept [configuration file](experiments_segmentation/sample_config.json) (JSON) by parameter `-cfg` with some extra parameters which was not passed in arguments, for instance:
+```json
+{
+	"slic_size": 35,
+	"slic_regul": 0.2,
+	"features": {"color_hsv": ["mean", "std", "eng"]},
+	"classif": "SVM",
+	"nb_classif_search": 150,
+	"gc_edge_type": "model",
+	"gc_regul": 3.0,
+	"run_LOO": false,
+	"run_LPO": true,
+	"cross_val": 0.1
+}
+```
 
 ### Center detection and ellipse fitting
 
@@ -210,18 +227,18 @@ In general, the input is a formatted list (CSV file) of input images and annotat
 1. With zone annotation, we train a classifier for center candidate prediction. The annotation can be a CSV file with annotated centers as points, and the zone of positive examples is set uniformly as the circular neighborhood around these points. Another way (preferable) is to use annotated image with marked zones for positive, negative and neutral examples.
     ```bash
     python experiments_ovary_centres/run_center_candidate_training.py -list none \
-        -segs "data_images/drosophila_ovary_slice/segm/*.png" \
-        -imgs "data_images/drosophila_ovary_slice/image/*.jpg" \
-        -centers "data_images/drosophila_ovary_slice/center_levels/*.png" \
-        -out results -n ovary
+        -segs "./data_images/drosophila_ovary_slice/segm/*.png" \
+        -imgs "./data_images/drosophila_ovary_slice/image/*.jpg" \
+        -centers "./data_images/drosophila_ovary_slice/center_levels/*.png" \
+        -out ./results -n ovary
     ```
 1. Having trained classifier we perfom center prediction composed from two steps: i. center candidate clustering and candidate clustering.
     ```bash
     python experiments_ovary_centres/run_center_prediction.py -list none \
-        -segs "data_images/drosophila_ovary_slice/segm/*.png" \
-        -imgs "data_images/drosophila_ovary_slice/image/*.jpg" \
-        -centers results/detect-centers-train_ovary/classifier_RandForest.pkl \
-        -out results -n ovary
+        -segs "./data_images/drosophila_ovary_slice/segm/*.png" \
+        -imgs "./data_images/drosophila_ovary_slice/image/*.jpg" \
+        -centers ./results/detect-centers-train_ovary/classifier_RandForest.pkl \
+        -out ./results -n ovary
     ```
 1. Assuming you have an expert annotation you can compute static such as missed eggs.
     ```bash
@@ -271,13 +288,13 @@ python setup.py install
     ```bash
     python experiments_ovary_detect/run_RG2Sp_estim_shape-models.py  \
         -annot "~/Medical-drosophila/egg_segmentation/mask_2d_slice_complete_ind_egg/*.png" \
-        -out data -nb 15
+        -out ./data_images -nb 15
     ```
 1. Run several segmentation techniques on each image.
     ```bash
     python experiments_ovary_detect/run_ovary_egg-segmentation.py  \
-        -list data_images/drosophila_ovary_slice/list_imgs-segm-center-points.csv \
-        -out output -n ovary_image --nb_jobs 1 \
+        -list ./data_images/drosophila_ovary_slice/list_imgs-segm-center-points.csv \
+        -out ./results -n ovary_image --nb_jobs 1 \
         -m ellipse_moments \
            ellipse_ransac_mmt \
            ellipse_ransac_crit \
@@ -289,16 +306,16 @@ python setup.py install
            rg2sp_GC-mixture \
            watershed_morph
     ```
-1. Evaluate your segmentation results to expert annotation.
+1. Evaluate your segmentation ./results to expert annotation.
     ```bash
     python experiments_ovary_detect/run_ovary_segm_evaluation.py --visual
     ```
 1. In the end, cut individual segmented objects comes as minimal bounding box.
     ```bash
     python experiments_ovary_detect/run_cut_segmented_objects.py \
-        -annot "data_images/drosophila_ovary_slice/annot_eggs/*.png" \
-        -img "data_images/drosophila_ovary_slice/segm/*.png" \
-        -out results/cut_images --padding 50
+        -annot "./data_images/drosophila_ovary_slice/annot_eggs/*.png" \
+        -img "./data_images/drosophila_ovary_slice/segm/*.png" \
+        -out ./results/cut_images --padding 50
     ```
 1. Finally, performing visualisation of segmentation results toghter with expert annotation.
     ```bash