Merge pull request #92 from pettarin/devel

readbeyond · web-flow · commit d30ad360c6fa · 2016-07-24T19:17:29.000+02:00
Added exact version requirements for lxml and bs4
diff --git a/README.md b/README.md
@@ -228,7 +228,7 @@ which explains how to use the built-in command line tools.
 * Arbitrary text fragment granularity (single word, subphrase, phrase, paragraph, etc.)
 * Input audio file formats: all those readable by `ffmpeg`
 * Output sync map formats: AUD, CSV, EAF, JSON, SMIL, SRT, SSV, SUB, TSV, TTML, TXT, VTT, XML
-* Tested languages: ARA, BUL, CAT, CYM, CES, DAN, DEU, ELL, ENG, EPO, EST, FAS, FIN, FRA, GLE, GRC, HRV, HUN, ISL, ITA, JPN, LAT, LAV, LIT, NLD, NOR, RON, RUS, POL, POR, SLK, SPA, SRP, SWA, SWE, TUR, UKR
+* Confirmed working on languages: ARA, BUL, CAT, CYM, CES, DAN, DEU, ELL, ENG, EPO, EST, FAS, FIN, FRA, GLE, GRC, HRV, HUN, ISL, ITA, JPN, LAT, LAV, LIT, NLD, NOR, RON, RUS, POL, POR, SLK, SPA, SRP, SWA, SWE, TUR, UKR
 * MFCC and DTW computed via Python C extensions to reduce the processing time
 * Several built-in TTS engine wrappers: eSpeak (default, FLOSS), Festival (FLOSS), Nuance TTS API (commercial)
 * Default TTS (eSpeak) called via a Python C extension for fast audio synthesis
@@ -242,6 +242,7 @@ which explains how to use the built-in command line tools.
 * Output an HTML file for fine tuning the sync map manually (`finetuneas` project)
 * Execution parameters tunable at runtime
 * Code suitable for Web app deployment (e.g., on-demand cloud computing)
+* Extensive test suite including 898 unit/integration/performance tests, that run and must pass before each release
 
 
 ## Limitations and Missing Features 
diff --git a/README.rst b/README.rst
@@ -235,9 +235,10 @@ Supported Features
 -  Input audio file formats: all those readable by ``ffmpeg``
 -  Output sync map formats: AUD, CSV, EAF, JSON, SMIL, SRT, SSV, SUB,
    TSV, TTML, TXT, VTT, XML
--  Tested languages: ARA, BUL, CAT, CYM, CES, DAN, DEU, ELL, ENG, EPO,
-   EST, FAS, FIN, FRA, GLE, GRC, HRV, HUN, ISL, ITA, JPN, LAT, LAV, LIT,
-   NLD, NOR, RON, RUS, POL, POR, SLK, SPA, SRP, SWA, SWE, TUR, UKR
+-  Confirmed working on languages: ARA, BUL, CAT, CYM, CES, DAN, DEU,
+   ELL, ENG, EPO, EST, FAS, FIN, FRA, GLE, GRC, HRV, HUN, ISL, ITA, JPN,
+   LAT, LAV, LIT, NLD, NOR, RON, RUS, POL, POR, SLK, SPA, SRP, SWA, SWE,
+   TUR, UKR
 -  MFCC and DTW computed via Python C extensions to reduce the
    processing time
 -  Several built-in TTS engine wrappers: eSpeak (default, FLOSS),
@@ -260,6 +261,8 @@ Supported Features
 -  Execution parameters tunable at runtime
 -  Code suitable for Web app deployment (e.g., on-demand cloud
    computing)
+-  Extensive test suite including 898 unit/integration/performance
+   tests, that run and must pass before each release
 
 Limitations and Missing Features
 --------------------------------
diff --git a/aeneas/runtimeconfiguration.py b/aeneas/runtimeconfiguration.py
@@ -59,7 +59,7 @@ class RuntimeConfiguration(Configuration):
     Otherwise, use pure Python code.
 
     This option is equivalent to
-    setting ``CDTW``, ``CEW`` and ``CMFCC`` at once.
+    setting ``CDTW``, ``CEW``, and ``CMFCC`` to ``True`` at once.
 
     Default: ``True``.
 
diff --git a/aeneas/tests/tool_test_execute_job.py b/aeneas/tests/tool_test_execute_job.py
@@ -42,6 +42,27 @@ def test_exec_container_pure(self):
             ("", "-r=\"c_extensions=False\"")
         ], 0)
 
+    def test_exec_container_no_cew(self):
+        self.execute([
+            ("in", "../tools/res/job.zip"),
+            ("out", ""),
+            ("", "-r=\"cew=False\"")
+        ], 0)
+
+    def test_exec_container_no_cmfcc(self):
+        self.execute([
+            ("in", "../tools/res/job.zip"),
+            ("out", ""),
+            ("", "-r=\"cmfcc=False\"")
+        ], 0)
+
+    def test_exec_container_no_cdtw(self):
+        self.execute([
+            ("in", "../tools/res/job.zip"),
+            ("out", ""),
+            ("", "-r=\"cdtw=False\"")
+        ], 0)
+
     def test_exec_container_cew_subprocess_flag(self):
         self.execute([
             ("in", "../tools/res/job.zip"),
diff --git a/aeneas/tests/tool_test_execute_task.py b/aeneas/tests/tool_test_execute_task.py
@@ -107,6 +107,33 @@ def test_exec_srt_pure(self):
             ("", "-r=\"c_extensions=False\"")
         ], 0)
 
+    def test_exec_srt_no_cew(self):
+        self.execute([
+            ("in", "../tools/res/audio.mp3"),
+            ("in", "../tools/res/subtitles.txt"),
+            ("", "task_language=eng|is_text_type=subtitles|os_task_file_format=srt"),
+            ("out", "sonnet.srt"),
+            ("", "-r=\"cew=False\"")
+        ], 0)
+
+    def test_exec_srt_no_cmfcc(self):
+        self.execute([
+            ("in", "../tools/res/audio.mp3"),
+            ("in", "../tools/res/subtitles.txt"),
+            ("", "task_language=eng|is_text_type=subtitles|os_task_file_format=srt"),
+            ("out", "sonnet.srt"),
+            ("", "-r=\"cmfcc=False\"")
+        ], 0)
+
+    def test_exec_srt_no_cdtw(self):
+        self.execute([
+            ("in", "../tools/res/audio.mp3"),
+            ("in", "../tools/res/subtitles.txt"),
+            ("", "task_language=eng|is_text_type=subtitles|os_task_file_format=srt"),
+            ("out", "sonnet.srt"),
+            ("", "-r=\"cdtw=False\"")
+        ], 0)
+
     def test_exec_srt_cew_subprocess(self):
         self.execute([
             ("in", "../tools/res/audio.mp3"),
diff --git a/aeneas/tests/tool_test_extract_mfcc.py b/aeneas/tests/tool_test_extract_mfcc.py
@@ -48,6 +48,13 @@ def test_extract_pure(self):
             ("", "-r=\"c_extensions=False\"")
         ], 0)
 
+    def test_extract_no_cmfcc(self):
+        self.execute([
+            ("in", "../tools/res/audio.wav"),
+            ("out", "audio.wav.mfcc.txt"),
+            ("", "-r=\"cmfcc=False\"")
+        ], 0)
+
     def test_extract_mfcc_filters(self):
         self.execute([
             ("in", "../tools/res/audio.wav"),
diff --git a/aeneas/tests/tool_test_run_sd.py b/aeneas/tests/tool_test_run_sd.py
@@ -186,6 +186,33 @@ def test_sd_pure(self):
             ("", "-r=\"c_extensions=False\"")
         ], 0)
 
+    def test_sd_no_cew(self):
+        self.execute([
+            ("", "list"),
+            ("", "From|fairest|creatures|we|desire|increase"),
+            ("", "eng"),
+            ("in", "../tools/res/audio.mp3"),
+            ("", "-r=\"cew=False\"")
+        ], 0)
+
+    def test_sd_no_cmfcc(self):
+        self.execute([
+            ("", "list"),
+            ("", "From|fairest|creatures|we|desire|increase"),
+            ("", "eng"),
+            ("in", "../tools/res/audio.mp3"),
+            ("", "-r=\"cmfcc=False\"")
+        ], 0)
+
+    def test_sd_no_cdtw(self):
+        self.execute([
+            ("", "list"),
+            ("", "From|fairest|creatures|we|desire|increase"),
+            ("", "eng"),
+            ("in", "../tools/res/audio.mp3"),
+            ("", "-r=\"cdtw=False\"")
+        ], 0)
+
 
 
 if __name__ == '__main__':
diff --git a/aeneas/tests/tool_test_run_vad.py b/aeneas/tests/tool_test_run_vad.py
@@ -58,6 +58,14 @@ def test_run_pure(self):
             ("", "-r=\"c_extensions=False\"")
         ], 0)
 
+    def test_run_no_cmfcc(self):
+        self.execute([
+            ("in", "../tools/res/audio.mp3"),
+            ("", "both"),
+            ("out", "both.txt"),
+            ("", "-r=\"cmfcc=False\"")
+        ], 0)
+
     def test_run_extend_after(self):
         self.execute([
             ("in", "../tools/res/audio.mp3"),
diff --git a/aeneas/tests/tool_test_synthesize_text.py b/aeneas/tests/tool_test_synthesize_text.py
@@ -209,6 +209,15 @@ def test_synt_plain_pure(self):
             ("", "-r=\"c_extensions=False\"")
         ], 0)
 
+    def test_synt_plain_no_cew(self):
+        self.execute([
+            ("", "plain"),
+            ("in", "../tools/res/plain.txt"),
+            ("", "eng"),
+            ("out", "synthesized.wav"),
+            ("", "-r=\"cew=False\"")
+        ], 0)
+
     def test_synt_plain_cew_subprocess(self):
         self.execute([
             ("", "plain"),
diff --git a/debian/changelog b/debian/changelog
@@ -7,7 +7,7 @@ aeneas (1.5.1) stable; urgency=medium
   * Extension cew compiled for Mac OS X and Windows
   * Added links to installers for Mac OS X and Windows in the documentation
 
- -- alberto <alberto@nievo>  Mon, 18 Jul 2016 10:21:51 +0200
+ -- alberto <alberto@nievo>  Mon, 25 Jul 2016 10:21:51 +0200
 
 aeneas (1.5.0) stable; urgency=medium
 
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
@@ -17,6 +17,7 @@ v1.5.1 (2016-07-25)
 #. Fixed a numerical issue in ``dtw.py`` by explicit stating ``dtype=int`` in ``centers`` initializer, pure Python code only
 #. Extension ``cew`` compiled for Mac OS X and Windows
 #. Added links to installers for Mac OS X and Windows in the documentation
+#. Explicitly requiring lxml v3.6.0 and BeautifulSoup4 v4.4.1 due to a change in BeautifulSoup4 v4.5.0 API (to be investigated later)
 
 v1.5.0.3 (2016-04-23)
 ---------------------
diff --git a/docs/source/clitutorial.rst b/docs/source/clitutorial.rst
@@ -366,6 +366,10 @@ Examples:
 
     python -m aeneas.tools.execute_task --example-json -r="c_extensions=False"
 
+#. disable only the ``cew`` Python C extension, while ``cdtw`` and ``cmfcc`` will still run (if compiled)::
+
+    python -m aeneas.tools.execute_task --example-json -r="cew=False"
+
 #. set the DTW margin to ``10.000`` seconds::
 
     python -m aeneas.tools.execute_task --example-json -r="dtw_margin=10"
diff --git a/docs/source/libtutorial.rst b/docs/source/libtutorial.rst
@@ -9,7 +9,7 @@ Overview
 Although a majority of ``aeneas`` users work with the built-in command line tools,
 ``aeneas`` is primarily designed for being used as a Python library.
 Even the ``aeneas.tools`` can be used programmatically,
-thanks to their standardize I/O interface.
+thanks to their standard I/O interface.
 
 .. Topic:: Example
 
@@ -181,9 +181,11 @@ The Python C extensions included in ``aeneas`` are:
 
 .. note::
     
-    Currently :mod:`aeneas.cew` is available only on Linux,
-    since compiling it on Mac OS X and Windows cannot be automated.
-    However, starting from v1.5.0, the pure Python code
+    Currently :mod:`aeneas.cew` is available on Linux, Mac OS X, and Windows.
+    On Windows 64 bit it does not seem to work, probably because
+    eSpeak is available only as a 32 bit program/library,
+    and hence ``aeneas`` will fall back to run the pure Python code.
+    Starting with v1.5.0, the pure Python code
     for synthesizing text with eSpeak via ``subprocess``
     is only 2-3 times slower than :mod:`aeneas.cew`.
     Unless you work with thousands of text fragments,
@@ -192,6 +194,7 @@ The Python C extensions included in ``aeneas`` are:
 .. note::
     
     Currently :mod:`aeneas.cwave` is not used.
+    It will be enabled in a future version of ``aeneas``.
     
 
 
diff --git a/install_dependencies.sh b/install_dependencies.sh
@@ -11,45 +11,53 @@
 # __email__ = "aeneas@readbeyond.it"
 # __status__ = "Production"
 
-DEB="deb-multimedia-keyring_2015.6.1_all.deb"
-
-echo "[INFO] A.2 Downloading and installing deb-multimedia keyring..."
-wget -t 5 "http://www.deb-multimedia.org/pool/main/d/deb-multimedia-keyring/$DEB"
-if ! [ -e "$DEB" ]
-then
-    echo "[ERRO] Cannot download the deb-multimedia keyring."
-    echo "[ERRO] This might be due to a temporary network or server problem."
-    echo "[ERRO] Please retry installing this Vagrant box later."
-    echo "[ERRO] If the problem persists, please send an email to aeneas@readbeyond.it"
-    exit 1
-fi
-dpkg -i "$DEB"
-rm "$DEB"
-echo "[INFO] A.2 Downloading and installing deb-multimedia keyring... done"
-
-echo "[INFO] A.3 Adding deb-multimedia to apt sources..."
-sh -c 'echo "" >> /etc/apt/sources.list'
-sh -c 'echo "deb http://www.deb-multimedia.org jessie main" >> /etc/apt/sources.list'
-sh -c 'echo "" >> /etc/apt/sources.list'
-echo "[INFO] A.3 Adding deb-multimedia to apt sources... done"
-
-echo "[INFO] A.4 Updating apt..."
-apt-get update
-echo "[INFO] A.4 Updating apt... done"
-
-echo "[INFO] B.1 Installing ffmpeg (from deb-multimedia)..."
-apt-get install -y ffmpeg
-echo "[INFO] B.1 Installing ffmpeg (from deb-multimedia)... done"
-
-echo "[INFO] B.2 Installing espeak..."
-apt-get install -y espeak espeak-data libespeak1 libespeak-dev
-echo "[INFO] B.2 Installing espeak... done"
-
-echo "[INFO] B.3 Installing common libs using apt-get..."
-apt-get install -y build-essential
-apt-get install -y flac libasound2-dev libsndfile1-dev vorbis-tools
-apt-get install -y libxml2-dev libxslt-dev
-apt-get install -y python-dev python-pip
-echo "[INFO] B.3 Installing common libs using apt-get... done"
+# DISABLED
+    #DEB="deb-multimedia-keyring_2015.6.1_all.deb"
+
+    # DISABLED
+        #echo "[INFO] A.2 Downloading and installing deb-multimedia keyring..."
+        #wget -t 5 "http://www.deb-multimedia.org/pool/main/d/deb-multimedia-keyring/$DEB"
+        #if ! [ -e "$DEB" ]
+        #then
+        #    echo "[ERRO] Cannot download the deb-multimedia keyring."
+        #    echo "[ERRO] This might be due to a temporary network or server problem."
+        #    echo "[ERRO] Please retry installing this Vagrant box later."
+        #    echo "[ERRO] If the problem persists, please send an email to aeneas@readbeyond.it"
+        #    exit 1
+        #fi
+        #dpkg -i "$DEB"
+        #rm "$DEB"
+        #echo "[INFO] A.2 Downloading and installing deb-multimedia keyring... done"
+
+    echo "[INFO] A.1 Adding deb-multimedia to apt sources..."
+    sudo sh -c 'echo "deb http://www.deb-multimedia.org jessie main" > /etc/apt/sources.list.d/deb-multimedia.list'
+    echo "[INFO] A.1 Adding deb-multimedia to apt sources... done"
+
+    echo "[INFO] A.2 Updating apt..."
+    sudo apt-get update
+    echo "[INFO] A.2 Updating apt... done"
+
+    echo "[INFO] A.3 Downloading and installing deb-multimedia keyring..."
+    sudo apt-get install -y --force-yes deb-multimedia-keyring
+    echo "[INFO] A.3 Downloading and installing deb-multimedia keyring... done"
+
+    echo "[INFO] A.4 Updating apt..."
+    sudo apt-get update
+    echo "[INFO] A.4 Updating apt... done"
+
+    echo "[INFO] B.1 Installing ffmpeg (from deb-multimedia)..."
+    sudo apt-get install -y --force-yes ffmpeg
+    echo "[INFO] B.1 Installing ffmpeg (from deb-multimedia)... done"
+
+    echo "[INFO] B.2 Installing espeak..."
+    sudo apt-get install -y espeak espeak-data libespeak1 libespeak-dev
+    echo "[INFO] B.2 Installing espeak... done"
+
+    echo "[INFO] B.3 Installing common libs using apt-get..."
+    sudo apt-get install -y build-essential
+    sudo apt-get install -y flac libasound2-dev libsndfile1-dev vorbis-tools
+    sudo apt-get install -y libxml2-dev libxslt-dev zlib1g-dev
+    sudo apt-get install -y python-dev python-pip
+    echo "[INFO] B.3 Installing common libs using apt-get... done"
 
 echo "[INFO] Congratulations, now you can use aeneas!"
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,3 @@
-BeautifulSoup4>=4.4
-lxml>=3.0
+BeautifulSoup4==4.4.1
+lxml==3.6.0
 numpy>=1.9
diff --git a/setup.py b/setup.py
diff --git a/wiki/INSTALL.md b/wiki/INSTALL.md
diff --git a/wiki/miniconda.png b/wiki/miniconda.png
diff --git a/wiki/path.png b/wiki/path.png
diff --git a/wiki/zip.png b/wiki/zip.png
