- added support in Python 3.9

- unigrams and bigrams files are now compressed and pickled to load
  faster and to save some space

Author: Gal Ben David

.github/workflows/build.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
       - name: Install latest rust
         uses: actions-rs/toolchain@v1
         with:
@@ -25,14 +25,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
         os: [ubuntu-latest , macos-latest, windows-latest]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Run image
@@ -46,5 +46,5 @@ jobs:
         run: poetry install
       - name: Build Python package
         run: poetry run maturin develop
-      - name: pytest
+      - name: Test
         run: poetry run pytest tests

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pywordsegment"
-version = "0.1.1"
+version = "0.1.2"
 authors = ["Gal Ben David <[email protected]>"]
 edition = "2018"
 description = "Concatenated-word segmentation Python library written in Rust"
@@ -14,9 +14,14 @@ keywords = ["word", "segment", "rust", "pyo3"]
 requires-python = ">=3.6"
 classifier = [
     "License :: OSI Approved :: MIT License",
+    "Operating System :: MacOS",
+    "Operating System :: Microsoft",
+    "Operating System :: POSIX :: Linux",
     "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Rust",
 ]
 
 [lib]

README.md

@@ -9,7 +9,7 @@
 
 
 ![license](https://img.shields.io/badge/MIT-License-blue)
-![Python](https://img.shields.io/badge/Python-3.6%20%7C%203.7%20%7C%203.8-blue)
+![Python](https://img.shields.io/badge/Python-3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9-blue)
 ![OS](https://img.shields.io/badge/OS-Mac%20%7C%20Linux%20%7C%20Windows-blue)
 ![Build](https://github.com/intsights/pywordsegment/workflows/Build/badge.svg)
 [![PyPi](https://img.shields.io/pypi/v/pywordsegment.svg)](https://pypi.org/project/pywordsegment/)

pyproject.toml

@@ -15,7 +15,7 @@ strip = true
 
 [tool.poetry]
 name = "pywordsegment"
-version = "0.1.1"
+version = "0.1.2"
 authors = ["Gal Ben David <[email protected]>"]
 description = "Concatenated-word segmentation Python library written in Rust"
 readme = "README.md"
@@ -36,7 +36,8 @@ classifiers = [
     "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
-    "Programming Language :: Rust"
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Rust",
 ]
 packages = [
     { include = "pywordsegment" },

pywordsegment/__init__.py

@@ -1,4 +1,6 @@
+import gzip
 import pathlib
+import pickle
 import typing
 
 from . import pywordsegment
@@ -11,9 +13,25 @@ def __init__(
         self,
     ) -> None:
         if WordSegmenter.word_segmenter is None:
+            current_file_dir = pathlib.Path(__file__).parent.absolute()
+
+            unigrams_file = current_file_dir.joinpath('unigrams.pkl.gz')
+            unigrams = pickle.load(
+                file=gzip.GzipFile(
+                    filename=str(unigrams_file),
+                ),
+            )
+
+            bigrams_file = current_file_dir.joinpath('bigrams.pkl.gz')
+            bigrams = pickle.load(
+                file=gzip.GzipFile(
+                    filename=str(bigrams_file),
+                ),
+            )
+
             WordSegmenter.word_segmenter = pywordsegment.WordSegmenter(
-                unigrams_file_path=str(pathlib.Path(__file__).parent.absolute().joinpath('unigrams.txt')),
-                bigrams_file_path=str(pathlib.Path(__file__).parent.absolute().joinpath('bigrams.txt')),
+                unigrams=unigrams,
+                bigrams=bigrams,
                 total_words_frequency=1024908267229.0,
             )