Merge pull request #95 from snipsco/release/0.13.0

Release 0.13.0

CHANGELOG.md

@@ -1,7 +1,18 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
-## [Unreleased]
+## [0.13.0] - 2018-07-25
+### Fixed
+- Crash while computing metrics when either actual or predicted intent is unknown
+
+### Removed
+- APIs depending implicitely on Snips NLU: 
+    - `compute_cross_val_nlu_metrics`
+    - `compute_train_test_nlu_metrics`
+
+### Changed
+- Use flexible version specifiers for dependencies
+
 
 ## [0.12.0] - 2018-03-29
 ### Added
@@ -10,5 +21,6 @@ All notable changes to this project will be documented in this file.
 - New option to exclude slot metrics in the output
 - Samples
 
-[Unreleased]: https://github.com/snipsco/snips-nlu-metrics/compare/0.12.0...HEAD
+
+[0.13.0]: https://github.com/snipsco/snips-nlu-metrics/compare/0.12.0...0.13.0
 [0.12.0]: https://github.com/snipsco/snips-nlu-metrics/compare/0.11.1...0.12.0

README.rst

@@ -22,7 +22,7 @@ Install
 
 .. code-block:: console
 
-    pip install snips_nlu_metrics
+    $ pip install snips_nlu_metrics
 
 
 NLU Metrics API
@@ -40,24 +40,64 @@ The metrics output (json) provides detailed information about:
 * parsing errors
 * `confusion matrix`_
 
+Data
+----
+
+Some sample datasets, that can be used to compute metrics, are available
+`here <samples/>`_. Alternatively, you can create your own dataset either by
+using ``snips-nlu``'s `dataset generation tool`_ or by going on the
+`Snips console`_.
+
 Examples
 --------
 
-The Snips NLU metrics library can be used either with `Snips NLU`_ or with a
-custom intent parsing pipeline.
+The Snips NLU metrics library can be used with any NLU pipeline which satisfies
+the ``Engine`` API:
+
+.. code-block:: python
+
+    from builtins import object
+
+    class Engine(object):
+        def fit(self, dataset):
+            # Perform training ...
+            return self
+
+        def parse(self, text):
+            # extract intent and slots ...
+            return {
+                "input": text,
+                "intent": {
+                    "intentName": intent_name,
+                    "probability": probability
+                },
+                "slots": slots
+            }
+
 
 ----------------
 Snips NLU Engine
 ----------------
 
-Here is how you can use the metrics API to compute metrics for the Snips NLU
-pipeline:
+This library can be used to benchmark NLU solutions such as `Snips NLU`_. To
+install the ``snips-nlu`` python library, and fetch the language resources for
+english, run the following commands:
+
+.. code-block:: bash
+
+    $ pip install snips-nlu
+    $ snips-nlu download en
+
+
+Then, you can compute metrics for the ``snips-nlu`` pipeline using the metrics
+API as follows:
 
 .. code-block:: python
 
-    from snips_nlu import SnipsNLUEngine
+    from snips_nlu import load_resources, SnipsNLUEngine
     from snips_nlu_metrics import compute_train_test_metrics, compute_cross_val_metrics
 
+    load_resources("en")
 
     tt_metrics = compute_train_test_metrics(train_dataset="samples/train_dataset.json",
                                             test_dataset="samples/test_dataset.json",
@@ -67,16 +107,6 @@ pipeline:
                                            engine_class=SnipsNLUEngine,
                                            nb_folds=5)
 
-Some `sample code and datasets <samples/>`_ are also available, you can have an
-overview of the metrics output by running:
-
-.. code-block:: bash
-
-    git clone https://github.com/snipsco/snips-nlu-metrics.git
-    cd snips-nlu-metrics
-    pip install -e ".[samples]"
-    python samples/sample.py train-test
-
 -----------------
 Custom NLU Engine
 -----------------
@@ -128,4 +158,6 @@ This library is provided by `Snips <https://www.snips.ai>`_ as Open Source softw
 .. _train/test: https://en.wikipedia.org/wiki/Training,_test,_and_validation_sets
 .. _Snips NLU: https://github.com/snipsco/snips-nlu
 .. _precision, recall and f1 scores: https://en.wikipedia.org/wiki/Precision_and_recall
-.. _confusion matrix: https://en.wikipedia.org/wiki/Confusion_matrix
+.. _confusion matrix: https://en.wikipedia.org/wiki/Confusion_matrix
+.. _dataset generation tool: http://snips-nlu.readthedocs.io/en/latest/tutorial.html#snips-dataset-format
+.. _Snips console: https://console.snips.ai

setup.py

@@ -19,17 +19,14 @@
 
 install_requires = [
     "future",
-    "numpy==1.14.0",
-    "scipy==1.0.0",
-    "scikit-learn==0.19.1",
+    "numpy>=1.7,<2.0",
+    "scipy>=1.0,<2.0",
+    "scikit-learn>=0.19,<0.20",
 ]
 
 extras_require = {
     "test": [
-        "mock==2.0.0",
-    ],
-    "samples": [
-        "snips-nlu==0.12.1"
+        "mock>=2.0,<3.0",
     ]
 }
 

snips_nlu_metrics/__init__.py

@@ -2,6 +2,4 @@
 
 from snips_nlu_metrics.engine import Engine
 from snips_nlu_metrics.metrics import (compute_train_test_metrics,
-                                       compute_train_test_nlu_metrics,
-                                       compute_cross_val_metrics,
-                                       compute_cross_val_nlu_metrics)
+                                       compute_cross_val_metrics)

snips_nlu_metrics/__version__

@@ -1 +1 @@
-0.12.0
+0.13.0

snips_nlu_metrics/engine.py

@@ -1,20 +1,10 @@
 from __future__ import unicode_literals
 
-import io
-import json
-import os
-import zipfile
 from abc import ABCMeta, abstractmethod
 from builtins import object
-from builtins import str
-from copy import deepcopy
 
 from future.utils import with_metaclass
 
-from snips_nlu_metrics.utils.temp_utils import tempdir_ctx
-
-TRAINED_ENGINE_FILENAME = "trained_assistant.json"
-
 
 class Engine(with_metaclass(ABCMeta, object)):
     """Abstract class which represents an engine that can be used in the
@@ -28,50 +18,3 @@ def fit(self, dataset):
     @abstractmethod
     def parse(self, text):
         pass
-
-
-def build_nlu_engine_class(training_class, inference_class,
-                           training_config=None):
-    _training_config = deepcopy(training_config)
-
-    class NLUEngine(Engine):
-        def __init__(self):
-            self.inference_engine = None
-            self.training_config = _training_config
-
-        def fit(self, dataset):
-            if self.training_config is not None:
-                training_engine = training_class(config=self.training_config)
-            else:
-                training_engine = training_class()
-            training_engine.fit(dataset)
-            trained_engine_dict = training_engine.to_dict()
-            self.inference_engine = get_inference_nlu_engine(
-                trained_engine_dict, inference_class)
-
-        def parse(self, text):
-            return self.inference_engine.parse(text)
-
-    return NLUEngine
-
-
-def get_trained_nlu_engine(train_dataset, training_engine_class):
-    language = train_dataset["language"]
-    engine = training_engine_class(language)
-    engine.fit(train_dataset)
-    return engine
-
-
-def get_inference_nlu_engine(trained_engine_dict, inference_engine_class):
-    with tempdir_ctx() as engine_dir:
-        trained_engine_path = os.path.join(engine_dir, TRAINED_ENGINE_FILENAME)
-        archive_path = os.path.join(engine_dir, 'assistant.zip')
-
-        with io.open(trained_engine_path, mode='w', encoding='utf8') as f:
-            f.write(str(json.dumps(trained_engine_dict)))
-        with zipfile.ZipFile(archive_path, 'w') as zf:
-            zf.write(trained_engine_path, arcname=TRAINED_ENGINE_FILENAME)
-        with io.open(archive_path, mode='rb') as f:
-            data_zip = bytearray(f.read())
-
-    return inference_engine_class(data_zip=data_zip)