diff --git a/README.md b/README.md index 7624393b01..1bc716d1a8 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,12 @@ DeepPavlov is designed for ## Quick Links -* Demo [*demo.ipavlov.ai*](https://demo.ipavlov.ai/) +* Demo [*demo.deeppavlov.ai*](https://demo.deeppavlov.ai/) * Documentation [*docs.deeppavlov.ai*](http://docs.deeppavlov.ai/) * Model List [*docs:features/*](http://docs.deeppavlov.ai/en/master/features/overview.html) * Contribution Guide [*docs:contribution_guide/*](http://docs.deeppavlov.ai/en/master/devguides/contribution_guide.html) * Issues [*github/issues/*](https://github.com/deepmipt/DeepPavlov/issues) -* Forum [*forum.ipavlov.ai*](https://forum.ipavlov.ai/) +* Forum [*forum.deeppavlov.ai*](https://forum.deeppavlov.ai/) * Blogs [*medium.com/deeppavlov*](https://medium.com/deeppavlov) * Tutorials [*examples/*](https://github.com/deepmipt/DeepPavlov/tree/master/examples) and [extended colab tutorials](https://github.com/deepmipt/dp_tutorials) * Docker Hub [*hub.docker.com/u/deeppavlov/*](https://hub.docker.com/u/deeppavlov/) diff --git a/deeppavlov/__init__.py b/deeppavlov/__init__.py index 8d259ea039..9b773df304 100644 --- a/deeppavlov/__init__.py +++ b/deeppavlov/__init__.py @@ -37,7 +37,7 @@ def evaluate_model(config: [str, Path, dict], download: bool = False, recursive: except ImportError: 'Assuming that requirements are not yet installed' -__version__ = '0.6.0' +__version__ = '0.6.1' __author__ = 'Neural Networks and Deep Learning lab, MIPT' __description__ = 'An open source library for building end-to-end dialog systems and training chatbots.' __keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot'] diff --git a/deeppavlov/configs/go_bot/gobot_dstc2_minimal.json b/deeppavlov/configs/go_bot/gobot_dstc2_minimal.json index c210b0c108..eeadeadc31 100644 --- a/deeppavlov/configs/go_bot/gobot_dstc2_minimal.json +++ b/deeppavlov/configs/go_bot/gobot_dstc2_minimal.json @@ -1,7 +1,7 @@ { "dataset_reader": { "class_name": "dstc2_reader", - "data_path": "{DOWNLOADS_PATH}/dstc2" + "data_path": "{DATA_PATH}" }, "dataset_iterator": { "class_name": "dialog_iterator" @@ -21,49 +21,41 @@ "id": "word_vocab", "class_name": "simple_vocab", "fit_on": ["x_tokens"], - "save_path": "{MODELS_PATH}/gobot_dstc2_minimal/word.dict", - "load_path": "{MODELS_PATH}/gobot_dstc2_minimal/word.dict" - }, - { - "id": "restaurant_database", - "class_name": "sqlite_database", - "table_name": "mytable", - "primary_keys": ["name"], - "save_path": "{DOWNLOADS_PATH}/dstc2/resto.sqlite" + "save_path": "{MODEL_PATH}/word.dict", + "load_path": "{MODEL_PATH}/word.dict" }, { "class_name": "go_bot", - "load_path": "{MODELS_PATH}/gobot_dstc2_minimal/model", - "save_path": "{MODELS_PATH}/gobot_dstc2_minimal/model", + "load_path": "{MODEL_PATH}/model", + "save_path": "{MODEL_PATH}/model", "in": ["x"], "in_y": ["y"], "out": ["y_predicted"], "main": true, "debug": false, - "learning_rate": [2e-3, 2e-2], - "learning_rate_decay": "1cycle", - "learning_rate_decay_epochs": 20, + "learning_rate": 0.003, + "learning_rate_drop_patience": 5, + "learning_rate_drop_div": 10.0, "momentum": 0.95, "optimizer": "tensorflow.train:AdamOptimizer", "clip_norm": 2.0, - "dropout_rate": 0.3, - "l2_reg_coef": 1e-4, + "dropout_rate": 0.4, + "l2_reg_coef": 3e-4, "hidden_size": 128, - "dense_size": 64, + "dense_size": 160, "word_vocab": "#word_vocab", - "template_path": "{DOWNLOADS_PATH}/dstc2/dstc2-templates.txt", - "template_type": "DualTemplate", - "database": "#restaurant_database", - "api_call_action": "api_call", + "template_path": "{DATA_PATH}/dstc2-templates.txt", + "template_type": "DefaultTemplate", + "database": null, + "api_call_action": null, "use_action_mask": false, "slot_filler": null, "intent_classifier": null, - "embedder": null, - "bow_embedder": { - "class_name": "bow", - "depth": "#word_vocab.__len__()", - "with_counts": true + "embedder": { + "class_name": "glove", + "load_path": "{DOWNLOADS_PATH}/embeddings/glove.6B.100d.txt" }, + "bow_embedder": null, "tokenizer": { "class_name": "stream_spacy_tokenizer", "lowercase": false @@ -81,10 +73,10 @@ "metrics": ["per_item_dialog_accuracy"], "validation_patience": 10, - "val_every_n_epochs": 1, + "val_every_n_batches": 15, - "log_every_n_batches": -1, - "log_every_n_epochs": 1, + "log_every_n_batches": 15, + "log_on_k_batches": -1, "show_examples": false, "evaluation_targets": [ "valid", @@ -95,12 +87,15 @@ "metadata": { "variables": { "ROOT_PATH": "~/.deeppavlov", + "CONFIGS_PATH": "{DEEPPAVLOV_PATH}/configs", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models" + "DATA_PATH": "{DOWNLOADS_PATH}/dstc2", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/gobot_dstc2_minimal" }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt", - "{DEEPPAVLOV_PATH}/requirements/fasttext.txt", + "{DEEPPAVLOV_PATH}/requirements/gensim.txt", "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], @@ -109,9 +104,17 @@ "server_utils": "GoalOrientedBot" }, "download": [ + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/gobot_dstc2_v9.tar.gz", + "subdir": "{MODELS_PATH}" + }, + { + "url": "http://files.deeppavlov.ai/embeddings/glove.6B.100d.txt", + "subdir": "{DOWNLOADS_PATH}/embeddings" + }, { "url": "http://files.deeppavlov.ai/datasets/dstc2_v2.tar.gz", - "subdir": "{DOWNLOADS_PATH}/dstc2" + "subdir": "{DATA_PATH}" } ] } diff --git a/deeppavlov/dataset_readers/conll2003_reader.py b/deeppavlov/dataset_readers/conll2003_reader.py index c204ddb4c4..19d2bc1d87 100644 --- a/deeppavlov/dataset_readers/conll2003_reader.py +++ b/deeppavlov/dataset_readers/conll2003_reader.py @@ -1,9 +1,11 @@ from pathlib import Path +from logging import getLogger from deeppavlov.core.common.registry import register from deeppavlov.core.data.dataset_reader import DatasetReader from deeppavlov.core.data.utils import download_decompress +log = getLogger(__name__) @register('conll2003_reader') class Conll2003DatasetReader(DatasetReader): @@ -80,10 +82,19 @@ def parse_ner_file(self, file_name: Path): tags = [] else: if self.provide_pos: - token, pos, *_, tag = line.split() - pos_tags.append(pos) + try: + token, pos, *_, tag = line.split() + pos_tags.append(pos) + except: + log.warning('Skip {}, splitted as {}'.format(repr(line), repr(line.split()))) + continue else: - token, *_, tag = line.split() + try: + token, *_, tag = line.split() + except: + log.warning('Skip {}, splitted as {}'.format(repr(line), repr(line.split()))) + continue + tags.append(tag) tokens.append(token) @@ -111,4 +122,3 @@ def _iob2_to_iob(tags): iob_tags.append(tag) return iob_tags - diff --git a/deeppavlov/utils/server/server.py b/deeppavlov/utils/server/server.py index 65a4439925..1c66e91cf8 100644 --- a/deeppavlov/utils/server/server.py +++ b/deeppavlov/utils/server/server.py @@ -12,14 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ssl -from logging import getLogger, Filter, LogRecord +import logging from pathlib import Path -from typing import List, Tuple +from ssl import PROTOCOL_TLSv1_2 +from typing import Dict, List, Optional -from flasgger import Swagger, swag_from -from flask import Flask, request, jsonify, redirect, Response -from flask_cors import CORS +import uvicorn +from fastapi import FastAPI, HTTPException +from fastapi.utils import generate_operation_id_for_path +from pydantic import BaseConfig, BaseModel, Schema +from pydantic.fields import Field +from pydantic.main import MetaModel +from starlette.responses import JSONResponse, RedirectResponse from deeppavlov.core.agent.dialog_logger import DialogLogger from deeppavlov.core.commands.infer import build_model @@ -32,24 +36,22 @@ SERVER_CONFIG_FILENAME = 'server_config.json' -class PollerFilter(Filter): - """PollerFilter class is used to filter POST requests to /probe endpoint from logs.""" - def filter(self, record: LogRecord) -> bool: +class ProbeFilter(logging.Filter): + """ProbeFilter class is used to filter POST requests to /probe endpoint from logs.""" + def filter(self, record: logging.LogRecord) -> bool: """To log the record method should return True.""" return 'POST /probe HTTP' not in record.getMessage() -log = getLogger(__name__) -werklog = getLogger('werkzeug') -werklog.addFilter(PollerFilter()) - -app = Flask(__name__) -CORS(app) +log = logging.getLogger(__name__) +uvicorn_log = logging.getLogger('uvicorn') +uvicorn_log.addFilter(ProbeFilter()) +app = FastAPI(__file__) dialog_logger = DialogLogger(agent_name='dp_api') -def get_server_params(server_config_path, model_config): +def get_server_params(server_config_path: Path, model_config: Path) -> Dict: server_config = read_json(server_config_path) model_config = parse_config(model_config) @@ -64,44 +66,34 @@ def get_server_params(server_config_path, model_config): server_params[param_name] = model_defaults[param_name] server_params['model_endpoint'] = server_params.get('model_endpoint', '/model') - server_params['model_args_names'] = server_params['model_args_names'] or model_config['chainer']['in'] - - return server_params + arg_names = server_params['model_args_names'] or model_config['chainer']['in'] + if isinstance(arg_names, str): + arg_names = [arg_names] + server_params['model_args_names'] = arg_names -def interact(model: Chainer, params_names: List[str]) -> Tuple[Response, int]: - if not request.is_json: - log.error("request Content-Type header is not application/json") - return jsonify({ - "error": "request Content-Type header is not application/json" - }), 400 - - model_args = [] + return server_params - data = request.get_json() - dialog_logger.log_in(data) - for param_name in params_names: - param_value = data.get(param_name) - if param_value is None or (isinstance(param_value, list) and len(param_value) > 0): - model_args.append(param_value) - else: - log.error(f"nonempty array expected but got '{param_name}'={repr(param_value)}") - return jsonify({'error': f"nonempty array expected but got '{param_name}'={repr(param_value)}"}), 400 - lengths = {len(i) for i in model_args if i is not None} +def interact(model: Chainer, payload: Dict[str, Optional[List]]) -> JSONResponse: + model_args = payload.values() + dialog_logger.log_in(payload) + error_msg = None + lengths = {len(model_arg) for model_arg in model_args if model_arg is not None} if not lengths: - log.error('got empty request') - return jsonify({'error': 'got empty request'}), 400 + error_msg = 'got empty request' + elif 0 in lengths: + error_msg = 'dot empty array as model argument' elif len(lengths) > 1: - log.error('got several different batch sizes') - return jsonify({'error': 'got several different batch sizes'}), 400 + error_msg = 'got several different batch sizes' - batch_size = list(lengths)[0] - model_args = [arg or [None] * batch_size for arg in model_args] + if error_msg is not None: + log.error(error_msg) + raise HTTPException(status_code=400, detail=error_msg) - # in case when some parameters were not described in model_args - model_args += [[None] * batch_size for _ in range(len(model.in_x) - len(model_args))] + batch_size = next(iter(lengths)) + model_args = [arg or [None] * batch_size for arg in model_args] prediction = model(*model_args) if len(model.out_params) == 1: @@ -109,23 +101,20 @@ def interact(model: Chainer, params_names: List[str]) -> Tuple[Response, int]: prediction = list(zip(*prediction)) result = jsonify_data(prediction) dialog_logger.log_out(result) - return jsonify(result), 200 + return JSONResponse(result) -def test_interact(model: Chainer, params_names: List[str]) -> Tuple[Response, int]: - data = request.get_json() - if not data: - model_args = [["Test string."] for _ in params_names] - else: - model_args = [data.get(param_name) for param_name in params_names] +def test_interact(model: Chainer, payload: Dict[str, Optional[List]]) -> JSONResponse: + model_args = [arg or ["Test string."] for arg in payload.values()] try: _ = model(*model_args) - return Response('["Test passed"]\n'), 200 - except Exception: - return Response('["Test failed"]\n'), 400 + return JSONResponse(["Test passed"]) + except Exception as e: + raise HTTPException(status_code=400, detail=e) -def start_model_server(model_config, https=False, ssl_key=None, ssl_cert=None, port=None): +def start_model_server(model_config: Path, https: bool = False, ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None, port: Optional[int] = None) -> None: server_config_path = get_settings_path() / SERVER_CONFIG_FILENAME server_params = get_server_params(server_config_path, model_config) @@ -135,9 +124,6 @@ def start_model_server(model_config, https=False, ssl_key=None, ssl_cert=None, p docs_endpoint = server_params['docs_endpoint'] model_args_names = server_params['model_args_names'] - Swagger.DEFAULT_CONFIG['specs_route'] = docs_endpoint - Swagger(app) - https = https or server_params['https'] if https: @@ -155,45 +141,44 @@ def start_model_server(model_config, https=False, ssl_key=None, ssl_cert=None, p log.error(e) raise e - ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) - ssl_context.load_cert_chain(ssh_cert_path, ssh_key_path) + ssl_version = PROTOCOL_TLSv1_2 + ssl_keyfile = str(ssh_key_path) + ssl_certfile = str(ssh_cert_path) else: - ssl_context = None + ssl_version = None + ssl_keyfile = None + ssl_certfile = None model = build_model(model_config) - @app.route('/') - def index(): - return redirect(docs_endpoint) - - endpoint_description = { - 'description': 'A model endpoint', - 'parameters': [ - { - 'name': 'data', - 'in': 'body', - 'required': 'true', - 'example': {arg: ['value'] for arg in model_args_names} - } - ], - 'responses': { - "200": { - "description": "A model response" - } - } - } - - @app.route(model_endpoint, methods=['POST']) - @swag_from(endpoint_description) - def answer(): - return interact(model, model_args_names) - - @app.route('/probe', methods=['POST']) - def probe(): - return test_interact(model, model_args_names) - - @app.route('/api', methods=['GET']) - def api(): - return jsonify(model_args_names), 200 - - app.run(host=host, port=port, threaded=False, ssl_context=ssl_context) + def batch_decorator(cls: MetaModel) -> MetaModel: + cls.__annotations__ = {arg_name: List[str] for arg_name in model_args_names} + cls.__fields__ = {arg_name: Field(name=arg_name, type_=List[str], class_validators=None, + model_config=BaseConfig, required=False, schema=Schema(None)) + for arg_name in model_args_names} + return cls + + @batch_decorator + class Batch(BaseModel): + pass + + @app.get('/', include_in_schema=False) + async def redirect_to_docs() -> RedirectResponse: + operation_id = generate_operation_id_for_path(name='answer', path=model_endpoint, method='post') + response = RedirectResponse(url=f'{docs_endpoint}#/default/{operation_id}') + return response + + @app.post(model_endpoint, status_code=200, summary='A model endpoint') + async def answer(item: Batch) -> JSONResponse: + return interact(model, item.dict()) + + @app.post('/probe', status_code=200, include_in_schema=False) + async def probe(item: Batch) -> JSONResponse: + return test_interact(model, item.dict()) + + @app.get('/api', status_code=200, summary='Model argument names') + async def api() -> JSONResponse: + return JSONResponse(model_args_names) + + uvicorn.run(app, host=host, port=port, logger=uvicorn_log, + ssl_version=ssl_version, ssl_keyfile=ssl_keyfile, ssl_certfile=ssl_certfile) diff --git a/deeppavlov/utils/settings/log_config.json b/deeppavlov/utils/settings/log_config.json index b3d030928c..0f25e18038 100644 --- a/deeppavlov/utils/settings/log_config.json +++ b/deeppavlov/utils/settings/log_config.json @@ -8,12 +8,23 @@ "stderr" ], "propagate": true + }, + "uvicorn": { + "level": "INFO", + "handlers": [ + "uvicorn_handler" + ], + "propagate": true } }, "formatters": { "default": { "format": "%(asctime)s.%(msecs)d %(levelname)s in '%(name)s'['%(module)s'] at line %(lineno)d: %(message)s", "datefmt": "%Y-%m-%d %H:%M:%S" + }, + "uvicorn_fmt": { + "format":"%(asctime)s %(message)s", + "datefmt": "%Y-%m-%d %H:%M:%S" } }, "handlers": { @@ -34,6 +45,12 @@ "level": "DEBUG", "formatter": "default", "stream": "ext://sys.stderr" + }, + "uvicorn_handler": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "uvicorn_fmt", + "stream": "ext://sys.stdout" } } } diff --git a/deeppavlov/utils/settings/server_config.json b/deeppavlov/utils/settings/server_config.json index 2b63e04e3e..4dd43c9a13 100644 --- a/deeppavlov/utils/settings/server_config.json +++ b/deeppavlov/utils/settings/server_config.json @@ -2,7 +2,7 @@ "common_defaults": { "host": "0.0.0.0", "port": 5000, - "docs_endpoint": "/docs/", + "docs_endpoint": "/docs", "model_args_names": "", "https": false, "https_cert_path": "", diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html index 56c16abd8b..c2b05c38af 100644 --- a/docs/_templates/footer.html +++ b/docs/_templates/footer.html @@ -16,7 +16,7 @@
{%- block extrafooter %} -

Problem? Ask a Question or try our Demo

+

Problem? Ask a Question or try our Demo

twitter facebook diff --git a/docs/devguides/contribution_guide.rst b/docs/devguides/contribution_guide.rst index 8ea975255e..fb5197ef4e 100644 --- a/docs/devguides/contribution_guide.rst +++ b/docs/devguides/contribution_guide.rst @@ -105,7 +105,7 @@ added to the framework. Your research will become a part of a common big work and other people will happily use it and thank you :D If you still have any questions, either on the contribution process or about -the framework itself, please ask us at our forum ``_. +the framework itself, please ask us at our forum ``_. Follow us on Facebook to get news on releases, new features, approved contributions and resolved issues ``_ diff --git a/docs/index.rst b/docs/index.rst index 867822c649..6e2cef9165 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,8 +5,8 @@ Welcome to DeepPavlov's documentation! :glob: :maxdepth: 1 - QuickStart Installation + QuickStart General concepts Configuration file Models/Skills overview diff --git a/docs/intro/quick_start.rst b/docs/intro/quick_start.rst index db70fdc193..e4b3b29aed 100644 --- a/docs/intro/quick_start.rst +++ b/docs/intro/quick_start.rst @@ -1,11 +1,13 @@ QuickStart ------------ -There is a bunch of great pre-trained NLP models in DeepPavlov. Each model is -determined by its config file. +First, follow instructions on :doc:`Installation page ` +to install ``deeppavlov`` package for Python 3.6/3.7. -List of models is available on :doc:`the doc page ` or in -the ``deeppavlov.configs`` (Python): +DeepPavlov contains a bunch of great pre-trained NLP models. Each model is +determined by it's config file. List of models is available on +:doc:`the doc page ` or in +the ``deeppavlov.configs``: .. code:: python @@ -130,3 +132,123 @@ There are also available integrations with various messengers, see :doc:`Telegram Bot doc page ` and others in the Integrations section for more info. + +Pretrained models +~~~~~~~~~~~~~~~~~ + +DeepPavlov provides a wide range of pretrained models and skills. +See :doc:`features overview ` for more info. Please +note that most of our models are trained on specific datasets for +specific tasks and may require further training on you data. +You can find a list of our out-of-the-box models `below <#out-of-the-box-pretrained-models>`_. + + +Docker images +~~~~~~~~~~~~~ + +You can run DeepPavlov models in :doc:`riseapi ` mode +via Docker without installing DP. Both your CPU and GPU (we support NVIDIA graphic +processors) can be utilised, please refer our `CPU `_ +and `GPU `_ Docker images run instructions. + + +Out-of-the-box pretrained models +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While the best way to solve most of the NLP tasks lies through collecting datasets +and training models according to the domain and an actual task itself, DeepPavlov +offers several pretrained models, which can be strong baselines for a wide range of tasks. + +You can run these models `via Docker <#docker-images>`_ or in ``riseapi``/``risesocket`` mode to use in +solutions. See :doc:`riseapi ` and :doc:`risesocket ` +modes documentation for API details. + + +Text Question Answering +======================= + +Text Question Answering component answers a question based on a given context (e.g, +a paragraph of text), where the answer to the question is a segment of the context. + +.. table:: + :widths: auto + + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + | Language | DeepPavlov config | Demo | + +==========+================================================================================================+===========================================+ + | Multi | :config:`squad_bert_multilingual_freezed_emb ` | https://demo.deeppavlov.ai/#/mu/textqa | + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + | En | :config:`squad_bert_infer ` | https://demo.deeppavlov.ai/#/en/textqa | + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + | Ru | :config:`squad_ru_bert_infer ` | https://demo.deeppavlov.ai/#/ru/textqa | + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + + +Name Entity Recognition +======================= + +Named Entity Recognition (NER) classifies tokens in text into predefined categories +(tags), such as person names, quantity expressions, percentage expressions, names +of locations, organizations, as well as expression of time, currency and others. + +.. table:: + :widths: auto + + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + | Language | DeepPavlov config | Demo | + +==========+================================================================================================+===========================================+ + | Multi | :config:`ner_ontonotes_bert_mult ` | https://demo.deeppavlov.ai/#/mu/ner | + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + | En | :config:`ner_ontonotes_bert_mult ` | https://demo.deeppavlov.ai/#/en/ner | + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + | Ru | :config:`ner_rus_bert ` | https://demo.deeppavlov.ai/#/ru/ner | + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + + +Insult Detection +================ + +Insult detection predicts whether a text (e.g, post or speech in some +public discussion) is considered insulting to one of the persons it is +related to. + +.. table:: + :widths: auto + + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + | Language | DeepPavlov config | Demo | + +==========+================================================================================================+===========================================+ + | En | :config:`insults_kaggle_conv_bert ` | https://demo.deeppavlov.ai/#/en/insult | + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + + +Sentiment Analysis +================== + +Classify text according to a prevailing emotion (positive, negative, etc.) in it. + +.. table:: + :widths: auto + + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + | Language | DeepPavlov config | Demo | + +==========+================================================================================================+===========================================+ + | Ru | :config:`rusentiment_elmo_twitter_cnn ` | https://demo.deeppavlov.ai/#/ru/sentiment | + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + + +Paraphrase Detection +==================== + +Detect if two given texts have the same meaning. + +.. table:: + :widths: auto + + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + | Language | DeepPavlov config | Demo | + +==========+================================================================================================+===========================================+ + | En | :config:`paraphraser_bert ` | None | + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ + | Ru | :config:`paraphraser_rubert ` | None | + +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ diff --git a/examples/Pseudo-labeling for classification.ipynb b/examples/Pseudo-labeling for classification.ipynb index 45029bf491..8d01922069 100644 --- a/examples/Pseudo-labeling for classification.ipynb +++ b/examples/Pseudo-labeling for classification.ipynb @@ -188,10 +188,10 @@ ], "metadata": { "kernelspec": { - "display_name": "deep36", - "language": "python", - "name": "deep36" + "display_name": "Python 3", + "name": "python3" }, + "accelerator": "GPU", "language_info": { "codemirror_mode": { "name": "ipython", diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000000..e3da3b553d --- /dev/null +++ b/examples/README.md @@ -0,0 +1,23 @@ +# Examples & Tutorials + +* Tutorial for simple bot [[notebook]](gobot_tutorial.ipynb) [[colab]](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/gobot_tutorial.ipynb) + +* Tutorial for advanced goal-oriented bot [[notebook]](gobot_extended_tutorial.ipynb) [[colab]](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/gobot_extended_tutorial.ipynb) + +* Tutorial for intent classifier [[notebook]](classification_tutorial.ipynb) [[colab]](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/classification_tutorial.ipynb) + +* Morphotagger model usage example [[notebook]](morphotagger_example.ipynb) [[colab]](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/morphotagger_example.ipynb) + +* Results for neural parameter evolution [[notebook]](evolution_results_analysis.ipynb) [[colab]](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/evolution_results_analysis.ipynb) + +* Pseudo-labeling for classication task [[notebook]](Pseudo-labeling%20for%20classification.ipynb) [[colab]](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/Pseudo-labeling%20for%20classification.ipynb) + +* Optimal learning rate search in DeepPavlov [[notebook]](super_convergence_tutorial.ipynb) [[colab]](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/super_convergence_tutorial.ipynb) + +* Integration of FAQ skill with Yandex Alice (RU) [[notebook]](yandex_faq_ru.ipynb) [[colab]](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/yandex_faq_ru.ipynb) + +# Links + +More examples are available: +* [github.com/deepmipt/dp_tutorials/](https://github.com/deepmipt/dp_tutorials) +* [github.com/deepmipt/db_notebooks/](https://github.com/deepmipt/dp_notebooks). diff --git a/examples/classification_tutorial.ipynb b/examples/classification_tutorial.ipynb index b0c7915b1d..e0ab7dccba 100644 --- a/examples/classification_tutorial.ipynb +++ b/examples/classification_tutorial.ipynb @@ -2940,10 +2940,10 @@ ], "metadata": { "kernelspec": { - "display_name": "tensorflow_kernel2", - "language": "python", - "name": "tensorflow_kernel2" + "display_name": "Python 3", + "name": "python3" }, + "accelerator": "GPU", "language_info": { "codemirror_mode": { "name": "ipython", diff --git a/examples/evolution_results_analysis.ipynb b/examples/evolution_results_analysis.ipynb index fd8d272f3f..83a7e89e28 100644 --- a/examples/evolution_results_analysis.ipynb +++ b/examples/evolution_results_analysis.ipynb @@ -339,10 +339,10 @@ ], "metadata": { "kernelspec": { - "display_name": "deep36", - "language": "python", - "name": "deep36" + "display_name": "Python 3", + "name": "python3" }, + "accelerator": "GPU", "language_info": { "codemirror_mode": { "name": "ipython", diff --git a/examples/gobot_extended_tutorial.ipynb b/examples/gobot_extended_tutorial.ipynb new file mode 100644 index 0000000000..d2ae991dae --- /dev/null +++ b/examples/gobot_extended_tutorial.ipynb @@ -0,0 +1,1386 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "K7nBJnADTgUw" + }, + "source": [ + "### You can also run the notebook in [COLAB](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/gobot_tutorial.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "iPbAiv8KTgU4" + }, + "source": [ + "# Goal-oriented bot in DeepPavlov" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "us6IsTUETgU5" + }, + "source": [ + "This tutorial describes how to build a goal/task-oriented dialogue system with DeepPavlov framework. It covers the following steps:\n", + "\n", + "0. [Data preparation](#0.-Data-Preparation)\n", + "1. [Build Database of items](#1.-Build-Database-of-items)\n", + "2. [Build Slot Filler](#2.-Build-Slot-Filler)\n", + "3. [Build and Train a Bot](#3.-Build-and-Train-a-Bot)\n", + "4. [Interact with bot](#4.-Interact-with-Bot)\n", + "\n", + "An example of the final model served as a telegram bot:\n", + "\n", + "![gobot_example.png](https://github.com/deepmipt/DeepPavlov/blob/master/examples/img/gobot_example.png?raw=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 806 + }, + "colab_type": "code", + "id": "Vtu-7ns2TgUz", + "outputId": "8cdc252f-1a35-4ed3-bf0a-f54046d8c6a8" + }, + "outputs": [], + "source": [ + "!pip install deeppavlov" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "4R066YWhTgU6" + }, + "source": [ + "## 0. Data Preparation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "gppbVe-HTgU7" + }, + "source": [ + "In this tutorial we build a chatbot for restaurant booking. To train our chatbot we use [Dialogue State Tracking Challenge 2 (DSTC-2)](http://camdial.org/~mh521/dstc/) dataset. DSTC-2 provides dialogues of a human talking to a booking system labelled with slots and dialogue actions. These labels will be used for training a dialogue policy network.\n", + "\n", + "First of all let's take a quick look at the data for the task. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 137 + }, + "colab_type": "code", + "id": "K9lF3QFJTgU8", + "outputId": "6ab259e2-3f88-4b25-9371-21d3f38fcef3" + }, + "outputs": [], + "source": [ + "from deeppavlov.dataset_readers.dstc2_reader import SimpleDSTC2DatasetReader\n", + "\n", + "data = SimpleDSTC2DatasetReader().read('my_data')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 50 + }, + "colab_type": "code", + "id": "uu56jAGJTgVD", + "outputId": "1536bb2c-6c1f-45a6-c0a7-a92106ed7dfe" + }, + "outputs": [], + "source": [ + "!ls my_data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "HmNmE80MTgVG" + }, + "source": [ + "The training/validation/test data are stored in json files (`simple-dstc2-trn.json`, `simple-dstc2-val.json` and `simple-dstc2-tst.json`):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "colab_type": "code", + "id": "LIm9DQyzTgVH", + "outputId": "0a82c3f1-8afb-42d5-e3e3-0e9dd9178a20" + }, + "outputs": [], + "source": [ + "!head -n 101 my_data/simple-dstc2-trn.json" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "zO4CWg0XYNSw" + }, + "source": [ + "To iterate over batches of preprocessed DSTC-2 we need to import `DatasetIterator`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "piBBcw9ZTgVK", + "scrolled": true + }, + "outputs": [], + "source": [ + "from deeppavlov.dataset_iterators.dialog_iterator import DialogDatasetIterator\n", + "\n", + "iterator = DialogDatasetIterator(data)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "jVU5JGnTTgVM" + }, + "source": [ + "You can now iterate over batches of preprocessed DSTC-2 dialogs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "colab_type": "code", + "id": "1RSwEH3CTgVN", + "outputId": "b2a0ecdb-89d1-4784-eeb9-749f7b754ff6" + }, + "outputs": [], + "source": [ + "from pprint import pprint\n", + "\n", + "for dialog in iterator.gen_batches(batch_size=1, data_type='train'):\n", + " turns_x, turns_y = dialog\n", + " \n", + " print(\"User utterances:\\n----------------\\n\")\n", + " pprint(turns_x[0], indent=4)\n", + " print(\"\\nSystem responses:\\n-----------------\\n\")\n", + " pprint(turns_y[0], indent=4)\n", + " \n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "AKTZWtm8ZtPi" + }, + "source": [ + "In real-life annotation of data is expensive. To make our tutorial closer to production use-cases we take only 50 dialogues for training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "UlappYTbTgVT" + }, + "outputs": [], + "source": [ + "!cp my_data/simple-dstc2-trn.json my_data/simple-dstc2-trn.full.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 33 + }, + "colab_type": "code", + "id": "tTU9yM-CTgVX", + "outputId": "1568aaed-7f8e-4f77-a637-cda5a9556740" + }, + "outputs": [], + "source": [ + "import json\n", + "\n", + "NUM_TRAIN = 50\n", + "\n", + "with open('my_data/simple-dstc2-trn.full.json', 'rt') as fin:\n", + " data = json.load(fin)\n", + "with open('my_data/simple-dstc2-trn.json', 'wt') as fout:\n", + " json.dump(data[:NUM_TRAIN], fout, indent=2)\n", + "print(f\"Train set is reduced to {NUM_TRAIN} dialogues (out of {len(data)}).\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "l5mjRphbTgVb" + }, + "source": [ + "## 1. Build Database of items" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "n597CLhqjqcd" + }, + "source": [ + "### Building database of restaurants" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "nJFkgfjTTgVf" + }, + "source": [ + "To assist with restaurant booking the chatbot should have access to a `database` of restaurants. The `database` contains task-specific information such as type of food, price range, location, etc.\n", + "\n", + " >> database([{'pricerange': 'cheap', 'area': 'south'}])\n", + " \n", + " Out[1]: \n", + " [[{'name': 'the lucky star',\n", + " 'food': 'chinese',\n", + " 'pricerange': 'cheap',\n", + " 'area': 'south',\n", + " 'addr': 'cambridge leisure park clifton way cherry hinton',\n", + " 'phone': '01223 244277',\n", + " 'postcode': 'c.b 1, 7 d.y'},\n", + " {'name': 'nandos',\n", + " 'food': 'portuguese',\n", + " 'pricerange': 'cheap',\n", + " 'area': 'south',\n", + " 'addr': 'cambridge leisure park clifton way',\n", + " 'phone': '01223 327908',\n", + " 'postcode': 'c.b 1, 7 d.y'}]]\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "rNpewHp-TgVd" + }, + "source": [ + " \n", + "![gobot_database.png](https://github.com/deepmipt/DeepPavlov/blob/master/examples/img/gobot_database.png?raw=1)\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-TU-NLnNa9tk" + }, + "source": [ + "The chatbot should be trained to make api calls. For this, training dataset contains a `\"db_result\"` dictionary key. It annotates turns where system performs an api call to the database of items. Rusulting value is stored in `\"db_result\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "EVNRZmeiTgVh", + "outputId": "edba5e2b-235f-423f-8bfa-8d02506c4c7e" + }, + "outputs": [], + "source": [ + "!head -n 78 my_data/simple-dstc2-trn.json | tail +51" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "GT4YBHMnl0Xd" + }, + "source": [ + "Set `primary_keys` to a list of slot names that have unique values for different items (common SQL term). For the case of DSTC-2, the primary slot is a restaurant name." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "JjKbIAyaTgVk", + "outputId": "07620401-80f5-490a-cff2-5d5f013a365b" + }, + "outputs": [], + "source": [ + "from deeppavlov.core.data.sqlite_database import Sqlite3Database\n", + "\n", + "database = Sqlite3Database(primary_keys=[\"name\"],\n", + " save_path=\"my_bot/db.sqlite\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "a2e1u-z0TgVo" + }, + "source": [ + "\n", + "Let's find all `\"db_result\"` api call results and add them to our database of restaurants:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "RlKg5UtqTgVp", + "outputId": "a387df1f-4418-498b-a125-9e351a8e0cf9" + }, + "outputs": [], + "source": [ + "db_results = []\n", + "\n", + "for dialog in iterator.gen_batches(batch_size=1, data_type='all'):\n", + " turns_x, turns_y = dialog\n", + " db_results.extend(x['db_result'] for x in turns_x[0] if x.get('db_result'))\n", + "\n", + "print(f\"Adding {len(db_results)} items.\")\n", + "if db_results:\n", + " database.fit(db_results)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "XeJMI9qaTgVt" + }, + "source": [ + "### Interacting with database" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "2JLUF2b_TgVu" + }, + "source": [ + "We can now play with the database and make requests to it:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "VRCU_MJnTgVv", + "outputId": "017803c4-36ab-49bc-ae40-7df87356f5c2" + }, + "outputs": [], + "source": [ + "database([{'pricerange': 'cheap', 'area': 'south'}])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "U2wOAIlpTgV1", + "outputId": "e83e53b9-3431-4d1c-9bed-0e841d2b6fc4" + }, + "outputs": [], + "source": [ + "!ls my_bot" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "mBoO34NzTgV4" + }, + "source": [ + "## 2. Build Slot Filler" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "TGlJRwTCYkiQ" + }, + "source": [ + "`Slot Filler` is a component that finds slot values in user input:\n", + "\n", + " >> slot_filler(['I would like some chineese food'])\n", + " \n", + " Out[1]: [{'food': 'chinese'}]\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "5RqXeLdTTgV4" + }, + "source": [ + " \n", + "![gobot_slotfiller.png](https://github.com/deepmipt/DeepPavlov/blob/master/examples/img/gobot_slotfiller.png?raw=1)\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "TcJGPFq4TgV5" + }, + "source": [ + "To implement a `Slot Filler` you need to provide\n", + " \n", + " - **slot types**,\n", + " - all possible **slot values**,\n", + " - also, it is good to have examples of mentions for every value of each slot.\n", + " \n", + "In this tutorial, a schema for `slot types` and `slot values` should be defined in `slot_vals.json` with the following format:\n", + "\n", + " {\n", + " 'food': {\n", + " 'chinese': ['chinese', 'chineese', 'chines'],\n", + " 'french': ['french', 'freench'],\n", + " 'dontcare': ['any food', 'any type of food']\n", + " }\n", + " }\n", + " \n", + "\n", + "Let's use a simple non-trainable slot filler that relies on Levenshtein distance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "zVi5XynnTgV6", + "outputId": "e9d68c8c-3bbb-4f80-98a5-92cbfe0eb5ac" + }, + "outputs": [], + "source": [ + "from deeppavlov.download import download_decompress\n", + "\n", + "download_decompress(url='http://files.deeppavlov.ai/deeppavlov_data/dstc_slot_vals.tar.gz',\n", + " download_path='my_bot/slotfill')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "NR1S3PXCTgV9", + "outputId": "013e9dba-427c-4255-aad5-0627477157e8" + }, + "outputs": [], + "source": [ + "!ls my_bot/slotfill" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-OZ9TqDKZ6Fv" + }, + "source": [ + "Print some `slot types` and `slot values`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "KqgfYr4RTgWE", + "outputId": "a6830aa3-0bcc-4011-a4ab-5b5e48e6a20f" + }, + "outputs": [], + "source": [ + "!head -n 10 my_bot/slotfill/dstc_slot_vals.json" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "eIufDAvATgWN" + }, + "source": [ + "Check performance of our slot filler on DSTC-2 dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "XUSj5R3uTgWP" + }, + "outputs": [], + "source": [ + "from deeppavlov import configs\n", + "from deeppavlov.core.common.file import read_json\n", + "\n", + "slotfill_config = read_json(configs.ner.slotfill_simple_dstc2_raw)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "pFda6_LBTgWT" + }, + "source": [ + "We take [original DSTC2 slot-filling config](https://github.com/deepmipt/DeepPavlov/blob/master/deeppavlov/configs/ner/slotfill_dstc2_raw.json) from DeepPavlov and change variables determining data paths:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "yr8MbFLwTgWV" + }, + "outputs": [], + "source": [ + "slotfill_config['metadata']['variables']['DATA_PATH'] = 'my_data'\n", + "slotfill_config['metadata']['variables']['SLOT_VALS_PATH'] = 'my_bot/slotfill/dstc_slot_vals.json'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ZxMTySrpaZVP" + }, + "source": [ + "Run evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "CdrDW4bVTgWZ", + "outputId": "ac56ae74-b368-437e-c70f-01b418ba883f" + }, + "outputs": [], + "source": [ + "from deeppavlov import evaluate_model\n", + "\n", + "slotfill = evaluate_model(slotfill_config);" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "azulujiLTgWb" + }, + "source": [ + "We've got slot accuracy of **93% on valid** set and **95% on test** set." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "FkZvQ-yNig1u" + }, + "source": [ + "Building `Slot Filler` model from DeepPavlov config." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "uWeXTtVhTgWc" + }, + "outputs": [], + "source": [ + "from deeppavlov import build_model\n", + "\n", + "slotfill = build_model(slotfill_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ihi4lpXUi-_V" + }, + "source": [ + "Testing the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "bMRSU_bnTgWf", + "outputId": "d224e4be-1537-428d-ff67-55076224946d" + }, + "outputs": [], + "source": [ + "slotfill(['i want cheap chinee food'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "U2PUxB5fTgWl" + }, + "source": [ + "Saving slotfill config file to disk (we will require it's path later)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "5MyFaEM7TgWl" + }, + "outputs": [], + "source": [ + "import json\n", + "\n", + "json.dump(slotfill_config, open('my_bot/slotfill_config.json', 'wt'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "colab_type": "code", + "id": "_ZlRvicuTgWo", + "outputId": "4f1c3d46-d3b1-4923-823e-e2df1027fc6f" + }, + "outputs": [], + "source": [ + "!ls my_bot" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "E_InRKO6TgWt" + }, + "source": [ + "## 3. Build and Train a Bot" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ySe2m9-5m6iW" + }, + "source": [ + "### Dialogue policy and response templates" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "qjwbkeDl3TBg" + }, + "source": [ + "A policy module of the bot decides what action should be taken in the current dialogue state. The policy in our bot is implemented as a recurrent neural network (recurrency over user utterances) followed by a dense layer with softmax function on top. The network classifies user input into one of predefined system actions. Examples of possible actions are to say hello, to request user's location or to make api call to a database. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "wLE1iebG3WJc" + }, + "source": [ + "![gobot_policy.png](https://github.com/deepmipt/DeepPavlov/blob/master/examples/img/gobot_policy.png?raw=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ghF-W56m3iW-" + }, + "source": [ + "All actions available for the system should be listed in a `simple-dstc2-templates.txt` file. Also, every action should be associated with a template string of the corresponding system response." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "TjDnGyiN3nIr" + }, + "source": [ + "![gobot_templates.png](https://github.com/deepmipt/DeepPavlov/blob/master/examples/img/gobot_templates.png?raw=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-xqGKtXBTgWu" + }, + "source": [ + "Templates for responses should be in the format `TAB