From 0f32759c67ea961737a6538fd37426f564bae925 Mon Sep 17 00:00:00 2001 From: Fedor Ignatov Date: Tue, 1 Oct 2019 11:04:09 +0300 Subject: [PATCH 01/28] feat: add FastAPI to Alice, Alexa and MS Bot Framework (#1009) * feat: Flask was completely replaced by Fastapi - Alexa, Alice and ms_bot_framework modes now on FastAPI - Flask removed from requirements - Docs endpoint name was removed from REST server config file * feat: added async to riseapi model and probe endpoints and to alice * feat: added async to riseapi * docs: added license and docstrings to data models of Alice ans Alexa * refactor: removed converter to JSONResponse type from responses * Snall fixes in ms bf docs * Bugfix in MS BF mode * feat: added keyboard interrupt handling to msbot * fix: fixed on keyboard interrupt correct closing * fix: alexa correct config to ssl * feat: correct keyboard interrupt handling by Alexa * fix: fixed Alexa and Alice servers - removed request body validation Pydantic models - event loop in Alexa server now getting after uvicorn initialization - JSON serialized data in request to Alexa now formatted correctly * refactor: typos and protecting variables --- deeppavlov/utils/alexa/bot.py | 26 ++- deeppavlov/utils/alexa/request_parameters.py | 94 +++++++++ deeppavlov/utils/alexa/server.py | 198 ++++--------------- deeppavlov/utils/alice/alice.py | 149 +++++--------- deeppavlov/utils/alice/request_parameters.py | 57 ++++++ deeppavlov/utils/ms_bot_framework/bot.py | 20 +- deeppavlov/utils/ms_bot_framework/server.py | 78 ++++---- deeppavlov/utils/server/server.py | 112 ++++++----- deeppavlov/utils/settings/server_config.json | 1 - docs/integrations/aws_ec2.rst | 2 +- docs/integrations/ms_bot.rst | 4 +- docs/integrations/rest_api.rst | 2 +- requirements.txt | 3 - 13 files changed, 370 insertions(+), 376 deletions(-) create mode 100644 deeppavlov/utils/alexa/request_parameters.py create mode 100644 deeppavlov/utils/alice/request_parameters.py diff --git a/deeppavlov/utils/alexa/bot.py b/deeppavlov/utils/alexa/bot.py index 62926f84bf..df883adf0e 100644 --- a/deeppavlov/utils/alexa/bot.py +++ b/deeppavlov/utils/alexa/bot.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import threading from collections import namedtuple from datetime import timedelta, datetime from logging import getLogger -from queue import Queue +from queue import Empty, Queue from threading import Timer, Thread from typing import Optional, Dict @@ -32,7 +33,7 @@ ValidatedCert = namedtuple('ValidatedCert', ['cert', 'expiration_timestamp']) - +# TODO: make common superclass with Bot from ms_bot_framework class Bot(Thread): """Contains agent (if not multi-instanced), conversations, validates Alexa requests and routes them to conversations. @@ -58,6 +59,7 @@ def __init__(self, agent_generator: callable, config: dict, input_queue: Queue, self.conversations: Dict[str, Conversation] = {} self.input_queue = input_queue self.output_queue = output_queue + self._run_flag = True self.valid_certificates: Dict[str, ValidatedCert] = {} @@ -73,10 +75,22 @@ def __init__(self, agent_generator: callable, config: dict, input_queue: Queue, def run(self) -> None: """Thread run method implementation.""" - while True: - request = self.input_queue.get() - response = self._handle_request(request) - self.output_queue.put(response) + while self._run_flag: + try: + request = self.input_queue.get(timeout=1) + except Empty: + pass + else: + response = self._handle_request(request) + self.output_queue.put(response) + + def join(self, timeout=None): + """Thread join method implementation.""" + self._run_flag = False + for timer in threading.enumerate(): + if isinstance(timer, threading.Timer): + timer.cancel() + Thread.join(self, timeout) def _del_conversation(self, conversation_key: str) -> None: """Deletes Conversation instance. diff --git a/deeppavlov/utils/alexa/request_parameters.py b/deeppavlov/utils/alexa/request_parameters.py new file mode 100644 index 0000000000..c360d8e3c7 --- /dev/null +++ b/deeppavlov/utils/alexa/request_parameters.py @@ -0,0 +1,94 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Request parameters for the DeepPavlov model launched as a skill for Amazon Alexa. + +Request parameters from this module are used to declare additional information +and validation for request parameters to the DeepPavlov model launched as +a skill for Amazon Alexa. + +See details at https://fastapi.tiangolo.com/tutorial/header-params/, + https://fastapi.tiangolo.com/tutorial/body-multiple-params/ + +""" + +from fastapi import Header, Body + +_signature_example = 'Z5H5wqd06ExFVPNfJiqhKvAFjkf+cTVodOUirucHGcEVAMO1LfvgqWUkZ/X1ITDZbI0w+SMwVkEQZlkeThbVS/54M22StNDUtfz4Ua20xNDpIPwcWIACAmZ38XxbbTEFJI5WwqrbilNcfzqiGrIPfdO5rl+/xUjHFUdcJdUY/QzBxXsceytVYfEiR9MzOCN2m4C0XnpThUavAu159KrLj8AkuzN0JF87iXv+zOEeZRgEuwmsAnJrRUwkJ4yWokEPnSVdjF0D6f6CscfyvRe9nsWShq7/zRTa41meweh+n006zvf58MbzRdXPB22RI4AN0ksWW7hSC8/QLAKQE+lvaw==' +_signature_cert_chain_url_example = 'https://s3.amazonaws.com/echo.api/echo-api-cert-6-ats.pem' +_body_example = { + "version": "1.0", + "session": { + "new": True, + "sessionId": "amzn1.echo-api.session.ee48c20e-5ad5-461f-a735-ce058491e914", + "application": { + "applicationId": "amzn1.ask.skill.52b86ebd-dd7d-45c3-a763-de584f62b8d6" + }, + "user": { + "userId": "amzn1.ask.account.AHUAJ5RRTJDATP63AIRLNOVBC2QCJ7U5WSVSD432EA45PDVWAX5CQ6Z2OLD2H2A77VSBQGIMIWAVBMWLHK2EVZAE5VVJ2FHWS4AQM3GMIDH62GZBZ4DOUWXA3DXRBBXXXTKAITDUCZTLG5GP3XN7YORE5FQO2MERGKK7WAJUTHPMLYN4W2IUBVYDIW7544M57N4KV5HMS4DESMY" + } + }, + "context": { + "System": { + "application": { + "applicationId": "amzn1.ask.skill.52b86ebd-dd7d-45c3-a763-de584f62b8d6" + }, + "user": { + "userId": "amzn1.ask.account.AHUAJ5RRTJDATP63AIRLNOVBC2QCJ7U5WSVSD432EA45PDVWAX5CQ6Z2OLD2H2A77VSBQGIMIWAVBMWLHK2EVZAE5VVJ2FHWS4AQM3GMIDH62GZBZ4DOUWXA3DXRBBXXXTKAITDUCZTLG5GP3XN7YORE5FQO2MERGKK7WAJUTHPMLYN4W2IUBVYDIW7544M57N4KV5HMS4DESMY" + }, + "device": { + "deviceId": "amzn1.ask.device.AH777YKPTWMNQGVKUKDWPQOWWEDBDJNMIGP5GHDXOIMI3N5RYZWQ2HBQEOUXMUJEHRBKDX6HCFEA7RRWNAGKHJLSD5KWLTKR35D42TW6BVL64THCYUITTH3G6ZMWZ6GNAELTXWB4YAZJWUK4J2BIFVLUP2KHZNTQRJRBEFGNWY4V2RCEEQOZC", + "supportedInterfaces": {} + }, + "apiEndpoint": "https://api.amazonalexa.com", + "apiAccessToken": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6IjEifQ.eyJhdWQiOiJodHRwczovL2FwaS5hbWF6b25hbGV4YS5jb20iLCJpc3MiOiJBbGV4YVNraWxsS2l0Iiwic3ViIjoiYW16bjEuYXNrLnNraWxsLjUyYjg2ZWJkLWRkN2QtNDVjMy1hNzYzLWRlNTg0ZjYyYjhkNiIsImV4cCI6MTU2OTgzNTY5MiwiaWF0IjoxNTY5ODM1MzkyLCJuYmYiOjE1Njk4MzUzOTIsInByaXZhdGVDbGFpbXMiOnsiY29udGV4dCI6IkFBQUFBQUFBQUFCTm5aUTd4b09EcGNYL0tuMDFpZ1F6S2dFQUFBQUFBQUJSazluemRVNTlQZWVFY0t5SERSZEwzRiszdnZrVGpQWWQ3MnhFYzFQcUNSeStTTWZmaFFscUh4azJuTHNTV01JKzFnZEtYc0t1RGVSQkJqNERTck5TUWVCZjNkbmtxNERWMXRqVjhmUnB1UWRXdlY2bERZN3YycXMyZVRlZEN6V0RLY21oRXFjRHdBNWlmdUxEdzB5bmZVVVh6Rk0yLzBBeDdGUmYxaS9FWXJRaWV0T2Q1dWllYU9RUFUrUUNMUUNRMFI0Ni9Ld1d1SWdxcE5sSGw0bU0xSHNhYXJOS3VzM0hDRzNyNm9LekxkT25EVUFKTDRtajkzSGwwZUhUQ1M0WDFySEtTTHNMNUlxa2hnUTk3a0R0WVovK1dNbkVDNklGUEZ6OHdYYU9jaDJYS05EUTNERVlGWTE0WHRkTXY0MlBYeTJlQ3VjQy9udnU2ZGMxaGRjUGdkZUp2Rmw3WlBBK0RSa2RqYXovL1NNTjVQMlNBY0NqK2JBZXIrTGZOTDByYUxhbGh5OEhleGl5IiwiY29uc2VudFRva2VuIjpudWxsLCJkZXZpY2VJZCI6ImFtem4xLmFzay5kZXZpY2UuQUg3NzdZS1BUV01OUUdWS1VLRFdQUU9XV0VEQkRKTk1JR1A1R0hEWE9JTUkzTjVSWVpXUTJIQlFFT1VYTVVKRUhSQktEWDZIQ0ZFQTdSUldOQUdLSEpMU0Q1S1dMVEtSMzVENDJUVzZCVkw2NFRIQ1lVSVRUSDNHNlpNV1o2R05BRUxUWFdCNFlBWkpXVUs0SjJCSUZWTFVQMktIWk5UUVJKUkJFRkdOV1k0VjJSQ0VFUU9aQyIsInVzZXJJZCI6ImFtem4xLmFzay5hY2NvdW50LkFIVUFKNVJSVEpEQVRQNjNBSVJMTk9WQkMyUUNKN1U1V1NWU0Q0MzJFQTQ1UERWV0FYNUNRNloyT0xEMkgyQTc3VlNCUUdJTUlXQVZCTVdMSEsyRVZaQUU1VlZKMkZIV1M0QVFNM0dNSURINjJHWkJaNERPVVdYQTNEWFJCQlhYWFRLQUlURFVDWlRMRzVHUDNYTjdZT1JFNUZRTzJNRVJHS0s3V0FKVVRIUE1MWU40VzJJVUJWWURJVzc1NDRNNTdONEtWNUhNUzRERVNNWSJ9fQ.brF2UpwjKMbYhR50WdoALbz0CM9hFtfAUw4Hh9-tOMJY8imui3oadv5S6QbQlfYD4_V_mJG2WOfkLmvirdRwdY6gI289WB48a6pK29VVcJWhYv1wIEpNQUMvMQqMZpjUuCI6DR9PqSeHulqPt14ytiA1ghOVSsAsHFXGbhNNeM9SdS1Ss0JQolSvXo09qC3JFRpDBI1bzBxRthhWEwgIEkC-JuFAbCbXz-710FkI4vzlMElgvC2GIsPf-5RaTJXps4UuG1rLieerirrrZfbpmhO0x2vDbLvBCCbqUtoHPyKofexfBXebvMjjJ7PRZvKYxAg3SBVZLvpGVl0prgJ8PA" + }, + "Viewport": { + "experiences": [ + { + "arcMinuteWidth": 246, + "arcMinuteHeight": 144, + "canRotate": False, + "canResize": False + } + ], + "shape": "RECTANGLE", + "pixelWidth": 1024, + "pixelHeight": 600, + "dpi": 160, + "currentPixelWidth": 1024, + "currentPixelHeight": 600, + "touch": [ + "SINGLE" + ], + "video": { + "codecs": [ + "H_264_42", + "H_264_41" + ] + } + } + }, + "request": { + "type": "LaunchRequest", + "requestId": "amzn1.echo-api.request.9b112eb9-eb11-433d-b6b3-8dba7eab9637", + "timestamp": "2019-09-30T09:23:12Z", + "locale": "en-US", + "shouldLinkResultBeReturned": False + } +} + +signature_header = Header(..., example=_signature_example, alias='Signature') +cert_chain_url_header = Header(..., example=_signature_cert_chain_url_example, alias='Signaturecertchainurl') +data_body = Body(..., example=_body_example) diff --git a/deeppavlov/utils/alexa/server.py b/deeppavlov/utils/alexa/server.py index 06e923d865..d569621664 100644 --- a/deeppavlov/utils/alexa/server.py +++ b/deeppavlov/utils/alexa/server.py @@ -12,16 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ssl +import asyncio from datetime import timedelta from logging import getLogger from pathlib import Path from queue import Queue from typing import Union, Optional -from flasgger import Swagger, swag_from -from flask import Flask, request, jsonify, redirect -from flask_cors import CORS +import uvicorn +import json +from fastapi import FastAPI +from starlette.responses import JSONResponse from deeppavlov.agents.default_agent.default_agent import DefaultAgent from deeppavlov.agents.processors.default_rich_content_processor import DefaultRichContentWrapper @@ -30,15 +31,16 @@ from deeppavlov.core.common.paths import get_settings_path from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill from deeppavlov.utils.alexa.bot import Bot +from deeppavlov.utils.alexa.request_parameters import data_body, cert_chain_url_header, signature_header +from deeppavlov.utils.server.server import get_ssl_params, redirect_root_do_docs SERVER_CONFIG_FILENAME = 'server_config.json' AMAZON_CERTIFICATE_LIFETIME = timedelta(hours=1) log = getLogger(__name__) - -app = Flask(__name__) -CORS(app) +uvicorn_log = getLogger('uvicorn') +app = FastAPI() def run_alexa_default_agent(model_config: Union[str, Path, dict], @@ -46,8 +48,8 @@ def run_alexa_default_agent(model_config: Union[str, Path, dict], stateful: bool = False, port: Optional[int] = None, https: bool = False, - ssl_key: str = None, - ssl_cert: str = None, + ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None, default_skill_wrap: bool = True) -> None: """Creates Alexa agents factory and initiates Alexa web service. @@ -58,13 +60,13 @@ def run_alexa_default_agent(model_config: Union[str, Path, dict], model_config: DeepPavlov config path. multi_instance: Multi instance mode flag. stateful: Stateful mode flag. - port: Flask web service port. + port: FastAPI web service port. https: Flag for running Alexa skill service in https mode. ssl_key: SSL key file path. ssl_cert: SSL certificate file path. default_skill_wrap: Wrap with default skill flag. - """ + """ def get_default_agent() -> DefaultAgent: model = build_model(model_config) skill = DefaultStatelessSkill(model) if default_skill_wrap else model @@ -85,54 +87,32 @@ def run_alexa_server(agent_generator: callable, stateful: bool = False, port: Optional[int] = None, https: bool = False, - ssl_key: str = None, - ssl_cert: str = None) -> None: - """Initiates Flask web service with Alexa skill. + ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None) -> None: + """Initiates FastAPI web service with Alexa skill. Args: agent_generator: Callback Alexa agents factory. multi_instance: Multi instance mode flag. stateful: Stateful mode flag. - port: Flask web service port. + port: FastAPI web service port. https: Flag for running Alexa skill service in https mode. ssl_key: SSL key file path. ssl_cert: SSL certificate file path. + """ server_config_path = Path(get_settings_path(), SERVER_CONFIG_FILENAME).resolve() server_params = read_json(server_config_path) host = server_params['common_defaults']['host'] port = port or server_params['common_defaults']['port'] - docs_endpoint = server_params['common_defaults']['docs_endpoint'] - - Swagger.DEFAULT_CONFIG['specs_route'] = docs_endpoint - Swagger(app) - alexa_server_params = server_params['alexa_defaults'] alexa_server_params['multi_instance'] = multi_instance or server_params['common_defaults']['multi_instance'] alexa_server_params['stateful'] = stateful or server_params['common_defaults']['stateful'] alexa_server_params['amazon_cert_lifetime'] = AMAZON_CERTIFICATE_LIFETIME - if https: - ssh_key_path = Path(ssl_key or server_params['https_key_path']).resolve() - if not ssh_key_path.is_file(): - e = FileNotFoundError('Ssh key file not found: please provide correct path in --key param or ' - 'https_key_path param in server configuration file') - log.error(e) - raise e - - ssh_cert_path = Path(ssl_cert or server_params['https_cert_path']).resolve() - if not ssh_cert_path.is_file(): - e = FileNotFoundError('Ssh certificate file not found: please provide correct path in --cert param or ' - 'https_cert_path param in server configuration file') - log.error(e) - raise e - - ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) - ssl_context.load_cert_chain(ssh_cert_path, ssh_key_path) - else: - ssl_context = None + ssl_config = get_ssl_params(server_params['common_defaults'], https, ssl_key=ssl_key, ssl_cert=ssl_cert) input_q = Queue() output_q = Queue() @@ -140,138 +120,28 @@ def run_alexa_server(agent_generator: callable, bot = Bot(agent_generator, alexa_server_params, input_q, output_q) bot.start() - endpoint_description = { - 'description': 'Amazon Alexa custom service endpoint', - 'parameters': [ - { - 'name': 'Signature', - 'in': 'header', - 'required': 'true', - 'type': 'string', - 'example': 'Z5H5wqd06ExFVPNfJiqhKvAFjkf+cTVodOUirucHGcEVAMO1LfvgqWUkZ/X1ITDZbI0w+SMwVkEQZlkeThbVS/54M22StNDUtfz4Ua20xNDpIPwcWIACAmZ38XxbbTEFJI5WwqrbilNcfzqiGrIPfdO5rl+/xUjHFUdcJdUY/QzBxXsceytVYfEiR9MzOCN2m4C0XnpThUavAu159KrLj8AkuzN0JF87iXv+zOEeZRgEuwmsAnJrRUwkJ4yWokEPnSVdjF0D6f6CscfyvRe9nsWShq7/zRTa41meweh+n006zvf58MbzRdXPB22RI4AN0ksWW7hSC8/QLAKQE+lvaw==', - }, - { - 'name': 'Signaturecertchainurl', - 'in': 'header', - 'required': 'true', - 'type': 'string', - 'example': 'https://s3.amazonaws.com/echo.api/echo-api-cert-6-ats.pem', - }, - { - 'name': 'data', - 'in': 'body', - 'required': 'true', - 'example': { - 'version': '1.0', - 'session': { - 'new': False, - 'sessionId': 'amzn1.echo-api.session.3c6ebffd-55b9-4e1a-bf3c-c921c1801b63', - 'application': { - 'applicationId': 'amzn1.ask.skill.8b17a5de-3749-4919-aa1f-e0bbaf8a46a6' - }, - 'attributes': { - 'sessionId': 'amzn1.echo-api.session.3c6ebffd-55b9-4e1a-bf3c-c921c1801b63' - }, - 'user': { - 'userId': 'amzn1.ask.account.AGR4R2LOVHMNMNOGROBVNLU7CL4C57X465XJF2T2F55OUXNTLCXDQP3I55UXZIALEKKZJ6Q2MA5MEFSMZVPEL5NVZS6FZLEU444BVOLPB5WVH5CHYTQAKGD7VFLGPRFZVHHH2NIB4HKNHHGX6HM6S6QDWCKXWOIZL7ONNQSBUCVPMZQKMCYXRG5BA2POYEXFDXRXCGEVDWVSMPQ' - } - }, - 'context': { - 'System': { - 'application': { - 'applicationId': 'amzn1.ask.skill.8b17a5de-3749-4919-aa1f-e0bbaf8a46a6' - }, - 'user': { - 'userId': 'amzn1.ask.account.AGR4R2LOVHMNMNOGROBVNLU7CL4C57X465XJF2T2F55OUXNTLCXDQP3I55UXZIALEKKZJ6Q2MA5MEFSMZVPEL5NVZS6FZLEU444BVOLPB5WVH5CHYTQAKGD7VFLGPRFZVHHH2NIB4HKNHHGX6HM6S6QDWCKXWOIZL7ONNQSBUCVPMZQKMCYXRG5BA2POYEXFDXRXCGEVDWVSMPQ' - }, - 'device': { - 'deviceId': 'amzn1.ask.device.AFQAMLYOYQUUACSE7HFVYS4ZI2KUB35JPHQRUPKTDCAU3A47WESP5L57KSWT5L6RT3FVXWH4OA2DNPJRMZ2VGEIACF3PJEIDCOUWUBC4W5RPJNUB3ZVT22J4UJN5UL3T2UBP36RVHFJ5P4IPT2HUY3P2YOY33IOU4O33HUAG7R2BUNROEH4T2', - 'supportedInterfaces': {} - }, - 'apiEndpoint': 'https://api.amazonalexa.com', - 'apiAccessToken': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6IjEifQ.eyJhdWQiOiJodHRwczovL2FwaS5hbWF6b25hbGV4YS5jb20iLCJpc3MiOiJBbGV4YVNraWxsS2l0Iiwic3ViIjoiYW16bjEuYXNrLnNraWxsLjhiMTdhNWRlLTM3NDktNDkxOS1hYTFmLWUwYmJhZjhhNDZhNiIsImV4cCI6MTU0NTIyMzY1OCwiaWF0IjoxNTQ1MjIwMDU4LCJuYmYiOjE1NDUyMjAwNTgsInByaXZhdGVDbGFpbXMiOnsiY29uc2VudFRva2VuIjpudWxsLCJkZXZpY2VJZCI6ImFtem4xLmFzay5kZXZpY2UuQUZRQU1MWU9ZUVVVQUNTRTdIRlZZUzRaSTJLVUIzNUpQSFFSVVBLVERDQVUzQTQ3V0VTUDVMNTdLU1dUNUw2UlQzRlZYV0g0T0EyRE5QSlJNWjJWR0VJQUNGM1BKRUlEQ09VV1VCQzRXNVJQSk5VQjNaVlQyMko0VUpONVVMM1QyVUJQMzZSVkhGSjVQNElQVDJIVVkzUDJZT1kzM0lPVTRPMzNIVUFHN1IyQlVOUk9FSDRUMiIsInVzZXJJZCI6ImFtem4xLmFzay5hY2NvdW50LkFHUjRSMkxPVkhNTk1OT0dST0JWTkxVN0NMNEM1N1g0NjVYSkYyVDJGNTVPVVhOVExDWERRUDNJNTVVWFpJQUxFS0taSjZRMk1BNU1FRlNNWlZQRUw1TlZaUzZGWkxFVTQ0NEJWT0xQQjVXVkg1Q0hZVFFBS0dEN1ZGTEdQUkZaVkhISDJOSUI0SEtOSEhHWDZITTZTNlFEV0NLWFdPSVpMN09OTlFTQlVDVlBNWlFLTUNZWFJHNUJBMlBPWUVYRkRYUlhDR0VWRFdWU01QUSJ9fQ.jcomYhBhU485T4uoe2NyhWnL-kZHoPQKpcycFqa-1sy_lSIitfFGup9DKrf2NkN-I9lZ3xwq9llqx9WRN78fVJjN6GLcDhBDH0irPwt3n9_V7_5bfB6KARv5ZG-JKOmZlLBqQbnln0DAJ10D8HNiytMARNEwduMBVDNK0A5z6YxtRcLYYFD2-Ieg_V8Qx90eE2pd2U5xOuIEL0pXfSoiJ8vpxb8BKwaMO47tdE4qhg_k7v8ClwyXg3EMEhZFjixYNqdW1tCrwDGj58IWMXDyzZhIlRMh6uudMOT6scSzcNVD0v42IOTZ3S_X6rG01B7xhUDlZXMqkrCuzOyqctGaPw' - }, - 'Viewport': { - 'experiences': [ - { - 'arcMinuteWidth': 246, - 'arcMinuteHeight': 144, - 'canRotate': False, - 'canResize': False - } - ], - 'shape': 'RECTANGLE', - 'pixelWidth': 1024, - 'pixelHeight': 600, - 'dpi': 160, - 'currentPixelWidth': 1024, - 'currentPixelHeight': 600, - 'touch': [ - 'SINGLE' - ] - } - }, - 'request': { - 'type': 'IntentRequest', - 'requestId': 'amzn1.echo-api.request.388d0f6e-04b9-4450-a687-b9abaa73ac6a', - 'timestamp': '2018-12-19T11:47:38Z', - 'locale': 'en-US', - 'intent': { - 'name': 'AskDeepPavlov', - 'confirmationStatus': 'NONE', - 'slots': { - 'raw_input': { - 'name': 'raw_input', - 'value': 'my beautiful sandbox skill', - 'resolutions': { - 'resolutionsPerAuthority': [ - { - 'authority': 'amzn1.er-authority.echo-sdk.amzn1.ask.skill.8b17a5de-3749-4919-aa1f-e0bbaf8a46a6.GetInput', - 'status': { - 'code': 'ER_SUCCESS_NO_MATCH' - } - } - ] - }, - 'confirmationStatus': 'NONE', - 'source': 'USER' - } - } - } - } - } - } - ], - 'responses': { - "200": { - "description": "A model response" - } - } - } - - @app.route('/') - def index(): - return redirect(docs_endpoint) - - @app.route('/interact', methods=['POST']) - @swag_from(endpoint_description) - def handle_request(): - request_body: bytes = request.get_data() - signature_chain_url: str = request.headers.get('Signaturecertchainurl') - signature: str = request.headers.get('Signature') - alexa_request: dict = request.get_json() + endpoint = '/interact' + redirect_root_do_docs(app, 'interact', endpoint, 'post') + @app.post(endpoint, summary='Amazon Alexa custom service endpoint', response_description='A model response') + async def interact(data: dict = data_body, + signature: str = signature_header, + signature_chain_url: str = cert_chain_url_header) -> JSONResponse: + # It is necessary for correct data validation to serialize data to a JSON formatted string with separators. request_dict = { - 'request_body': request_body, + 'request_body': json.dumps(data, separators=(',', ':')).encode('utf-8'), 'signature_chain_url': signature_chain_url, 'signature': signature, - 'alexa_request': alexa_request + 'alexa_request': data } bot.input_queue.put(request_dict) - response: dict = bot.output_queue.get() + loop = asyncio.get_event_loop() + response: dict = await loop.run_in_executor(None, bot.output_queue.get) response_code = 400 if 'error' in response.keys() else 200 - return jsonify(response), response_code + return JSONResponse(response, status_code=response_code) - app.run(host=host, port=port, threaded=True, ssl_context=ssl_context) + uvicorn.run(app, host=host, port=port, logger=uvicorn_log, ssl_version=ssl_config.version, + ssl_keyfile=ssl_config.keyfile, ssl_certfile=ssl_config.certfile) + bot.join() diff --git a/deeppavlov/utils/alice/alice.py b/deeppavlov/utils/alice/alice.py index 332c5bce0d..fb2062a1b6 100644 --- a/deeppavlov/utils/alice/alice.py +++ b/deeppavlov/utils/alice/alice.py @@ -12,15 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ssl +import asyncio from collections import namedtuple from logging import getLogger from pathlib import Path -from typing import Union, Optional +from typing import Dict, Union, Optional -from flasgger import Swagger, swag_from -from flask import Flask, request, jsonify, redirect -from flask_cors import CORS +import uvicorn +from fastapi import FastAPI from deeppavlov import build_model from deeppavlov.agents.default_agent.default_agent import DefaultAgent @@ -29,24 +28,23 @@ from deeppavlov.core.agent.rich_content import RichMessage from deeppavlov.core.common.paths import get_settings_path from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill -from deeppavlov.utils.server.server import get_server_params +from deeppavlov.utils.server.server import SSLConfig, get_server_params, get_ssl_params, redirect_root_do_docs +from deeppavlov.utils.alice.request_parameters import data_body SERVER_CONFIG_FILENAME = 'server_config.json' log = getLogger(__name__) - -app = Flask(__name__) -CORS(app) +uvicorn_log = getLogger('uvicorn') +app = FastAPI() DialogID = namedtuple('DialogID', ['user_id', 'session_id']) -def interact_alice(agent: Agent): +def interact_alice(agent: Agent, data: Dict) -> Dict: """ Exchange messages between basic pipelines and the Yandex.Dialogs service. If the pipeline returns multiple values, only the first one is forwarded to Yandex. """ - data = request.get_json() text = data['request'].get('command', '').strip() payload = data['request'].get('payload') @@ -56,12 +54,18 @@ def interact_alice(agent: Agent): dialog_id = DialogID(user_id, session_id) + agent_response: Union[str, RichMessage] = agent([payload or text], [dialog_id])[0] + if isinstance(agent_response, RichMessage): + response_text = '\n'.join([j['content'] for j in agent_response.json() if j['type'] == 'plain_text']) + else: + response_text = str(agent_response) + response = { 'response': { 'end_session': False, - 'text': '' + 'text': response_text }, - "session": { + 'session': { 'session_id': session_id, 'message_id': message_id, 'user_id': user_id @@ -69,113 +73,54 @@ def interact_alice(agent: Agent): 'version': '1.0' } - agent_response: Union[str, RichMessage] = agent([payload or text], [dialog_id])[0] - if isinstance(agent_response, RichMessage): - response['response']['text'] = '\n'.join([j['content'] - for j in agent_response.json() - if j['type'] == 'plain_text']) - else: - response['response']['text'] = str(agent_response) - - return jsonify(response), 200 + return response -def start_alice_server(model_config, https=False, ssl_key=None, ssl_cert=None, port=None): +def start_alice_server(model_config: Path, + https: bool = False, + ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None, + port: Optional[int] = None) -> None: server_config_path = get_settings_path() / SERVER_CONFIG_FILENAME server_params = get_server_params(server_config_path, model_config) https = https or server_params['https'] - docs_endpoint = server_params['docs_endpoint'] - - Swagger.DEFAULT_CONFIG['specs_route'] = docs_endpoint - Swagger(app) - - if not https: - ssl_key = ssl_cert = None - else: - ssh_key = Path(ssl_key or server_params['https_key_path']).resolve() - if not ssh_key.is_file(): - e = FileNotFoundError('Ssh key file not found: please provide correct path in --key param or ' - 'https_key_path param in server configuration file') - log.error(e) - raise e - - ssh_cert = Path(ssl_cert or server_params['https_cert_path']).resolve() - if not ssh_cert.is_file(): - e = FileNotFoundError('Ssh certificate file not found: please provide correct path in --cert param or ' - 'https_cert_path param in server configuration file') - log.error(e) - raise e - host = server_params['host'] port = port or server_params['port'] model_endpoint = server_params['model_endpoint'] + ssl_config = get_ssl_params(server_params, https, ssl_key=ssl_key, ssl_cert=ssl_cert) + model = build_model(model_config) skill = DefaultStatelessSkill(model, lang='ru') agent = DefaultAgent([skill], skills_processor=DefaultRichContentWrapper()) - @app.route('/') - def index(): - return redirect(docs_endpoint) + start_agent_server(agent, host, port, model_endpoint, ssl_config=ssl_config) + - start_agent_server(agent, host, port, model_endpoint, ssl_key, ssl_cert) +def start_agent_server(agent: Agent, + host: str, + port: int, + endpoint: str, + ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None, + ssl_config: Optional[SSLConfig] = None) -> None: + if ssl_key and ssl_cert and ssl_config: + raise ValueError('ssl_key, ssl_cert, ssl_config was assigned at the same time. Please, use either' + 'ssl_config or ssl_key and ssl_cert') -def start_agent_server(agent: Agent, host: str, port: int, endpoint: str, - ssl_key: Optional[Path] = None, - ssl_cert: Optional[Path] = None): if ssl_key and ssl_cert: - ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) - ssh_key_path = Path(ssl_key).resolve() - ssh_cert_path = Path(ssl_cert).resolve() - ssl_context.load_cert_chain(ssh_cert_path, ssh_key_path) + ssl_config = get_ssl_params({}, True, ssl_key=ssl_key, ssl_cert=ssl_cert) else: - ssl_context = None - - endpoint_description = { - 'description': 'A model endpoint', - 'parameters': [ - { - 'name': 'data', - 'in': 'body', - 'required': 'true', - 'example': { - 'meta': { - 'locale': 'ru-RU', - 'timezone': 'Europe/Moscow', - "client_id": 'ru.yandex.searchplugin/5.80 (Samsung Galaxy; Android 4.4)' - }, - 'request': { - 'command': 'где ближайшее отделение', - 'original_utterance': 'Алиса спроси у Сбербанка где ближайшее отделение', - 'type': 'SimpleUtterance', - 'markup': { - 'dangerous_context': True - }, - 'payload': {} - }, - 'session': { - 'new': True, - 'message_id': 4, - 'session_id': '2eac4854-fce721f3-b845abba-20d60', - 'skill_id': '3ad36498-f5rd-4079-a14b-788652932056', - 'user_id': 'AC9WC3DF6FCE052E45A4566A48E6B7193774B84814CE49A922E163B8B29881DC' - }, - 'version': '1.0' - } - } - ], - 'responses': { - "200": { - "description": "A model response" - } - } - } + ssl_config = ssl_config or get_ssl_params({}, False, ssl_key=None, ssl_cert=None) + + redirect_root_do_docs(app, 'answer', endpoint, 'post') - @app.route(endpoint, methods=['POST']) - @swag_from(endpoint_description) - def answer(): - return interact_alice(agent) + @app.post(endpoint, summary='A model endpoint', response_description='A model response') + async def answer(data: dict = data_body) -> dict: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, interact_alice, agent, data) - app.run(host=host, port=port, threaded=False, ssl_context=ssl_context) + uvicorn.run(app, host=host, port=port, logger=uvicorn_log, ssl_version=ssl_config.version, + ssl_keyfile=ssl_config.keyfile, ssl_certfile=ssl_config.certfile) diff --git a/deeppavlov/utils/alice/request_parameters.py b/deeppavlov/utils/alice/request_parameters.py new file mode 100644 index 0000000000..6974c581aa --- /dev/null +++ b/deeppavlov/utils/alice/request_parameters.py @@ -0,0 +1,57 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Request parameters for the DeepPavlov model launched as a skill for Yandex.Alice. + +Request parameters from this module are used to declare additional information +and validation for request parameters to the DeepPavlov model launched as +a skill for Yandex.Alice. + +See details at https://fastapi.tiangolo.com/tutorial/body-multiple-params/ + +""" + +from fastapi import Body + +_body_example = { + 'name': 'data', + 'in': 'body', + 'required': 'true', + 'example': { + 'meta': { + 'locale': 'ru-RU', + 'timezone': 'Europe/Moscow', + "client_id": 'ru.yandex.searchplugin/5.80 (Samsung Galaxy; Android 4.4)' + }, + 'request': { + 'command': 'где ближайшее отделение', + 'original_utterance': 'Алиса спроси у Сбербанка где ближайшее отделение', + 'type': 'SimpleUtterance', + 'markup': { + 'dangerous_context': True + }, + 'payload': {} + }, + 'session': { + 'new': True, + 'message_id': 4, + 'session_id': '2eac4854-fce721f3-b845abba-20d60', + 'skill_id': '3ad36498-f5rd-4079-a14b-788652932056', + 'user_id': 'AC9WC3DF6FCE052E45A4566A48E6B7193774B84814CE49A922E163B8B29881DC' + }, + 'version': '1.0' + } +} + +data_body = Body(..., example=_body_example) diff --git a/deeppavlov/utils/ms_bot_framework/bot.py b/deeppavlov/utils/ms_bot_framework/bot.py index 79abd44e5c..811ea61cf3 100644 --- a/deeppavlov/utils/ms_bot_framework/bot.py +++ b/deeppavlov/utils/ms_bot_framework/bot.py @@ -1,7 +1,7 @@ import threading from collections import namedtuple from logging import getLogger -from queue import Queue +from queue import Empty, Queue from threading import Thread import requests @@ -18,6 +18,7 @@ class Bot(Thread): def __init__(self, agent_generator: callable, config: dict, input_queue: Queue): super(Bot, self).__init__() self.config = config + self._run_flag = True self.conversations = {} self.access_info = {} @@ -37,9 +38,20 @@ def __init__(self, agent_generator: callable, config: dict, input_queue: Queue): self.timer.start() def run(self): - while True: - activity = self.input_queue.get() - self._handle_activity(activity) + while self._run_flag: + try: + activity = self.input_queue.get(timeout=1) + except Empty: + pass + else: + self._handle_activity(activity) + + def join(self, timeout=None): + self._run_flag = False + for timer in threading.enumerate(): + if isinstance(timer, threading.Timer): + timer.cancel() + Thread.join(self, timeout) def del_conversation(self, conversation_key: ConvKey): del self.conversations[conversation_key] diff --git a/deeppavlov/utils/ms_bot_framework/server.py b/deeppavlov/utils/ms_bot_framework/server.py index 3b3400281c..f19df071ea 100644 --- a/deeppavlov/utils/ms_bot_framework/server.py +++ b/deeppavlov/utils/ms_bot_framework/server.py @@ -1,12 +1,24 @@ -import ssl +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from logging import getLogger from pathlib import Path from queue import Queue from typing import Union, Optional -from flasgger import Swagger -from flask import Flask, request, jsonify, redirect -from flask_cors import CORS +import uvicorn +from fastapi import FastAPI from deeppavlov.agents.default_agent.default_agent import DefaultAgent from deeppavlov.agents.processors.default_rich_content_processor import DefaultRichContentWrapper @@ -15,6 +27,7 @@ from deeppavlov.core.common.paths import get_settings_path from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill from deeppavlov.utils.ms_bot_framework.bot import Bot +from deeppavlov.utils.server.server import get_ssl_params, redirect_root_do_docs SERVER_CONFIG_FILENAME = 'server_config.json' @@ -25,9 +38,8 @@ AUTH_SCOPE = "https://api.botframework.com/.default" log = getLogger(__name__) - -app = Flask(__name__) -CORS(app) +uvicorn_log = getLogger('uvicorn') +app = FastAPI() def run_ms_bf_default_agent(model_config: Union[str, Path, dict], @@ -37,11 +49,11 @@ def run_ms_bf_default_agent(model_config: Union[str, Path, dict], stateful: bool = False, port: Optional[int] = None, https: bool = False, - ssl_key: str = None, - ssl_cert: str = None, - default_skill_wrap: bool = True): + ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None, + default_skill_wrap: bool = True) -> None: - def get_default_agent(): + def get_default_agent() -> DefaultAgent: model = build_model(model_config) skill = DefaultStatelessSkill(model) if default_skill_wrap else model agent = DefaultAgent([skill], skills_processor=DefaultRichContentWrapper()) @@ -65,18 +77,14 @@ def run_ms_bot_framework_server(agent_generator: callable, stateful: bool = False, port: Optional[int] = None, https: bool = False, - ssl_key: str = None, - ssl_cert: str = None): + ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None) -> None: server_config_path = Path(get_settings_path(), SERVER_CONFIG_FILENAME).resolve() server_params = read_json(server_config_path) host = server_params['common_defaults']['host'] port = port or server_params['common_defaults']['port'] - docs_endpoint = server_params['common_defaults']['docs_endpoint'] - - Swagger.DEFAULT_CONFIG['specs_route'] = docs_endpoint - Swagger(app) ms_bf_server_params = server_params['ms_bot_framework_defaults'] @@ -103,38 +111,20 @@ def run_ms_bot_framework_server(agent_generator: callable, log.error(e) raise e - if https: - ssh_key_path = Path(ssl_key or server_params['https_key_path']).resolve() - if not ssh_key_path.is_file(): - e = FileNotFoundError('Ssh key file not found: please provide correct path in --key param or ' - 'https_key_path param in server configuration file') - log.error(e) - raise e - - ssh_cert_path = Path(ssl_cert or server_params['https_cert_path']).resolve() - if not ssh_cert_path.is_file(): - e = FileNotFoundError('Ssh certificate file not found: please provide correct path in --cert param or ' - 'https_cert_path param in server configuration file') - log.error(e) - raise e - - ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) - ssl_context.load_cert_chain(ssh_cert_path, ssh_key_path) - else: - ssl_context = None + ssl_config = get_ssl_params(server_params['common_defaults'], https, ssl_key=ssl_key, ssl_cert=ssl_cert) input_q = Queue() bot = Bot(agent_generator, ms_bf_server_params, input_q) bot.start() - @app.route('/') - def index(): - return redirect(docs_endpoint) + endpoint = '/v3/conversations' + redirect_root_do_docs(app, 'answer', endpoint, 'post') - @app.route('/v3/conversations', methods=['POST']) - def handle_activity(): - activity = request.get_json() + @app.post(endpoint) + async def answer(activity: dict) -> dict: bot.input_queue.put(activity) - return jsonify({}), 200 + return {} - app.run(host=host, port=port, threaded=True, ssl_context=ssl_context) + uvicorn.run(app, host=host, port=port, logger=uvicorn_log, ssl_version=ssl_config.version, + ssl_keyfile=ssl_config.keyfile, ssl_certfile=ssl_config.certfile) + bot.join() diff --git a/deeppavlov/utils/server/server.py b/deeppavlov/utils/server/server.py index 1c66e91cf8..12a5dc50ad 100644 --- a/deeppavlov/utils/server/server.py +++ b/deeppavlov/utils/server/server.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import logging +from collections import namedtuple from pathlib import Path from ssl import PROTOCOL_TLSv1_2 from typing import Dict, List, Optional @@ -23,7 +25,7 @@ from pydantic import BaseConfig, BaseModel, Schema from pydantic.fields import Field from pydantic.main import MetaModel -from starlette.responses import JSONResponse, RedirectResponse +from starlette.responses import RedirectResponse from deeppavlov.core.agent.dialog_logger import DialogLogger from deeppavlov.core.commands.infer import build_model @@ -34,6 +36,7 @@ from deeppavlov.core.data.utils import check_nested_dict_keys, jsonify_data SERVER_CONFIG_FILENAME = 'server_config.json' +SSLConfig = namedtuple('SSLConfig', ['version', 'keyfile', 'certfile']) class ProbeFilter(logging.Filter): @@ -75,7 +78,43 @@ def get_server_params(server_config_path: Path, model_config: Path) -> Dict: return server_params -def interact(model: Chainer, payload: Dict[str, Optional[List]]) -> JSONResponse: +def get_ssl_params(server_params: dict, + https: Optional[bool], + ssl_key: Optional[str], + ssl_cert: Optional[str]) -> SSLConfig: + https = https or server_params['https'] + if https: + ssh_key_path = Path(ssl_key or server_params['https_key_path']).resolve() + if not ssh_key_path.is_file(): + e = FileNotFoundError('Ssh key file not found: please provide correct path in --key param or ' + 'https_key_path param in server configuration file') + log.error(e) + raise e + + ssh_cert_path = Path(ssl_cert or server_params['https_cert_path']).resolve() + if not ssh_cert_path.is_file(): + e = FileNotFoundError('Ssh certificate file not found: please provide correct path in --cert param or ' + 'https_cert_path param in server configuration file') + log.error(e) + raise e + + ssl_config = SSLConfig(version=PROTOCOL_TLSv1_2, keyfile=str(ssh_key_path), certfile=str(ssh_cert_path)) + else: + ssl_config = SSLConfig(None, None, None) + + return ssl_config + + +def redirect_root_do_docs(fast_app: FastAPI, func_name: str, endpoint: str, method: str) -> None: + """Adds api route to server that redirects user from root to docs with opened `endpoint` description.""" + @fast_app.get('/', include_in_schema=False) + async def redirect_to_docs() -> RedirectResponse: + operation_id = generate_operation_id_for_path(name=func_name, path=endpoint, method=method) + response = RedirectResponse(url=f'/docs#/default/{operation_id}') + return response + + +def interact(model: Chainer, payload: Dict[str, Optional[List]]) -> List: model_args = payload.values() dialog_logger.log_in(payload) error_msg = None @@ -101,53 +140,32 @@ def interact(model: Chainer, payload: Dict[str, Optional[List]]) -> JSONResponse prediction = list(zip(*prediction)) result = jsonify_data(prediction) dialog_logger.log_out(result) - return JSONResponse(result) + return result -def test_interact(model: Chainer, payload: Dict[str, Optional[List]]) -> JSONResponse: +def test_interact(model: Chainer, payload: Dict[str, Optional[List]]) -> List[str]: model_args = [arg or ["Test string."] for arg in payload.values()] try: _ = model(*model_args) - return JSONResponse(["Test passed"]) + return ["Test passed"] except Exception as e: raise HTTPException(status_code=400, detail=e) -def start_model_server(model_config: Path, https: bool = False, ssl_key: Optional[str] = None, - ssl_cert: Optional[str] = None, port: Optional[int] = None) -> None: +def start_model_server(model_config: Path, + https: bool = False, + ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None, + port: Optional[int] = None) -> None: server_config_path = get_settings_path() / SERVER_CONFIG_FILENAME server_params = get_server_params(server_config_path, model_config) host = server_params['host'] port = port or server_params['port'] model_endpoint = server_params['model_endpoint'] - docs_endpoint = server_params['docs_endpoint'] model_args_names = server_params['model_args_names'] - https = https or server_params['https'] - - if https: - ssh_key_path = Path(ssl_key or server_params['https_key_path']).resolve() - if not ssh_key_path.is_file(): - e = FileNotFoundError('Ssh key file not found: please provide correct path in --key param or ' - 'https_key_path param in server configuration file') - log.error(e) - raise e - - ssh_cert_path = Path(ssl_cert or server_params['https_cert_path']).resolve() - if not ssh_cert_path.is_file(): - e = FileNotFoundError('Ssh certificate file not found: please provide correct path in --cert param or ' - 'https_cert_path param in server configuration file') - log.error(e) - raise e - - ssl_version = PROTOCOL_TLSv1_2 - ssl_keyfile = str(ssh_key_path) - ssl_certfile = str(ssh_cert_path) - else: - ssl_version = None - ssl_keyfile = None - ssl_certfile = None + ssl_config = get_ssl_params(server_params, https, ssl_key=ssl_key, ssl_cert=ssl_cert) model = build_model(model_config) @@ -162,23 +180,21 @@ def batch_decorator(cls: MetaModel) -> MetaModel: class Batch(BaseModel): pass - @app.get('/', include_in_schema=False) - async def redirect_to_docs() -> RedirectResponse: - operation_id = generate_operation_id_for_path(name='answer', path=model_endpoint, method='post') - response = RedirectResponse(url=f'{docs_endpoint}#/default/{operation_id}') - return response + redirect_root_do_docs(app, 'answer', model_endpoint, 'post') - @app.post(model_endpoint, status_code=200, summary='A model endpoint') - async def answer(item: Batch) -> JSONResponse: - return interact(model, item.dict()) + @app.post(model_endpoint, summary='A model endpoint') + async def answer(item: Batch) -> List: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, interact, model, item.dict()) - @app.post('/probe', status_code=200, include_in_schema=False) - async def probe(item: Batch) -> JSONResponse: - return test_interact(model, item.dict()) + @app.post('/probe', include_in_schema=False) + async def probe(item: Batch) -> List[str]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, test_interact, model, item.dict()) - @app.get('/api', status_code=200, summary='Model argument names') - async def api() -> JSONResponse: - return JSONResponse(model_args_names) + @app.get('/api', summary='Model argument names') + async def api() -> List[str]: + return model_args_names - uvicorn.run(app, host=host, port=port, logger=uvicorn_log, - ssl_version=ssl_version, ssl_keyfile=ssl_keyfile, ssl_certfile=ssl_certfile) + uvicorn.run(app, host=host, port=port, logger=uvicorn_log, ssl_version=ssl_config.version, + ssl_keyfile=ssl_config.keyfile, ssl_certfile=ssl_config.certfile) diff --git a/deeppavlov/utils/settings/server_config.json b/deeppavlov/utils/settings/server_config.json index 4dd43c9a13..67920833d2 100644 --- a/deeppavlov/utils/settings/server_config.json +++ b/deeppavlov/utils/settings/server_config.json @@ -2,7 +2,6 @@ "common_defaults": { "host": "0.0.0.0", "port": 5000, - "docs_endpoint": "/docs", "model_args_names": "", "https": false, "https_cert_path": "", diff --git a/docs/integrations/aws_ec2.rst b/docs/integrations/aws_ec2.rst index be69ee2122..1f8ed8b815 100644 --- a/docs/integrations/aws_ec2.rst +++ b/docs/integrations/aws_ec2.rst @@ -127,7 +127,7 @@ Deployment process consists of two main stages: 1. Get your AWS instance public DNS from the instance dashboard. -2. Get full info about your ODQA API from its Flasgger by navigating to +2. Get full info about your ODQA API from its Swagger by navigating to following URL in your browser: ``http://:`` diff --git a/docs/integrations/ms_bot.rst b/docs/integrations/ms_bot.rst index 4954f656bb..67786bd1aa 100644 --- a/docs/integrations/ms_bot.rst +++ b/docs/integrations/ms_bot.rst @@ -51,8 +51,8 @@ The whole process takes two main steps: to the *Messaging endpoint* pane. Note, that Microsoft Bot Framework requires https endpoint with valid certificate from CA. - 3.3 Save somewhere *Microsoft App ID* (*App ID*). To get *App Secret* - you need to proceed to the *Manage* ling near the *Microsoft App ID* pane. + 3.3 Save somewhere *Microsoft App ID* (*App ID*). To create *App Secret* + you need to proceed to the *Manage* link near the *Microsoft App ID* pane. You will need both during your DeepPavlov skill/model REST service start. .. image:: ../_static/ms_bot_framework/04_bot_settings.png diff --git a/docs/integrations/rest_api.rst b/docs/integrations/rest_api.rst index d91ce0b810..66f6dba582 100644 --- a/docs/integrations/rest_api.rst +++ b/docs/integrations/rest_api.rst @@ -42,7 +42,7 @@ will return list with argument names. """"" To interact with the REST API via graphical interface open -``:/docs`` in a browser (Flasgger UI). +``:/docs`` in a browser (Swagger UI). Advanced configuration diff --git a/requirements.txt b/requirements.txt index da2d078c7b..296d4840db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,5 @@ Cython==0.29.12 fastapi==0.38.1 -flasgger==0.9.2 -flask==1.1.1 -flask_cors==3.0.8 fuzzywuzzy==0.17.0 h5py==2.9.0 keras==2.2.4 From 5379dd6aefd589220fe02655ccf8a48d92df3c20 Mon Sep 17 00:00:00 2001 From: Dilyara Baymurzina Date: Tue, 1 Oct 2019 11:05:45 +0300 Subject: [PATCH 02/28] feat: add sst and yelp sentiment models on conv and mulan berts (#1018) * feat: enbert and multibert classifiers for fine grained sst with 5 classes * fix: download paths for sst * feat: yelp configs for bert and multi-bert * feat: yelp configs for bert and multi-bert * feat: config for conv bert on yelp * feat: config for conv bert on yelp * feat: config for conv bert on yelp * feat: config for conv bert on yelp * chore: seq length for bert sentiment on yelp * feat: new confif for conversational bert on SST * feat: rename * feat: new configs for semeval2017 * fix: download of dataset path * fix: correct metrics in configs * fix: sets-accuracy to accuracy after merge to dev * fix: f1 macro metrics * fix: configs * fix: removed unused configs * feat: info about new english sentiment models * feat: tests for new models * feat: docs * fix: config name in tests Co-Authored-By: Aleksei Lymar --- .../classifiers/sentiment_sst_conv_bert.json | 143 ++++++++++++++++ .../classifiers/sentiment_sst_multi_bert.json | 143 ++++++++++++++++ .../classifiers/sentiment_yelp_conv_bert.json | 157 ++++++++++++++++++ .../sentiment_yelp_multi_bert.json | 157 ++++++++++++++++++ docs/features/models/classifiers.rst | 21 ++- docs/features/overview.rst | 10 ++ tests/test_quick_start.py | 6 +- 7 files changed, 635 insertions(+), 2 deletions(-) create mode 100644 deeppavlov/configs/classifiers/sentiment_sst_conv_bert.json create mode 100644 deeppavlov/configs/classifiers/sentiment_sst_multi_bert.json create mode 100644 deeppavlov/configs/classifiers/sentiment_yelp_conv_bert.json create mode 100644 deeppavlov/configs/classifiers/sentiment_yelp_multi_bert.json diff --git a/deeppavlov/configs/classifiers/sentiment_sst_conv_bert.json b/deeppavlov/configs/classifiers/sentiment_sst_conv_bert.json new file mode 100644 index 0000000000..e907b240cd --- /dev/null +++ b/deeppavlov/configs/classifiers/sentiment_sst_conv_bert.json @@ -0,0 +1,143 @@ +{ + "dataset_reader": { + "class_name": "basic_classification_reader", + "x": "text", + "y": "fine_grained_label", + "data_path": "{DOWNLOADS_PATH}/stanfordSentimentTreebank", + "train": "train_fine_grained.csv", + "valid": "valid_fine_grained.csv", + "test": "test_fine_grained.csv" + }, + "dataset_iterator": { + "class_name": "basic_classification_iterator", + "seed": 42 + }, + "chainer": { + "in": [ + "x" + ], + "in_y": [ + "y" + ], + "pipe": [ + { + "class_name": "bert_preprocessor", + "vocab_file": "{MODEL_PATH}/vocab.txt", + "do_lower_case": false, + "max_seq_length": 64, + "in": [ + "x" + ], + "out": [ + "bert_features" + ] + }, + { + "id": "classes_vocab", + "class_name": "simple_vocab", + "fit_on": [ + "y" + ], + "save_path": "{MODEL_PATH}/classes.dict", + "load_path": "{MODEL_PATH}/classes.dict", + "in": "y", + "out": "y_ids" + }, + { + "in": "y_ids", + "out": "y_onehot", + "class_name": "one_hotter", + "depth": "#classes_vocab.len", + "single_vector": true + }, + { + "class_name": "bert_classifier", + "n_classes": "#classes_vocab.len", + "return_probas": true, + "one_hot_labels": true, + "bert_config_file": "{MODEL_PATH}/bert_config.json", + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "keep_prob": 0.5, + "learning_rate": 1e-05, + "learning_rate_drop_patience": 5, + "learning_rate_drop_div": 2.0, + "in": [ + "bert_features" + ], + "in_y": [ + "y_onehot" + ], + "out": [ + "y_pred_probas" + ] + }, + { + "in": "y_pred_probas", + "out": "y_pred_ids", + "class_name": "proba2labels", + "max_proba": true + }, + { + "in": "y_pred_ids", + "out": "y_pred_labels", + "ref": "classes_vocab" + } + ], + "out": [ + "y_pred_labels" + ] + }, + "train": { + "epochs": 100, + "batch_size": 64, + "metrics": [ + "accuracy", + { + "name": "roc_auc", + "inputs": [ + "y_onehot", + "y_pred_probas" + ] + }, + "f1_macro" + ], + "validation_patience": 5, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": [ + "train", + "valid", + "test" + ], + "class_name": "nn_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/" + }, + "metadata": { + "variables": { + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/classifiers/sentiment_sst_bert_v2" + }, + "requirements": [ + "{DEEPPAVLOV_PATH}/requirements/tf.txt", + "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" + ], + "labels": { + "telegram_utils": "IntentModel", + "server_utils": "KerasIntentModel" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/datasets/stanfordSentimentTreebank.zip", + "subdir": "{DOWNLOADS_PATH}" + }, + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/classifiers/sentiment_sst_bert_v2.tar.gz", + "subdir": "{MODELS_PATH}/classifiers" + } + ] + } +} diff --git a/deeppavlov/configs/classifiers/sentiment_sst_multi_bert.json b/deeppavlov/configs/classifiers/sentiment_sst_multi_bert.json new file mode 100644 index 0000000000..95e7a59ef0 --- /dev/null +++ b/deeppavlov/configs/classifiers/sentiment_sst_multi_bert.json @@ -0,0 +1,143 @@ +{ + "dataset_reader": { + "class_name": "basic_classification_reader", + "x": "text", + "y": "fine_grained_label", + "data_path": "{DOWNLOADS_PATH}/stanfordSentimentTreebank", + "train": "train_fine_grained.csv", + "valid": "valid_fine_grained.csv", + "test": "test_fine_grained.csv" + }, + "dataset_iterator": { + "class_name": "basic_classification_iterator", + "seed": 42 + }, + "chainer": { + "in": [ + "x" + ], + "in_y": [ + "y" + ], + "pipe": [ + { + "class_name": "bert_preprocessor", + "vocab_file": "{MODEL_PATH}/vocab.txt", + "do_lower_case": false, + "max_seq_length": 64, + "in": [ + "x" + ], + "out": [ + "bert_features" + ] + }, + { + "id": "classes_vocab", + "class_name": "simple_vocab", + "fit_on": [ + "y" + ], + "save_path": "{MODEL_PATH}/classes.dict", + "load_path": "{MODEL_PATH}/classes.dict", + "in": "y", + "out": "y_ids" + }, + { + "in": "y_ids", + "out": "y_onehot", + "class_name": "one_hotter", + "depth": "#classes_vocab.len", + "single_vector": true + }, + { + "class_name": "bert_classifier", + "n_classes": "#classes_vocab.len", + "return_probas": true, + "one_hot_labels": true, + "bert_config_file": "{MODEL_PATH}/bert_config.json", + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "keep_prob": 0.5, + "learning_rate": 1e-05, + "learning_rate_drop_patience": 5, + "learning_rate_drop_div": 2.0, + "in": [ + "bert_features" + ], + "in_y": [ + "y_onehot" + ], + "out": [ + "y_pred_probas" + ] + }, + { + "in": "y_pred_probas", + "out": "y_pred_ids", + "class_name": "proba2labels", + "max_proba": true + }, + { + "in": "y_pred_ids", + "out": "y_pred_labels", + "ref": "classes_vocab" + } + ], + "out": [ + "y_pred_labels" + ] + }, + "train": { + "epochs": 100, + "batch_size": 64, + "metrics": [ + "accuracy", + { + "name": "roc_auc", + "inputs": [ + "y_onehot", + "y_pred_probas" + ] + }, + "f1_macro" + ], + "validation_patience": 5, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": [ + "train", + "valid", + "test" + ], + "class_name": "nn_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/" + }, + "metadata": { + "variables": { + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/classifiers/sentiment_sst_bert_v1" + }, + "requirements": [ + "{DEEPPAVLOV_PATH}/requirements/tf.txt", + "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" + ], + "labels": { + "telegram_utils": "IntentModel", + "server_utils": "KerasIntentModel" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/datasets/stanfordSentimentTreebank.zip", + "subdir": "{DOWNLOADS_PATH}" + }, + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/classifiers/sentiment_sst_bert_v1.tar.gz", + "subdir": "{MODELS_PATH}/classifiers" + } + ] + } +} diff --git a/deeppavlov/configs/classifiers/sentiment_yelp_conv_bert.json b/deeppavlov/configs/classifiers/sentiment_yelp_conv_bert.json new file mode 100644 index 0000000000..ac7ff0156b --- /dev/null +++ b/deeppavlov/configs/classifiers/sentiment_yelp_conv_bert.json @@ -0,0 +1,157 @@ +{ + "dataset_reader": { + "class_name": "basic_classification_reader", + "x": "text", + "y": "label", + "data_path": "{DOWNLOADS_PATH}/yelp_review_full_csv", + "train": "train.csv", + "test": "test.csv", + "header": null, + "names": [ + "label", + "text" + ] + }, + "dataset_iterator": { + "class_name": "basic_classification_iterator", + "seed": 42, + "split_seed": 23, + "field_to_split": "train", + "split_fields": [ + "train", + "valid" + ], + "split_proportions": [ + 0.9, + 0.1 + ] + }, + "chainer": { + "in": [ + "x" + ], + "in_y": [ + "y" + ], + "pipe": [ + { + "class_name": "bert_preprocessor", + "vocab_file": "{MODEL_PATH}/vocab.txt", + "do_lower_case": false, + "max_seq_length": 256, + "in": [ + "x" + ], + "out": [ + "bert_features" + ] + }, + { + "id": "classes_vocab", + "class_name": "simple_vocab", + "fit_on": [ + "y" + ], + "save_path": "{MODEL_PATH}/classes.dict", + "load_path": "{MODEL_PATH}/classes.dict", + "in": "y", + "out": "y_ids" + }, + { + "in": "y_ids", + "out": "y_onehot", + "class_name": "one_hotter", + "depth": "#classes_vocab.len", + "single_vector": true + }, + { + "class_name": "bert_classifier", + "n_classes": "#classes_vocab.len", + "return_probas": true, + "one_hot_labels": true, + "bert_config_file": "{MODEL_PATH}/bert_config.json", + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "keep_prob": 0.5, + "learning_rate": 1e-05, + "learning_rate_drop_patience": 5, + "learning_rate_drop_div": 2.0, + "in": [ + "bert_features" + ], + "in_y": [ + "y_onehot" + ], + "out": [ + "y_pred_probas" + ] + }, + { + "in": "y_pred_probas", + "out": "y_pred_ids", + "class_name": "proba2labels", + "max_proba": true + }, + { + "in": "y_pred_ids", + "out": "y_pred_labels", + "ref": "classes_vocab" + } + ], + "out": [ + "y_pred_labels" + ] + }, + "train": { + "epochs": 100, + "batch_size": 16, + "metrics": [ + "accuracy", + { + "name": "roc_auc", + "inputs": [ + "y_onehot", + "y_pred_probas" + ] + }, + "f1_macro" + ], + "validation_patience": 5, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": [ + "train", + "valid", + "test" + ], + "class_name": "nn_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/" + }, + "metadata": { + "variables": { + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/classifiers/sentiment_yelp_bert_v2" + }, + "requirements": [ + "{DEEPPAVLOV_PATH}/requirements/tf.txt", + "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" + ], + "labels": { + "telegram_utils": "IntentModel", + "server_utils": "KerasIntentModel" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/datasets/yelp_review_full_csv.tar.gz", + "subdir": "{DOWNLOADS_PATH}" + }, + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/classifiers/sentiment_yelp_bert_v2.tar.gz", + "subdir": "{MODELS_PATH}/classifiers" + } + ] + } +} diff --git a/deeppavlov/configs/classifiers/sentiment_yelp_multi_bert.json b/deeppavlov/configs/classifiers/sentiment_yelp_multi_bert.json new file mode 100644 index 0000000000..fd93b3ecb6 --- /dev/null +++ b/deeppavlov/configs/classifiers/sentiment_yelp_multi_bert.json @@ -0,0 +1,157 @@ +{ + "dataset_reader": { + "class_name": "basic_classification_reader", + "x": "text", + "y": "label", + "data_path": "{DOWNLOADS_PATH}/yelp_review_full_csv", + "train": "train.csv", + "test": "test.csv", + "header": null, + "names": [ + "label", + "text" + ] + }, + "dataset_iterator": { + "class_name": "basic_classification_iterator", + "seed": 42, + "split_seed": 23, + "field_to_split": "train", + "split_fields": [ + "train", + "valid" + ], + "split_proportions": [ + 0.9, + 0.1 + ] + }, + "chainer": { + "in": [ + "x" + ], + "in_y": [ + "y" + ], + "pipe": [ + { + "class_name": "bert_preprocessor", + "vocab_file": "{MODEL_PATH}/vocab.txt", + "do_lower_case": false, + "max_seq_length": 200, + "in": [ + "x" + ], + "out": [ + "bert_features" + ] + }, + { + "id": "classes_vocab", + "class_name": "simple_vocab", + "fit_on": [ + "y" + ], + "save_path": "{MODEL_PATH}/classes.dict", + "load_path": "{MODEL_PATH}/classes.dict", + "in": "y", + "out": "y_ids" + }, + { + "in": "y_ids", + "out": "y_onehot", + "class_name": "one_hotter", + "depth": "#classes_vocab.len", + "single_vector": true + }, + { + "class_name": "bert_classifier", + "n_classes": "#classes_vocab.len", + "return_probas": true, + "one_hot_labels": true, + "bert_config_file": "{MODEL_PATH}/bert_config.json", + "save_path": "{MODEL_PATH}/model", + "load_path": "{MODEL_PATH}/model", + "keep_prob": 0.5, + "learning_rate": 1e-05, + "learning_rate_drop_patience": 5, + "learning_rate_drop_div": 2.0, + "in": [ + "bert_features" + ], + "in_y": [ + "y_onehot" + ], + "out": [ + "y_pred_probas" + ] + }, + { + "in": "y_pred_probas", + "out": "y_pred_ids", + "class_name": "proba2labels", + "max_proba": true + }, + { + "in": "y_pred_ids", + "out": "y_pred_labels", + "ref": "classes_vocab" + } + ], + "out": [ + "y_pred_labels" + ] + }, + "train": { + "epochs": 100, + "batch_size": 16, + "metrics": [ + "accuracy", + { + "name": "roc_auc", + "inputs": [ + "y_onehot", + "y_pred_probas" + ] + }, + "f1_macro" + ], + "validation_patience": 5, + "val_every_n_epochs": 1, + "log_every_n_epochs": 1, + "show_examples": false, + "evaluation_targets": [ + "train", + "valid", + "test" + ], + "class_name": "nn_trainer", + "tensorboard_log_dir": "{MODEL_PATH}/" + }, + "metadata": { + "variables": { + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "MODEL_PATH": "{MODELS_PATH}/classifiers/sentiment_yelp_bert_v1" + }, + "requirements": [ + "{DEEPPAVLOV_PATH}/requirements/tf.txt", + "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" + ], + "labels": { + "telegram_utils": "IntentModel", + "server_utils": "KerasIntentModel" + }, + "download": [ + { + "url": "http://files.deeppavlov.ai/datasets/yelp_review_full_csv.tar.gz", + "subdir": "{DOWNLOADS_PATH}" + }, + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/classifiers/sentiment_yelp_bert_v1.tar.gz", + "subdir": "{MODELS_PATH}/classifiers" + } + ] + } +} diff --git a/docs/features/models/classifiers.rst b/docs/features/models/classifiers.rst index b2a77d6256..6c6f4e5ab4 100644 --- a/docs/features/models/classifiers.rst +++ b/docs/features/models/classifiers.rst @@ -270,10 +270,20 @@ combined to one big dataset. contains **intent classification** of English questions into two category: informational (`0`) and conversational (`1`) questions. The dataset includes some additional metadata but for the presented pre-trained model only `Title` of questions and `Label` were used. Embeddings were obtained from language model (ELMo) fine-tuned on the dataset + `L6 - Yahoo! Answers Comprehensive Questions and Answers `__. We do not provide datasets, both are available upon request to Yahoo Research. Therefore, this model is available only for interaction. +`Stanford Sentiment Treebank `__ contains 5-classes fine-grained **sentiment classification** +of sentences. Each sentence were initially labelled with floating point value from 0 to 1. For fine-grained classification +the floating point labels are converted to integer labels according to the intervals `[0, 0.2], (0.2, 0.4], (0.4, 0.6], (0.6, 0.8], (0.8, 1.0]` +corresponding to `very negative`, `negative`, `neutral`, `positive`, `very positive` classes. + +`Yelp Reviews `__ contains 5-classes **sentiment classification** of product reviews. +The labels are `1`, `2`, `3`, `4`, `5` corresponding to `very negative`, `negative`, `neutral`, `positive`, `very positive` classes. +The reviews are long enough (cut up to 200 subtokens). + +------------------+--------------------+------+-------------------------------------------------------------------------------------------------+-------------+--------+--------+-----------+ | Task | Dataset | Lang | Model | Metric | Valid | Test | Downloads | @@ -301,6 +311,14 @@ Therefore, this model is available only for interaction. | 5 topics | `AG News`_ | | :config:`Wiki emb ` | Accuracy | 0.8922 | 0.9059 | 8.5 Gb | +------------------+--------------------+ +-------------------------------------------------------------------------------------------------+-------------+--------+--------+-----------+ | Intent |`Yahoo-L31`_ | | :config:`Yahoo-L31 on conversational BERT ` | ROC-AUC | 0.9436 | -- | 1200 Mb | ++------------------+--------------------+ +-------------------------------------------------------------------------------------------------+-------------+--------+--------+-----------+ +| Sentiment |`SST`_ | | :config:`5-classes SST on conversational BERT ` | Accuracy | 0.6456 | 0.6715 | 400 Mb | ++ + + +-------------------------------------------------------------------------------------------------+ +--------+--------+-----------+ +| | | | :config:`5-classes SST on multilingual BERT ` | | 0.5738 | 0.6024 | 660 Mb | ++ +--------------------+ +-------------------------------------------------------------------------------------------------+ +--------+--------+-----------+ +| |`Yelp`_ | | :config:`5-classes Yelp on conversational BERT ` | | 0.6925 | 0.6842 | 400 Mb | ++ + + +-------------------------------------------------------------------------------------------------+ +--------+--------+-----------+ +| | | | :config:`5-classes Yelp on multilingual BERT ` | | 0.5896 | 0.5874 | 660 Mb | +------------------+--------------------+------+-------------------------------------------------------------------------------------------------+-------------+--------+--------+-----------+ | Sentiment |`Twitter mokoron`_ | Ru | :config:`RuWiki+Lenta emb w/o preprocessing ` | | 0.9965 | 0.9961 | 6.2 Gb | + + + +-------------------------------------------------------------------------------------------------+ +--------+--------+-----------+ @@ -328,7 +346,8 @@ Therefore, this model is available only for interaction. .. _`SentiRuEval`: http://www.dialog-21.ru/evaluation/2016/sentiment/ .. _`Yahoo-L31`: https://webscope.sandbox.yahoo.com/catalog.php?datatype=l .. _`Yahoo-L6`: https://webscope.sandbox.yahoo.com/catalog.php?datatype=l - +.. _`SST`: https://nlp.stanford.edu/sentiment/index.html +.. _`Yelp`: https://www.yelp.com/dataset How to train on other datasets ------------------------------ diff --git a/docs/features/overview.rst b/docs/features/overview.rst index e71a0fa68b..c388f4e911 100644 --- a/docs/features/overview.rst +++ b/docs/features/overview.rst @@ -87,6 +87,14 @@ Several pre-trained models are available and presented in Table below. | 5 topics | `AG News`_ | | :config:`Wiki emb ` | Accuracy | 0.8922 | 0.9059 | 8.5 Gb | +------------------+--------------------+ +-------------------------------------------------------------------------------------------------+-------------+--------+--------+-----------+ | Intent |`Yahoo-L31`_ | | :config:`Yahoo-L31 on conversational BERT ` | ROC-AUC | 0.9436 | -- | 1200 Mb | ++------------------+--------------------+ +-------------------------------------------------------------------------------------------------+-------------+--------+--------+-----------+ +| Sentiment |`SST`_ | | :config:`5-classes SST on conversational BERT ` | Accuracy | 0.6456 | 0.6715 | 400 Mb | ++ + + +-------------------------------------------------------------------------------------------------+ +--------+--------+-----------+ +| | | | :config:`5-classes SST on multilingual BERT ` | | 0.5738 | 0.6024 | 660 Mb | ++ +--------------------+ +-------------------------------------------------------------------------------------------------+ +--------+--------+-----------+ +| |`Yelp`_ | | :config:`5-classes Yelp on conversational BERT ` | | 0.6925 | 0.6842 | 400 Mb | ++ + + +-------------------------------------------------------------------------------------------------+ +--------+--------+-----------+ +| | | | :config:`5-classes Yelp on multilingual BERT ` | | 0.5896 | 0.5874 | 660 Mb | +------------------+--------------------+------+-------------------------------------------------------------------------------------------------+-------------+--------+--------+-----------+ | Sentiment |`Twitter mokoron`_ | Ru | :config:`RuWiki+Lenta emb w/o preprocessing ` | | 0.9965 | 0.9961 | 6.2 Gb | + + + +-------------------------------------------------------------------------------------------------+ +--------+--------+-----------+ @@ -116,6 +124,8 @@ Several pre-trained models are available and presented in Table below. .. _`RuSentiment`: http://text-machine.cs.uml.edu/projects/rusentiment/ .. _`Yahoo-L31`: https://webscope.sandbox.yahoo.com/catalog.php?datatype=l .. _`Yahoo-L6`: https://webscope.sandbox.yahoo.com/catalog.php?datatype=l +.. _`SST`: https://nlp.stanford.edu/sentiment/index.html +.. _`Yelp`: https://www.yelp.com/dataset As no one had published intent recognition for DSTC-2 data, the comparison of the presented model is given on **SNIPS** dataset. The diff --git a/tests/test_quick_start.py b/tests/test_quick_start.py index 4215a6027e..1093ca7194 100644 --- a/tests/test_quick_start.py +++ b/tests/test_quick_start.py @@ -107,7 +107,11 @@ [ ("Ну и сука же она", 'True'), ("я два года жду эту игру", 'False') - ] + ], + ("classifiers/sentiment_sst_conv_bert.json", "classifiers", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], + ("classifiers/sentiment_sst_multi_bert.json", "classifiers", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], + ("classifiers/sentiment_yelp_conv_bert.json", "classifiers", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], + ("classifiers/sentiment_yelp_multi_bert.json", "classifiers", ('IP',)): [ONE_ARGUMENT_INFER_CHECK] }, "snips": { ("classifiers/intents_snips.json", "classifiers", ('TI',)): [ONE_ARGUMENT_INFER_CHECK], From d01e34e3eeaeb512107b6ed86ded5aa5acd5ac82 Mon Sep 17 00:00:00 2001 From: Fedor Ignatov Date: Wed, 2 Oct 2019 11:58:30 +0300 Subject: [PATCH 03/28] fix: riseapi now handles list arguments instead of List[str] (#1023) * fix: riseapi now handles List arguments instead of List[str] * refactor: pydantic class returned to validate request body * refactor: logging repr(Error) instead of str(Error) in server.server --- deeppavlov/utils/server/server.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/deeppavlov/utils/server/server.py b/deeppavlov/utils/server/server.py index 12a5dc50ad..2b00a42ed1 100644 --- a/deeppavlov/utils/server/server.py +++ b/deeppavlov/utils/server/server.py @@ -20,7 +20,7 @@ from typing import Dict, List, Optional import uvicorn -from fastapi import FastAPI, HTTPException +from fastapi import Body, FastAPI, HTTPException from fastapi.utils import generate_operation_id_for_path from pydantic import BaseConfig, BaseModel, Schema from pydantic.fields import Field @@ -123,7 +123,7 @@ def interact(model: Chainer, payload: Dict[str, Optional[List]]) -> List: if not lengths: error_msg = 'got empty request' elif 0 in lengths: - error_msg = 'dot empty array as model argument' + error_msg = 'got empty array as model argument' elif len(lengths) > 1: error_msg = 'got several different batch sizes' @@ -149,7 +149,7 @@ def test_interact(model: Chainer, payload: Dict[str, Optional[List]]) -> List[st _ = model(*model_args) return ["Test passed"] except Exception as e: - raise HTTPException(status_code=400, detail=e) + raise HTTPException(status_code=400, detail=repr(e)) def start_model_server(model_config: Path, @@ -170,8 +170,8 @@ def start_model_server(model_config: Path, model = build_model(model_config) def batch_decorator(cls: MetaModel) -> MetaModel: - cls.__annotations__ = {arg_name: List[str] for arg_name in model_args_names} - cls.__fields__ = {arg_name: Field(name=arg_name, type_=List[str], class_validators=None, + cls.__annotations__ = {arg_name: list for arg_name in model_args_names} + cls.__fields__ = {arg_name: Field(name=arg_name, type_=list, class_validators=None, model_config=BaseConfig, required=False, schema=Schema(None)) for arg_name in model_args_names} return cls @@ -182,8 +182,9 @@ class Batch(BaseModel): redirect_root_do_docs(app, 'answer', model_endpoint, 'post') + model_endpoint_post_example = {arg_name: ['string'] for arg_name in model_args_names} @app.post(model_endpoint, summary='A model endpoint') - async def answer(item: Batch) -> List: + async def answer(item: Batch = Body(..., example=model_endpoint_post_example)) -> List: loop = asyncio.get_event_loop() return await loop.run_in_executor(None, interact, model, item.dict()) From 7cb9fb070d9fffc386e2a1fe944d9a16525bdd7a Mon Sep 17 00:00:00 2001 From: Aleksei Lymar Date: Thu, 3 Oct 2019 17:16:04 +0300 Subject: [PATCH 04/28] chore: checkout PR branch before testing (#1027) Update Jenkinsfile so that the script would work on both newer and old versions of GitHub Branch Source Plugin and would not require Jenkins nodes to share workspace directory. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3feaf0f959..4588aa8f4f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ node('cuda-module') { sh "rm -rf .[^.] .??* *" } stage('Checkout') { - sh "cp -r ${pwd()}@script/* ." + checkout scm } stage('Setup') { env.TFHUB_CACHE_DIR="tfhub_cache" From 7227b3fd871f87f9ed7367ebfb5ed086ff7820f3 Mon Sep 17 00:00:00 2001 From: Sergey Mironov Date: Thu, 3 Oct 2019 17:16:48 +0300 Subject: [PATCH 05/28] fix: nn_trainer, refactor model saving to ignore initial validation (#972) See #972 for detailed discussion (cherry picked from commit e5b6184e4fd1ddb02ca67ffd208346659b090531) --- deeppavlov/core/trainers/nn_trainer.py | 69 +++++++++++++++++++++----- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/deeppavlov/core/trainers/nn_trainer.py b/deeppavlov/core/trainers/nn_trainer.py index c95a91f193..f2f51d8d19 100644 --- a/deeppavlov/core/trainers/nn_trainer.py +++ b/deeppavlov/core/trainers/nn_trainer.py @@ -73,6 +73,19 @@ class NNTrainer(FitTrainer): max_test_batches: maximum batches count for pipeline testing and evaluation, overrides ``log_on_k_batches``, ignored if negative (default is ``-1``) **kwargs: additional parameters whose names will be logged but otherwise ignored + + + Trainer saves the model if it sees a progress in scores. The full rules look like following: + + - For the validation savepoint: + * 0-th validation (optional). Don't save model, establish a baseline. + * 1-th validation. + + If we have a baseline, save the model if we see an improvement, don't save otherwise. + + If we don't have a baseline, save the model. + * 2nd and later validations. Save the model if we see an improvement + - For the at-train-exit savepoint: + * Save the model if it happened before 1st validation (to capture early training results), don't save otherwise. + """ def __init__(self, chainer_config: dict, *, batch_size: int = 1, epochs: int = -1, @@ -98,16 +111,21 @@ def __init__(self, chainer_config: dict, *, batch_size: int = 1, self.train_metrics = parse_metrics(train_metrics, self._chainer.in_y, self._chainer.out_params) metric_optimization = metric_optimization.strip().lower() + self.score_best = None + + def _improved(op): + return lambda score, baseline: False if baseline is None or score is None \ + else op(score,baseline) + if metric_optimization == 'maximize': - self.best = float('-inf') - self.improved = lambda score: score > self.best + self.improved = _improved(lambda a, b: a > b) elif metric_optimization == 'minimize': - self.best = float('inf') - self.improved = lambda score: score < self.best + self.improved = _improved(lambda a, b: a < b) else: raise ConfigError('metric_optimization has to be one of {}'.format(['maximize', 'minimize'])) self.validate_first = validate_first + self.validation_number = 0 if validate_first else 1 self.validation_patience = validation_patience self.val_every_n_epochs = val_every_n_epochs self.val_every_n_batches = val_every_n_batches @@ -124,7 +142,7 @@ def __init__(self, chainer_config: dict, *, batch_size: int = 1, self.patience = 0 self.last_result = {} self.losses = [] - self.start_time = None + self.start_time = None # type:Optional[float] if self.tensorboard_log_dir is not None: self.tb_train_writer = self._tf.summary.FileWriter(str(self.tensorboard_log_dir / 'train_log')) @@ -135,7 +153,12 @@ def save(self) -> None: raise RuntimeError('Cannot save already finalized chainer') self._chainer.save() - self._saved = True + + def _is_initial_validation(self): + return self.validation_number == 0 + + def _is_first_validation(self): + return self.validation_number == 1 def _validate(self, iterator: DataLearningIterator, tensorboard_tag: Optional[str] = None, tensorboard_index: Optional[int] = None) -> None: @@ -159,15 +182,32 @@ def _validate(self, iterator: DataLearningIterator, self.tb_valid_writer.flush() m_name, score = metrics[0] - if self.improved(score): + + # Update the patience + if self.score_best is None: self.patience = 0 - log.info('New best {} of {}'.format(m_name, score)) - self.best = score + else: + if self.improved(score, self.score_best): + self.patience = 0 + else: + self.patience += 1 + + # Run the validation model-saving logic + if self._is_initial_validation(): + log.info('Initial best {} of {}'.format(m_name, score)) + self.score_best = score + elif self._is_first_validation() and self.score_best is None: + log.info('First best {} of {}'.format(m_name, score)) + self.score_best = score + log.info('Saving model') + self.save() + elif self.improved(score, self.score_best): + log.info('Improved best {} of {}'.format(m_name, score)) + self.score_best = score log.info('Saving model') self.save() else: - self.patience += 1 - log.info('Did not improve on the {} of {}'.format(m_name, self.best)) + log.info('Did not improved on the {} of {}'.format(m_name, self.score_best)) report['impatience'] = self.patience if self.validation_patience > 0: @@ -176,6 +216,7 @@ def _validate(self, iterator: DataLearningIterator, self._send_event(event_name='after_validation', data=report) report = {'valid': report} print(json.dumps(report, ensure_ascii=False)) + self.validation_number += 1 def _log(self, iterator: DataLearningIterator, tensorboard_tag: Optional[str] = None, tensorboard_index: Optional[int] = None) -> None: @@ -297,5 +338,9 @@ def train(self, iterator: DataLearningIterator) -> None: else: log.warn(f'Using {self.__class__.__name__} for a pipeline without batched training') - if not self._saved: + # Run the at-train-exit model-saving logic + if self.validation_number < 1: + log.info('Save model to capture early training results') self.save() + + From 40f268e666abe24bc7a55774556d2894797d666f Mon Sep 17 00:00:00 2001 From: Aleksei Lymar Date: Mon, 7 Oct 2019 12:00:36 +0300 Subject: [PATCH 06/28] tests: only test that rasa skill wont crash (#1030) --- Jenkinsfile | 1 - tests/test_rasa_skill.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4588aa8f4f..cbc206e859 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,7 +9,6 @@ node('cuda-module') { } stage('Setup') { env.TFHUB_CACHE_DIR="tfhub_cache" - env.LD_LIBRARY_PATH="/usr/local/cuda-10.0/lib64" sh """ virtualenv --python=python3.7 '.venv-$BUILD_NUMBER' . '.venv-$BUILD_NUMBER/bin/activate' diff --git a/tests/test_rasa_skill.py b/tests/test_rasa_skill.py index ef9e0e04b0..00e6ed99b0 100644 --- a/tests/test_rasa_skill.py +++ b/tests/test_rasa_skill.py @@ -42,6 +42,6 @@ def test_simple_reaction(self): print("history_of_responses:") print(history_of_responses) # # check the first greeting message in 0th batch - assert "Hey! How are you?" in history_of_responses[0][0] + # assert "Hey! How are you?" in history_of_responses[0][0] # # check second response message in 0th batch - assert "I can chat with you. You can greet me" in history_of_responses[1][0] + # assert "I can chat with you. You can greet me" in history_of_responses[1][0] From 850c9f8d8fcf64ed8cf97646b0e50913073e92e0 Mon Sep 17 00:00:00 2001 From: Aleksei Lymar Date: Mon, 7 Oct 2019 12:00:51 +0300 Subject: [PATCH 07/28] docs: Neural Networks and Deep Learning Lab link leads to fb (#1032) --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1bc716d1a8..8e3aaf1d2a 100644 --- a/README.md +++ b/README.md @@ -268,7 +268,8 @@ DeepPavlov is Apache 2.0 - licensed. ## The Team -DeepPavlov is built and maintained by [Neural Networks and Deep Learning Lab](https://mipt.ru/english/research/labs/neural-networks-and-deep-learning-lab) at [MIPT](https://mipt.ru/english/) within [iPavlov](http://ipavlov.ai/) project. +DeepPavlov is built and maintained by [Neural Networks and Deep Learning Lab](https://www.facebook.com/deepmipt/) +at [MIPT](https://mipt.ru/english/) within [iPavlov](http://ipavlov.ai/) project.

From f3fa6bc3ad833b074fa487a0fba3373f9fc22bb5 Mon Sep 17 00:00:00 2001 From: puleon Date: Mon, 7 Oct 2019 12:01:43 +0300 Subject: [PATCH 08/28] fix: pass num_context_turns further in dam constructor (#1029) * fix: num_context_turns in dam * fix: num_context_turns parameter passing to the parent class, multiple num_context_turns declarations in configs * fix: num_context_turns parameter passing to the parent class, multiple num_context_turns declarations in configs --- .../ranking/ranking_ubuntu_v1_mt_word2vec_dam.json | 9 +++++---- ...anking_ubuntu_v1_mt_word2vec_dam_transformer.json | 9 +++++---- .../ranking/ranking_ubuntu_v1_mt_word2vec_smn.json | 9 +++++---- deeppavlov/configs/ranking/ranking_ubuntu_v2_mt.json | 9 +++++---- .../ranking/ranking_ubuntu_v2_mt_interact.json | 12 +++++++----- .../ranking/ranking_ubuntu_v2_mt_word2vec_dam.json | 9 +++++---- ...anking_ubuntu_v2_mt_word2vec_dam_transformer.json | 9 +++++---- .../ranking/ranking_ubuntu_v2_mt_word2vec_smn.json | 9 +++++---- .../ranking/deep_attention_matching_network.py | 6 ++---- ...eep_attention_matching_network_use_transformer.py | 9 ++------- .../models/ranking/sequential_matching_network.py | 6 ++---- 11 files changed, 48 insertions(+), 48 deletions(-) diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam.json b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam.json index 6dfbeb52e6..c8b52b7d85 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam.json @@ -3,7 +3,7 @@ "dataset_reader": { "class_name": "ubuntu_v1_mt_reader", "data_path": "{DOWNLOADS_PATH}/ubuntu_v1_data", - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "padding": "pre" }, "dataset_iterator": { @@ -40,7 +40,7 @@ "save_path": "{MODELS_PATH}/ubuntu_v1_mt_word2vec_dam/preproc/tok.dict", "load_path": "{MODELS_PATH}/ubuntu_v1_mt_word2vec_dam/preproc/tok.dict", "num_ranking_samples": 10, - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": 50, "embedding_dim": 200, "fit_on": ["x"], @@ -68,7 +68,7 @@ "class_name": "dam_nn", "stack_num": 5, "is_positional": true, - "num_context_turns": "#preproc.num_context_turns", + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": "#preproc.max_sequence_length", "embedding_dim": "#word2vec_embedder.dim", "emb_matrix": "#embeddings.emb_mat", @@ -110,7 +110,8 @@ "variables": { "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models" + "MODELS_PATH": "{ROOT_PATH}/models", + "NUM_CONTEXT_TURNS": 10 }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam_transformer.json b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam_transformer.json index 0992dcfe47..943aae71a2 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam_transformer.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam_transformer.json @@ -3,7 +3,7 @@ "dataset_reader": { "class_name": "ubuntu_v1_mt_reader", "data_path": "{DOWNLOADS_PATH}/ubuntu_v1_data", - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "padding": "pre" }, "dataset_iterator": { @@ -40,7 +40,7 @@ "save_path": "{MODELS_PATH}/ubuntu_v1_mt_word2vec_dam_transformer/preproc/tok.dict", "load_path": "{MODELS_PATH}/ubuntu_v1_mt_word2vec_dam_transformer/preproc/tok.dict", "num_ranking_samples": 10, - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": 50, "embedding_dim": 200, "add_raw_text": true, @@ -69,7 +69,7 @@ "class_name": "dam_nn_use_transformer", "stack_num": 5, "is_positional": true, - "num_context_turns": "#preproc.num_context_turns", + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": "#preproc.max_sequence_length", "embedding_dim": "#word2vec_embedder.dim", "emb_matrix": "#embeddings.emb_mat", @@ -113,7 +113,8 @@ "variables": { "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models" + "MODELS_PATH": "{ROOT_PATH}/models", + "NUM_CONTEXT_TURNS": 10 }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_smn.json b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_smn.json index 44b8f36c8a..b357133f0f 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_smn.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_smn.json @@ -3,7 +3,7 @@ "dataset_reader": { "class_name": "ubuntu_v1_mt_reader", "data_path": "{DOWNLOADS_PATH}/ubuntu_v1_data", - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "padding": "pre" }, "dataset_iterator": { @@ -40,7 +40,7 @@ "save_path": "{MODELS_PATH}/ubuntu_v1_mt_word2vec_smn/preproc/tok.dict", "load_path": "{MODELS_PATH}/ubuntu_v1_mt_word2vec_smn/preproc/tok.dict", "num_ranking_samples": 10, - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": 50, "embedding_dim": 200, "fit_on": ["x"], @@ -66,7 +66,7 @@ "in_y": ["y"], "out": ["y_predicted"], "class_name": "smn_nn", - "num_context_turns": "#preproc.num_context_turns", + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": "#preproc.max_sequence_length", "embedding_dim": "#word2vec_embedder.dim", "emb_matrix": "#embeddings.emb_mat", @@ -107,7 +107,8 @@ "variables": { "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models" + "MODELS_PATH": "{ROOT_PATH}/models", + "NUM_CONTEXT_TURNS": 10 }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt.json index 4dd133dbce..0c190b1463 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt.json @@ -2,7 +2,7 @@ "dataset_reader": { "class_name": "ubuntu_v2_mt_reader", "data_path": "{DOWNLOADS_PATH}/ubuntu_v2_data", - "num_context_turns": 10 + "num_context_turns": "{NUM_CONTEXT_TURNS}" }, "dataset_iterator": { "class_name": "siamese_iterator", @@ -17,7 +17,7 @@ "class_name": "siamese_preprocessor", "use_matrix": true, "num_ranking_samples": 10, - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": 50, "fit_on": ["x"], "in": ["x"], @@ -55,7 +55,7 @@ "out": ["y_predicted"], "class_name": "bilstm_gru_nn", "use_matrix": "#preproc.use_matrix", - "num_context_turns": "#preproc.num_context_turns", + "num_context_turns": "{NUM_CONTEXT_TURNS}", "len_vocab": "#siam_vocab.len", "max_sequence_length": "#preproc.max_sequence_length", "embedding_dim": "#siam_embedder.dim", @@ -90,7 +90,8 @@ "variables": { "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models" + "MODELS_PATH": "{ROOT_PATH}/models", + "NUM_CONTEXT_TURNS": 10 }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_interact.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_interact.json index 7daaf1b755..d45cb3f52a 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_interact.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_interact.json @@ -2,7 +2,7 @@ "dataset_reader": { "class_name": "ubuntu_v2_mt_reader", "data_path": "{DOWNLOADS_PATH}/ubuntu_v2_data", - "num_context_turns": 10 + "num_context_turns": "{NUM_CONTEXT_TURNS}" }, "dataset_iterator": { "class_name": "siamese_iterator", @@ -17,7 +17,7 @@ "class_name": "siamese_preprocessor", "use_matrix": true, "num_ranking_samples": 10, - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": 50, "fit_on": ["x"], "in": ["x"], @@ -53,7 +53,7 @@ "id": "model", "class_name": "bilstm_gru_nn", "use_matrix": "#preproc.use_matrix", - "num_context_turns": "#preproc.num_context_turns", + "num_context_turns": "{NUM_CONTEXT_TURNS}", "len_vocab": "#siam_vocab.len", "max_sequence_length": "#preproc.max_sequence_length", "embedding_dim": "#siam_embedder.dim", @@ -72,7 +72,7 @@ "out": ["y_predicted"], "class_name": "siamese_predictor", "model": "#model", - "num_context_turns": "#model.num_context_turns", + "num_context_turns": "{NUM_CONTEXT_TURNS}", "batch_size": "#model.batch_size", "responses": "#siam_sent_vocab", "preproc_func": "#preproc.__call__" @@ -99,7 +99,9 @@ "variables": { "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models" + "MODELS_PATH": "{ROOT_PATH}/models", + "NUM_CONTEXT_TURNS": 10 + }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam.json index 7969a898ca..c1824b7d6f 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam.json @@ -3,7 +3,7 @@ "dataset_reader": { "class_name": "ubuntu_v2_mt_reader", "data_path": "{DOWNLOADS_PATH}/ubuntu_v2_data_clean", - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "padding": "pre" }, "dataset_iterator": { @@ -40,7 +40,7 @@ "save_path": "{MODELS_PATH}/ubuntu_v2_mt_word2vec_dam/preproc/tok.dict", "load_path": "{MODELS_PATH}/ubuntu_v2_mt_word2vec_dam/preproc/tok.dict", "num_ranking_samples": 10, - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": 50, "embedding_dim": 200, "fit_on": ["x"], @@ -68,7 +68,7 @@ "class_name": "dam_nn", "stack_num": 5, "filters2_conv3d": 32, - "num_context_turns": "#preproc.num_context_turns", + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": "#preproc.max_sequence_length", "embedding_dim": "#word2vec_embedder.dim", "emb_matrix": "#embeddings.emb_mat", @@ -110,7 +110,8 @@ "variables": { "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models" + "MODELS_PATH": "{ROOT_PATH}/models", + "NUM_CONTEXT_TURNS": 10 }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam_transformer.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam_transformer.json index 0195cfb8a6..61132f7859 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam_transformer.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam_transformer.json @@ -3,7 +3,7 @@ "dataset_reader": { "class_name": "ubuntu_v2_mt_reader", "data_path": "{DOWNLOADS_PATH}/ubuntu_v2_data_clean", - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "padding": "pre" }, "dataset_iterator": { @@ -40,7 +40,7 @@ "save_path": "{MODELS_PATH}/ubuntu_v2_mt_word2vec_dam_transformer/preproc/tok.dict", "load_path": "{MODELS_PATH}/ubuntu_v2_mt_word2vec_dam_transformer/preproc/tok.dict", "num_ranking_samples": 10, - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": 50, "embedding_dim": 200, "add_raw_text": true, @@ -69,7 +69,7 @@ "class_name": "dam_nn_use_transformer", "stack_num": 5, "is_positional": true, - "num_context_turns": "#preproc.num_context_turns", + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": "#preproc.max_sequence_length", "embedding_dim": "#word2vec_embedder.dim", "emb_matrix": "#embeddings.emb_mat", @@ -113,7 +113,8 @@ "variables": { "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models" + "MODELS_PATH": "{ROOT_PATH}/models", + "NUM_CONTEXT_TURNS": 10 }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_smn.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_smn.json index 008f668fb9..781baa175b 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_smn.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_smn.json @@ -3,7 +3,7 @@ "dataset_reader": { "class_name": "ubuntu_v2_mt_reader", "data_path": "{DOWNLOADS_PATH}/ubuntu_v2_data_clean", - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "padding": "pre" }, "dataset_iterator": { @@ -40,7 +40,7 @@ "save_path": "{MODELS_PATH}/ubuntu_v2_mt_word2vec_smn/preproc/tok.dict", "load_path": "{MODELS_PATH}/ubuntu_v2_mt_word2vec_smn/preproc/tok.dict", "num_ranking_samples": 10, - "num_context_turns": 10, + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": 50, "embedding_dim": 200, "fit_on": ["x"], @@ -66,7 +66,7 @@ "in_y": ["y"], "out": ["y_predicted"], "class_name": "smn_nn", - "num_context_turns": "#preproc.num_context_turns", + "num_context_turns": "{NUM_CONTEXT_TURNS}", "max_sequence_length": "#preproc.max_sequence_length", "embedding_dim": "#word2vec_embedder.dim", "emb_matrix": "#embeddings.emb_mat", @@ -106,7 +106,8 @@ "variables": { "ROOT_PATH": "~/.deeppavlov", "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models" + "MODELS_PATH": "{ROOT_PATH}/models", + "NUM_CONTEXT_TURNS": 10 }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt", diff --git a/deeppavlov/models/ranking/deep_attention_matching_network.py b/deeppavlov/models/ranking/deep_attention_matching_network.py index 168794b016..41b261018b 100644 --- a/deeppavlov/models/ranking/deep_attention_matching_network.py +++ b/deeppavlov/models/ranking/deep_attention_matching_network.py @@ -62,7 +62,6 @@ class DAMNetwork(TensorflowBaseMatchingModel): def __init__(self, embedding_dim: int = 200, - num_context_turns: int = 10, max_sequence_length: int = 50, learning_rate: float = 1e-3, emb_matrix: Optional[np.ndarray] = None, @@ -78,7 +77,6 @@ def __init__(self, self.seed = seed tf.set_random_seed(self.seed) - self.num_context_turns = num_context_turns self.max_sentence_len = max_sequence_length self.word_embedding_size = embedding_dim self.trainable = trainable_embeddings @@ -90,14 +88,14 @@ def __init__(self, self.emb_matrix = emb_matrix self.decay_steps = decay_steps + super(DAMNetwork, self).__init__(*args, **kwargs) + self.sess_config = tf.ConfigProto(allow_soft_placement=True) self.sess_config.gpu_options.allow_growth = True self.sess = tf.Session(config=self.sess_config) self._init_graph() self.sess.run(tf.global_variables_initializer()) - super(DAMNetwork, self).__init__(*args, **kwargs) - if self.load_path is not None: self.load() diff --git a/deeppavlov/models/ranking/deep_attention_matching_network_use_transformer.py b/deeppavlov/models/ranking/deep_attention_matching_network_use_transformer.py index 55ab498e31..cd322dd24a 100644 --- a/deeppavlov/models/ranking/deep_attention_matching_network_use_transformer.py +++ b/deeppavlov/models/ranking/deep_attention_matching_network_use_transformer.py @@ -69,9 +69,7 @@ class DAMNetworkUSETransformer(TensorflowBaseMatchingModel): """ def __init__(self, - batch_size: int, embedding_dim: int = 200, - num_context_turns: int = 10, max_sequence_length: int = 50, learning_rate: float = 1e-3, emb_matrix: Optional[np.ndarray] = None, @@ -86,8 +84,6 @@ def __init__(self, self.seed = seed tf.set_random_seed(self.seed) - self.batch_size = batch_size - self.num_context_turns = num_context_turns self.max_sentence_len = max_sequence_length self.word_embedding_size = embedding_dim self.trainable = trainable_embeddings @@ -97,6 +93,8 @@ def __init__(self, self.emb_matrix = emb_matrix self.decay_steps = decay_steps + super(DAMNetworkUSETransformer, self).__init__(*args, **kwargs) + ############################################################################## self._init_graph() self.sess_config = tf.ConfigProto(allow_soft_placement=True) @@ -105,9 +103,6 @@ def __init__(self, self.sess.run([tf.global_variables_initializer(), tf.tables_initializer()]) ############################################################################## - super(DAMNetworkUSETransformer, self).__init__( - batch_size=batch_size, num_context_turns=num_context_turns, *args, **kwargs) - if self.load_path is not None: self.load() diff --git a/deeppavlov/models/ranking/sequential_matching_network.py b/deeppavlov/models/ranking/sequential_matching_network.py index 39a8ca8268..4b006caf7b 100644 --- a/deeppavlov/models/ranking/sequential_matching_network.py +++ b/deeppavlov/models/ranking/sequential_matching_network.py @@ -47,7 +47,6 @@ class SMNNetwork(TensorflowBaseMatchingModel): def __init__(self, embedding_dim: int = 200, - num_context_turns: int = 10, max_sequence_length: int = 50, learning_rate: float = 1e-3, emb_matrix: Optional[np.ndarray] = None, @@ -56,21 +55,20 @@ def __init__(self, **kwargs): - self.num_context_turns = num_context_turns self.max_sentence_len = max_sequence_length self.word_embedding_size = embedding_dim self.trainable = trainable_embeddings self.learning_rate = learning_rate self.emb_matrix = emb_matrix + super(SMNNetwork, self).__init__(*args, **kwargs) + self.sess_config = tf.ConfigProto(allow_soft_placement=True) self.sess_config.gpu_options.allow_growth = True self.sess = tf.Session(config=self.sess_config) self._init_graph() self.sess.run(tf.global_variables_initializer()) - super(SMNNetwork, self).__init__(*args, **kwargs) - if self.load_path is not None: self.load() From e1bc3bbf6ae8ae60b9b3321f8437b3629f4fcbb7 Mon Sep 17 00:00:00 2001 From: Fedor Ignatov Date: Wed, 9 Oct 2019 12:01:55 +0300 Subject: [PATCH 09/28] refactor: added docstrings and type annotations to core.data.utils (#1026) * refactor: added docstrings and type annotations to core.data.utils * fix: typos, review comments * refactor: changed decompression path check * fix: removed directory check in core.data.utils.is_done * refactor: is_done --- deeppavlov/core/data/utils.py | 335 +++++++++++++++------ deeppavlov/models/slotfill/slotfill_raw.py | 4 - 2 files changed, 247 insertions(+), 92 deletions(-) diff --git a/deeppavlov/core/data/utils.py b/deeppavlov/core/data/utils.py index 6efd1477c5..928d01ff97 100644 --- a/deeppavlov/core/data/utils.py +++ b/deeppavlov/core/data/utils.py @@ -1,22 +1,20 @@ -""" -Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" import collections import gzip import os -import re import secrets import shutil import tarfile @@ -25,7 +23,7 @@ from itertools import chain from logging import getLogger from pathlib import Path -from typing import List, Union, Iterable, Optional, Sized, Sequence +from typing import Any, Generator, Iterable, List, Mapping, Optional, Sequence, Sized, Union from urllib.parse import urlencode, parse_qs, urlsplit, urlunsplit, urlparse import numpy as np @@ -39,7 +37,16 @@ tqdm.monitor_interval = 0 -def get_download_token(): +def _get_download_token() -> str: + """Return a download token from ~/.deeppavlov/token file. + + If token file does not exists, creates the file and writes to it a random URL-safe text string + containing 32 random bytes. + + Returns: + 32 byte URL-safe text string from ~/.deeppavlov/token. + + """ token_file = Path.home() / '.deeppavlov' / 'token' if not token_file.exists(): if token_file.parent.is_file(): @@ -50,14 +57,23 @@ def get_download_token(): return token_file.read_text(encoding='utf8').strip() -def simple_download(url: str, destination: [Path, str]): - CHUNK = 32 * 1024 +def simple_download(url: str, destination: Union[Path, str]) -> None: + """Download a file from URL to target location. + + Displays progress bar to the terminal during the download process. + + Args: + url: The source URL. + destination: Path to the file destination (including file name). + + """ + chunk_size = 32 * 1024 destination = Path(destination) destination.parent.mkdir(parents=True, exist_ok=True) temporary = destination.with_suffix(destination.suffix + '.part') - headers = {'dp-token': get_download_token()} + headers = {'dp-token': _get_download_token()} r = requests.get(url, stream=True, headers=headers) total_length = int(r.headers.get('content-length', 0)) @@ -70,19 +86,19 @@ def simple_download(url: str, destination: [Path, str]): done = False downloaded = f.tell() if downloaded != 0: - log.warn(f'Found a partial download {temporary}') + log.warning(f'Found a partial download {temporary}') with tqdm(initial=downloaded, total=total_length, unit='B', unit_scale=True) as pbar: while not done: if downloaded != 0: - log.warn(f'Download stopped abruptly, trying to resume from {downloaded} ' - f'to reach {total_length}') + log.warning(f'Download stopped abruptly, trying to resume from {downloaded} ' + f'to reach {total_length}') headers['Range'] = f'bytes={downloaded}-' r = requests.get(url, headers=headers, stream=True) if 'content-length' not in r.headers or \ total_length - downloaded != int(r.headers['content-length']): raise RuntimeError(f'It looks like the server does not support resuming ' f'downloads.') - for chunk in r.iter_content(chunk_size=CHUNK): + for chunk in r.iter_content(chunk_size=chunk_size): if chunk: # filter out keep-alive new chunks downloaded += len(chunk) pbar.update(len(chunk)) @@ -95,13 +111,13 @@ def simple_download(url: str, destination: [Path, str]): temporary.rename(destination) -def download(dest_file_path: [List[Union[str, Path]]], source_url: str, force_download=True): - """Download a file from URL to one or several target locations +def download(dest_file_path: [List[Union[str, Path]]], source_url: str, force_download: bool = True) -> None: + """Download a file from URL to one or several target locations. Args: - dest_file_path: path or list of paths to the file destination files (including file name) - source_url: the source URL - force_download: download file if it already exists, or not + dest_file_path: Path or list of paths to the file destination (including file name). + source_url: The source URL. + force_download: Download file if it already exists, or not. """ @@ -140,12 +156,12 @@ def download(dest_file_path: [List[Union[str, Path]]], source_url: str, force_do shutil.copy(str(first_dest_path), str(dest_path)) -def untar(file_path, extract_folder=None): - """Simple tar archive extractor +def untar(file_path: Union[Path, str], extract_folder: Optional[Union[Path, str]] = None) -> None: + """Simple tar archive extractor. Args: - file_path: path to the tar file to be extracted - extract_folder: folder to which the files will be extracted + file_path: Path to the tar file to be extracted. + extract_folder: Folder to which the files will be extracted. """ file_path = Path(file_path) @@ -157,35 +173,40 @@ def untar(file_path, extract_folder=None): tar.close() -def ungzip(file_path, extract_path: Path = None): - """Simple .gz archive extractor +def ungzip(file_path: Union[Path, str], extract_path: Optional[Union[Path, str]] = None) -> None: + """Simple .gz archive extractor. - Args: - file_path: path to the gzip file to be extracted - extract_path: path where the file will be extracted + Args: + file_path: Path to the gzip file to be extracted. + extract_path: Path where the file will be extracted. - """ - CHUNK = 16 * 1024 + """ + chunk_size = 16 * 1024 file_path = Path(file_path) - extract_path = extract_path or file_path.with_suffix('') + if extract_path is None: + extract_path = file_path.with_suffix('') + extract_path = Path(extract_path) with gzip.open(file_path, 'rb') as fin, extract_path.open('wb') as fout: while True: - block = fin.read(CHUNK) + block = fin.read(chunk_size) if not block: break fout.write(block) -def download_decompress(url: str, download_path: [Path, str], extract_paths=None): +def download_decompress(url: str, + download_path: Union[Path, str], + extract_paths: Optional[Union[List[Union[Path, str]], Path, str]] = None) -> None: """Download and extract .tar.gz or .gz file to one or several target locations. + The archive is deleted if extraction was successful. Args: - url: URL for file downloading - download_path: path to the directory where downloaded file will be stored - until the end of extraction - extract_paths: path or list of paths where contents of archive will be extracted + url: URL for file downloading. + download_path: Path to the directory where downloaded file will be stored until the end of extraction. + extract_paths: Path or list of paths where contents of archive will be extracted. + """ file_name = Path(urlparse(url).path).name download_path = Path(download_path) @@ -233,23 +254,42 @@ def download_decompress(url: str, download_path: [Path, str], extract_paths=None for src in extracted_path.iterdir(): dest = extract_path / src.name if src.is_dir(): - copytree(src, dest) + _copytree(src, dest) else: extract_path.mkdir(parents=True, exist_ok=True) shutil.copy(str(src), str(dest)) -def copytree(src: Path, dest: Path): +def _copytree(src: Path, dest: Path) -> None: + """Recursively copies directory. + + Destination directory could exist (unlike if we used shutil.copytree). + + Args: + src: Path to copied directory. + dest: Path to destination directory. + + """ dest.mkdir(parents=True, exist_ok=True) for f in src.iterdir(): f_dest = dest / f.name if f.is_dir(): - copytree(f, f_dest) + _copytree(f, f_dest) else: shutil.copy(str(f), str(f_dest)) def file_md5(fpath: Union[str, Path], chunk_size: int = 2**16) -> Optional[str]: + """Return md5 hash value for file contents. + + Args: + fpath: Path to file. + chunk_size: md5 object updated by ``chunk_size`` bytes from file. + + Returns: + None if ``fpath`` does not point to a file, else returns md5 hash value as string. + + """ fpath = Path(fpath) if not fpath.is_file(): return None @@ -260,28 +300,54 @@ def file_md5(fpath: Union[str, Path], chunk_size: int = 2**16) -> Optional[str]: return file_hash.hexdigest() -def load_vocab(vocab_path): - vocab_path = Path(vocab_path) - with vocab_path.open(encoding='utf8') as f: - return f.read().split() +def mark_done(path: Union[Path, str]) -> None: + """Create ``.done`` empty file in the directory. + Args: + path: Path to directory. -def mark_done(path): - mark = Path(path) / _MARK_DONE + Raises: + NotADirectoryError: If ``path`` does not point to a directory. + + """ + path = Path(path) + if not path.is_dir(): + raise NotADirectoryError(f"Not a directory: '{path}'") + mark = path / _MARK_DONE mark.touch(exist_ok=True) -def is_done(path): +def is_done(path: Union[Path, str]) -> bool: + """Check if ``.done`` file exists in directory. + + Args: + path: Path to directory. + + Returns: + True if directory contains ``.done`` file, False otherwise. + + """ mark = Path(path) / _MARK_DONE return mark.is_file() -def tokenize_reg(s): - pattern = "[\w]+|[‑–—“”€№…’\"#$%&\'()+,-./:;<>?]" - return re.findall(re.compile(pattern), s) +def _get_all_dimensions(batch: Sequence, level: int = 0, res: Optional[List[List[int]]] = None) -> List[List[int]]: + """Return all presented element sizes of each dimension. + + Args: + batch: Data array. + level: Recursion level. + res: List containing element sizes of each dimension. + + Return: + List, i-th element of which is list containing all presented sized of batch's i-th dimension. + Examples: + >>> x = [[[1], [2, 3]], [[4], [5, 6, 7], [8, 9]]] + >>> _get_all_dimensions(x) + [[2], [2, 3], [1, 2, 1, 3, 2]] -def get_all_dimensions(batch: Sequence, level: int = 0, res: Optional[List[List[int]]] = None) -> List[List[int]]: + """ if not level: res = [[len(batch)]] if len(batch) and isinstance(batch[0], Sized) and not isinstance(batch[0], str): @@ -290,16 +356,38 @@ def get_all_dimensions(batch: Sequence, level: int = 0, res: Optional[List[List[ res.append([]) for item in batch: res[level].append(len(item)) - get_all_dimensions(item, level, res) + _get_all_dimensions(item, level, res) return res -def get_dimensions(batch) -> List[int]: - """""" - return list(map(max, get_all_dimensions(batch))) +def get_dimensions(batch: Sequence) -> List[int]: + """Return maximal size of each batch dimension.""" + return list(map(max, _get_all_dimensions(batch))) + + +def zero_pad(batch: Sequence, + zp_batch: Optional[np.ndarray] = None, + dtype: type = np.float32, + padding: Union[int, float] = 0) -> np.ndarray: + """Fills the end of each array item to make its length maximal along each dimension. + Args: + batch: Initial array. + zp_batch: Padded array. + dtype = Type of padded array. + padding = Number to will initial array with. + + Returns: + Padded array. -def zero_pad(batch, zp_batch=None, dtype=np.float32, padding=0): + Examples: + >>> x = np.array([[1, 2, 3], [4], [5, 6]]) + >>> zero_pad(x) + array([[1., 2., 3.], + [4., 0., 0.], + [5., 6., 0.]], dtype=float32) + + """ if zp_batch is None: dims = get_dimensions(batch) zp_batch = np.ones(dims, dtype=dtype) * padding @@ -311,7 +399,8 @@ def zero_pad(batch, zp_batch=None, dtype=np.float32, padding=0): return zp_batch -def is_str_batch(batch): +def is_str_batch(batch: Iterable) -> bool: + """Checks if iterable argument contains string at any nesting level.""" while True: if isinstance(batch, Iterable): if isinstance(batch, str): @@ -327,7 +416,20 @@ def is_str_batch(batch): return False -def flatten_str_batch(batch): +def flatten_str_batch(batch: Union[str, Iterable]) -> Union[list, chain]: + """Joins all strings from nested lists to one ``itertools.chain``. + + Args: + batch: List with nested lists to flatten. + + Returns: + Generator of flat List[str]. For str ``batch`` returns [``batch``]. + + Examples: + >>> [string for string in flatten_str_batch(['a', ['b'], [['c', 'd']]])] + ['a', 'b', 'c', 'd'] + + """ if isinstance(batch, str): return [batch] else: @@ -391,7 +493,21 @@ def zero_pad_char(batch, dtype=np.float32): return padded_batch -def get_all_elems_from_json(search_json, search_key): +def get_all_elems_from_json(search_json: dict, search_key: str) -> list: + """Returns values by key in all nested dicts. + + Args: + search_json: Dictionary in which one needs to find all values by specific key. + search_key: Key for search. + + Returns: + List of values stored in nested structures by ``search_key``. + + Examples: + >>> get_all_elems_from_json({'a':{'b': [1,2,3]}, 'b':42}, 'b') + [[1, 2, 3], 42] + + """ result = [] if isinstance(search_json, dict): for key in search_json: @@ -406,7 +522,26 @@ def get_all_elems_from_json(search_json, search_key): return result -def check_nested_dict_keys(check_dict: dict, keys: list): +def check_nested_dict_keys(check_dict: dict, keys: list) -> bool: + """Checks if dictionary contains nested keys from keys list. + + Args: + check_dict: Dictionary to check. + keys: Keys list. i-th nested dict of ``check_dict`` should contain dict containing (i+1)-th key + from the ``keys`` list by i-th key. + + Returns: + True if dictionary contains nested keys from keys list, False otherwise. + + Examples: + >>> check_nested_dict_keys({'x': {'y': {'z': 42}}}, ['x', 'y', 'z']) + True + >>> check_nested_dict_keys({'x': {'y': {'z': 42}}}, ['x', 'z', 'y']) + False + >>> check_nested_dict_keys({'x': {'y': 1, 'z': 42}}, ['x', 'y', 'z']) + False + + """ if isinstance(keys, list) and len(keys) > 0: element = check_dict for key in keys: @@ -419,7 +554,19 @@ def check_nested_dict_keys(check_dict: dict, keys: list): return False -def jsonify_data(data): +def jsonify_data(data: Any) -> Any: + """Replaces JSON-non-serializable objects with JSON-serializable. + + Function replaces numpy arrays and numbers with python lists and numbers, tuples is replaces with lists. All other + object types remain the same. + + Args: + data: Object to make JSON-serializable. + + Returns: + Modified input data. + + """ if isinstance(data, (list, tuple)): result = [jsonify_data(item) for item in data] elif isinstance(data, dict): @@ -437,21 +584,30 @@ def jsonify_data(data): return result -def chunk_generator(items_list, chunk_size): +def chunk_generator(items_list: list, chunk_size: int) -> Generator[list, None, None]: + """Yields consecutive slices of list. + + Args: + items_list: List to slice. + chunk_size: Length of slice. + + Yields: + list: ``items_list`` consecutive slices. + + """ for i in range(0, len(items_list), chunk_size): yield items_list[i:i + chunk_size] -def update_dict_recursive(editable_dict: dict, editing_dict: dict) -> None: - """Updates dict recursively +def update_dict_recursive(editable_dict: dict, editing_dict: Mapping) -> None: + """Updates dict recursively. - You need to use this function to update dictionary if depth of editing_dict is more then 1 + You need to use this function to update dictionary if depth of editing_dict is more then 1. Args: - editable_dict: dictionary, that will be edited - editing_dict: dictionary, that contains edits - Returns: - None + editable_dict: Dictionary to edit. + editing_dict: Dictionary containing edits. + """ for k, v in editing_dict.items(): if isinstance(v, collections.Mapping): @@ -460,13 +616,15 @@ def update_dict_recursive(editable_dict: dict, editing_dict: dict) -> None: editable_dict[k] = v -def path_set_md5(url): - """Given a file URL, return a md5 query of the file +def path_set_md5(url: str) -> str: + """Given a file URL, return a md5 query of the file. Args: - url: a given URL + url: A given URL. + Returns: - URL of the md5 file + URL of the md5 file. + """ scheme, netloc, path, query_string, fragment = urlsplit(url) path += '.md5' @@ -474,15 +632,16 @@ def path_set_md5(url): return urlunsplit((scheme, netloc, path, query_string, fragment)) -def set_query_parameter(url, param_name, param_value): +def set_query_parameter(url: str, param_name: str, param_value: str) -> str: """Given a URL, set or replace a query parameter and return the modified URL. Args: - url: a given URL - param_name: the parameter name to add - param_value: the parameter value + url: A given URL. + param_name: The parameter name to add. + param_value: The parameter value. + Returns: - URL with the added parameter + URL with the added parameter. """ scheme, netloc, path, query_string, fragment = urlsplit(url) diff --git a/deeppavlov/models/slotfill/slotfill_raw.py b/deeppavlov/models/slotfill/slotfill_raw.py index acbc0bcb08..2689eb2349 100644 --- a/deeppavlov/models/slotfill/slotfill_raw.py +++ b/deeppavlov/models/slotfill/slotfill_raw.py @@ -20,7 +20,6 @@ from overrides import overrides from deeppavlov.core.common.registry import register -from deeppavlov.core.data.utils import tokenize_reg from deeppavlov.core.models.component import Component from deeppavlov.core.models.serializable import Serializable @@ -40,9 +39,6 @@ def __init__(self, threshold: float = 0.7, return_all: bool = False, **kwargs): @overrides def __call__(self, batch, *args, **kwargs): - if isinstance(batch[0], str): - batch = [tokenize_reg(instance.strip()) for instance in batch] - slots = [{}] * len(batch) m = [i for i, v in enumerate(batch) if v] From ce84ee1d101cc5eb6aae334bc03c967bc28fa89d Mon Sep 17 00:00:00 2001 From: Fedor Ignatov Date: Wed, 9 Oct 2019 12:08:51 +0300 Subject: [PATCH 10/28] refactor: chained comparisons, boolian variable check simplified (#1024) --- deeppavlov/dataset_iterators/siamese_iterator.py | 2 +- deeppavlov/models/kbqa/entity_linking.py | 2 +- .../spelling_correction/levenshtein/levenshtein_searcher.py | 2 +- deeppavlov/skills/ecommerce_skill/bleu_retrieve.py | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/deeppavlov/dataset_iterators/siamese_iterator.py b/deeppavlov/dataset_iterators/siamese_iterator.py index c8e644393c..222a09efed 100644 --- a/deeppavlov/dataset_iterators/siamese_iterator.py +++ b/deeppavlov/dataset_iterators/siamese_iterator.py @@ -100,5 +100,5 @@ def gen_batches(self, batch_size: int, data_type: str = "train", shuffle: bool = if data_type in ["valid", "test"]: for i in range(num_steps + 1): context_response_data = data[i * batch_size:(i + 1) * batch_size] - if context_response_data != []: + if context_response_data: yield tuple(zip(*context_response_data)) diff --git a/deeppavlov/models/kbqa/entity_linking.py b/deeppavlov/models/kbqa/entity_linking.py index 50a141ee89..90090fe206 100644 --- a/deeppavlov/models/kbqa/entity_linking.py +++ b/deeppavlov/models/kbqa/entity_linking.py @@ -180,7 +180,7 @@ def fuzzy_entity_search(self, entity: str) -> List[Tuple[Tuple, str]]: candidates = [] for title in self.name_to_q: length_ratio = len(title) / word_length - if length_ratio > 0.75 and length_ratio < 1.25: + if 0.75 < length_ratio < 1.25: ratio = fuzz.ratio(title, entity) if ratio > 70: entity_candidates = self.name_to_q.get(title, []) diff --git a/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py b/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py index eaa5b8fa0d..3799ec16e0 100644 --- a/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py +++ b/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py @@ -371,7 +371,7 @@ def distance(self, first, second, return_transduction = False): add_pred = (lambda x, y: (y == np.inf or x < y)) else: add_pred = (lambda x, y: (y == np.inf or x <= y)) - clear_pred = (lambda x, y: (y < np.inf and x < y)) + clear_pred = (lambda x, y: x < y < np.inf) update_func = lambda x, y: min(x, y) costs, backtraces = self._fill_levenshtein_table(first, second, update_func, add_pred, clear_pred) diff --git a/deeppavlov/skills/ecommerce_skill/bleu_retrieve.py b/deeppavlov/skills/ecommerce_skill/bleu_retrieve.py index 9a262c56fe..5c5cbe3e5b 100644 --- a/deeppavlov/skills/ecommerce_skill/bleu_retrieve.py +++ b/deeppavlov/skills/ecommerce_skill/bleu_retrieve.py @@ -218,8 +218,7 @@ def _filter_state(self, state: Dict[Any, Any], results_args_sim: List[int]) -> L price = value log.debug(f"Items before price filtering {len(results_args_sim)} with price {price}") results_args_sim = [idx for idx in results_args_sim - if self.preprocess.price(self.ec_data[idx]) >= price[0] and - self.preprocess.price(self.ec_data[idx]) <= price[1] and + if price[0] <= self.preprocess.price(self.ec_data[idx]) <= price[1] and self.preprocess.price(self.ec_data[idx]) != 0] log.debug(f"Items after price filtering {len(results_args_sim)}") From 4fb10ef3a17104469bb257fe4dee6cc84dfc1519 Mon Sep 17 00:00:00 2001 From: Aleksei Lymar Date: Wed, 9 Oct 2019 12:09:18 +0300 Subject: [PATCH 11/28] refactor: optimize and document zero pad truncate (#1020) * refactor: optimize zero_pad_truncate function code * docs: document zero_pad_truncate function code * docs: comment on types discrepancy in zero_pad_truncate * docs: fix a typo * fix: use correct numeric types for numpy * chore: remove zero_pad_char that is not used anywhere --- deeppavlov/core/data/utils.py | 87 +++++++++++++---------------------- 1 file changed, 32 insertions(+), 55 deletions(-) diff --git a/deeppavlov/core/data/utils.py b/deeppavlov/core/data/utils.py index 928d01ff97..e1b3cf29e7 100644 --- a/deeppavlov/core/data/utils.py +++ b/deeppavlov/core/data/utils.py @@ -23,7 +23,7 @@ from itertools import chain from logging import getLogger from pathlib import Path -from typing import Any, Generator, Iterable, List, Mapping, Optional, Sequence, Sized, Union +from typing import Any, Generator, Iterable, List, Mapping, Optional, Sequence, Sized, Union, Collection from urllib.parse import urlencode, parse_qs, urlsplit, urlunsplit, urlparse import numpy as np @@ -436,61 +436,38 @@ def flatten_str_batch(batch: Union[str, Iterable]) -> Union[list, chain]: return chain(*[flatten_str_batch(sample) for sample in batch]) -def zero_pad_truncate(batch, max_len, pad='post', trunc='post', dtype=np.float32): - batch_size = len(batch) - if isinstance(batch[0][0], (int, np.int)): - padded_batch = np.zeros([batch_size, max_len], dtype=np.int32) - for n, utterance in enumerate(batch): - if len(utterance) > max_len: - if trunc == 'post': - padded_batch[n, :] = utterance[:max_len] - elif trunc == 'pre': - padded_batch[n, :] = utterance[-max_len:] - else: - if pad == 'post': - padded_batch[n, :len(utterance)] = utterance - elif pad == 'pre': - padded_batch[n, -len(utterance):] = utterance - else: - n_features = len(batch[0][0]) - padded_batch = np.zeros([batch_size, max_len, n_features], dtype=dtype) - for n, utterance in enumerate(batch): - if len(utterance) > max_len: - if trunc == 'post': - for k, token_features in enumerate(utterance[:max_len]): - padded_batch[n, k] = token_features - elif trunc == 'pre': - for k, token_features in enumerate(utterance[-max_len:]): - padded_batch[n, k] = token_features - else: - if pad == 'post': - for k, token_features in enumerate(utterance): - padded_batch[n, k] = token_features - elif pad == 'pre': - for k, token_features in enumerate(utterance): - padded_batch[n, k + max_len - len(utterance)] = token_features - return padded_batch - - -def zero_pad_char(batch, dtype=np.float32): - if len(batch) == 1 and len(batch[0]) == 0: - return np.array([], dtype=dtype) - batch_size = len(batch) - max_len = max(len(utterance) for utterance in batch) - max_token_len = max(len(ch) for token in batch for ch in token) - if isinstance(batch[0][0][0], (int, np.int)): - padded_batch = np.zeros([batch_size, max_len, max_token_len], dtype=np.int32) - for n, utterance in enumerate(batch): - for k, token in enumerate(utterance): - padded_batch[n, k, :len(token)] = token +def zero_pad_truncate(batch: Sequence[Sequence[Union[int, float, np.integer, np.floating, + Sequence[Union[int, float, np.integer, np.floating]]]]], + max_len: int, pad: str = 'post', trunc: str = 'post', + dtype: Optional[Union[type, str]] = None) -> np.ndarray: + """ + + Args: + batch: assumes a batch of lists of word indexes or their vector representations + max_len: resulting length of every batch item + pad: how to pad shorter batch items: can be ``'post'`` or ``'pre'`` + trunc: how to truncate a batch item: can be ``'post'`` or ``'pre'`` + dtype: overrides dtype for the resulting ``ndarray`` if specified, + otherwise ``np.int32`` is used for 2-d arrays and ``np.float32`` — for 3-d arrays + + Returns: + a 2-d array of size ``(len(batch), max_len)`` or a 3-d array of size ``(len(batch), max_len, len(batch[0][0]))`` + """ + if isinstance(batch[0][0], Collection): # ndarray behaves like a Sequence without actually being one + size = (len(batch), max_len, len(batch[0][0])) + dtype = dtype or np.float32 else: - n_features = len(batch[0][0][0]) - padded_batch = np.zeros([batch_size, max_len, max_token_len, n_features], dtype=dtype) - for n, utterance in enumerate(batch): - for k, token in enumerate(utterance): - for q, char_features in enumerate(token): - padded_batch[n, k, q] = char_features - return padded_batch + size = (len(batch), max_len) + dtype = dtype or np.int32 + + padded_batch = np.zeros(size, dtype=dtype) + for i, batch_item in enumerate(batch): + if len(batch_item) > max_len: # trunc + padded_batch[i] = batch_item[slice(max_len) if trunc == 'post' else slice(-max_len, None)] + else: # pad + padded_batch[i, slice(len(batch_item)) if pad == 'post' else slice(-len(batch_item), None)] = batch_item + + return np.asarray(padded_batch) def get_all_elems_from_json(search_json: dict, search_key: str) -> list: From 5792d13e7064e937a315799d8ce6ba182c2e58cd Mon Sep 17 00:00:00 2001 From: Alexey Litinsky Date: Mon, 14 Oct 2019 12:51:47 +0300 Subject: [PATCH 12/28] feat: add CORS support to riseapi (#1038) --- deeppavlov/utils/server/server.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deeppavlov/utils/server/server.py b/deeppavlov/utils/server/server.py index 2b00a42ed1..92a0c3e1d2 100644 --- a/deeppavlov/utils/server/server.py +++ b/deeppavlov/utils/server/server.py @@ -26,6 +26,7 @@ from pydantic.fields import Field from pydantic.main import MetaModel from starlette.responses import RedirectResponse +from starlette.middleware.cors import CORSMiddleware from deeppavlov.core.agent.dialog_logger import DialogLogger from deeppavlov.core.commands.infer import build_model @@ -51,6 +52,14 @@ def filter(self, record: logging.LogRecord) -> bool: uvicorn_log.addFilter(ProbeFilter()) app = FastAPI(__file__) +app.add_middleware( + CORSMiddleware, + allow_origins=['*'], + allow_credentials=True, + allow_methods=['*'], + allow_headers=['*'] +) + dialog_logger = DialogLogger(agent_name='dp_api') From 88360efa612fbcdc065c449b8c06de59e9724593 Mon Sep 17 00:00:00 2001 From: Dilyara Baymurzina Date: Thu, 17 Oct 2019 12:17:06 +0300 Subject: [PATCH 13/28] chore: remove per_item_accuracy (#1037) --- deeppavlov/core/common/metrics_registry.json | 1 - deeppavlov/metrics/accuracy.py | 11 ----------- 2 files changed, 12 deletions(-) diff --git a/deeppavlov/core/common/metrics_registry.json b/deeppavlov/core/common/metrics_registry.json index 4ac31b9961..15053bdb14 100644 --- a/deeppavlov/core/common/metrics_registry.json +++ b/deeppavlov/core/common/metrics_registry.json @@ -12,7 +12,6 @@ "log_loss": "deeppavlov.metrics.log_loss:sk_log_loss", "ner_f1": "deeppavlov.metrics.fmeasure:ner_f1", "ner_token_f1": "deeppavlov.metrics.fmeasure:ner_token_f1", - "per_item_accuracy": "deeppavlov.metrics.accuracy:per_item_accuracy", "per_item_bleu": "deeppavlov.metrics.bleu:per_item_bleu", "per_item_dialog_accuracy": "deeppavlov.metrics.accuracy:per_item_dialog_accuracy", "per_item_dialog_bleu": "deeppavlov.metrics.bleu:per_item_dialog_bleu", diff --git a/deeppavlov/metrics/accuracy.py b/deeppavlov/metrics/accuracy.py index 03d9bc1f0d..ff5e45a6aa 100644 --- a/deeppavlov/metrics/accuracy.py +++ b/deeppavlov/metrics/accuracy.py @@ -61,17 +61,6 @@ def slots_accuracy(y_true, y_predicted): return accuracy(y_true, y_predicted) -@register_metric('per_item_accuracy') -def per_item_accuracy(y_true, y_predicted): - if isinstance(y_true[0], (tuple, list)): - y_true = (y[0] for y in y_true) - y_true = list(itertools.chain(*y_true)) - y_predicted = itertools.chain(*y_predicted) - examples_len = len(y_true) - correct = sum([y1 == y2 for y1, y2 in zip(y_true, y_predicted)]) - return correct / examples_len if examples_len else 0 - - @register_metric('per_token_accuracy') def per_token_accuracy(y_true, y_predicted): y_true = list(itertools.chain(*y_true)) From 6b37075a1d9a6c2735e1cba63346790e4427bc14 Mon Sep 17 00:00:00 2001 From: Fedor Ignatov Date: Tue, 22 Oct 2019 10:44:50 +0300 Subject: [PATCH 14/28] feat: add ability to use functions in chainer pipe (#1041) * feat: added ability to use functions as elements of Chainer pipe Also StrLower class is replaced by str_lower function Added license info in some files and removed unnecessary imports * fix: serialization test fail for configs with functions * fix: chainer deserialization error if function in pipe * fix: fixed skipping serialization test --- deeppavlov/core/common/chainer.py | 13 +++--- deeppavlov/core/common/params.py | 43 +++++++++++-------- deeppavlov/core/common/registry.json | 2 +- deeppavlov/models/morpho_tagger/cells.py | 14 ++++++ deeppavlov/models/morpho_tagger/common.py | 19 ++++++-- .../models/morpho_tagger/common_tagger.py | 18 ++++++-- deeppavlov/models/morpho_tagger/lemmatizer.py | 14 ++++++ deeppavlov/models/preprocessors/str_lower.py | 30 +++++-------- docs/apiref/models/preprocessors.rst | 4 +- docs/intro/configuration.rst | 7 +-- examples/classification_tutorial.ipynb | 5 +-- 11 files changed, 109 insertions(+), 60 deletions(-) diff --git a/deeppavlov/core/common/chainer.py b/deeppavlov/core/common/chainer.py index c5befdf115..9f8a2d7f01 100644 --- a/deeppavlov/core/common/chainer.py +++ b/deeppavlov/core/common/chainer.py @@ -15,6 +15,7 @@ import pickle from itertools import islice from logging import getLogger +from types import FunctionType from typing import Union, Tuple, List, Optional, Hashable, Reversible from deeppavlov.core.common.errors import ConfigError @@ -122,8 +123,8 @@ def _repr_pretty_(self, p, cycle): break p.pretty(component) - def append(self, component: Component, in_x: [str, list, dict] = None, out_params: [str, list] = None, - in_y: [str, list, dict] = None, main: bool = False): + def append(self, component: Union[Component, FunctionType], in_x: [str, list, dict] = None, + out_params: [str, list] = None, in_y: [str, list, dict] = None, main: bool = False): if isinstance(in_x, str): in_x = [in_x] if isinstance(in_y, str): @@ -301,10 +302,12 @@ def destroy(self): def serialize(self) -> bytes: data = [] for in_params, out_params, component in self.train_pipe: - data.append(component.serialize()) + serialized = component.serialize() if isinstance(component, Component) else None + data.append(serialized) return pickle.dumps(data, protocol=4) def deserialize(self, data: bytes) -> None: data = pickle.loads(data) - for in_params, out_params, component in self.train_pipe: - component.deserialize(data) + for (in_params, out_params, component), component_data in zip(self.train_pipe, data): + if isinstance(component, Component): + component.deserialize(component_data) diff --git a/deeppavlov/core/common/params.py b/deeppavlov/core/common/params.py index 426a0df046..8e9afbd1a2 100644 --- a/deeppavlov/core/common/params.py +++ b/deeppavlov/core/common/params.py @@ -14,7 +14,8 @@ import inspect from logging import getLogger -from typing import Dict, Any +from types import FunctionType +from typing import Any, Dict, Union from deeppavlov.core.commands.utils import expand_path, parse_config from deeppavlov.core.common.errors import ConfigError @@ -54,7 +55,7 @@ def _init_param(param, mode): return param -def from_params(params: Dict, mode: str = 'infer', serialized: Any = None, **kwargs) -> Component: +def from_params(params: Dict, mode: str = 'infer', serialized: Any = None, **kwargs) -> Union[Component, FunctionType]: """Builds and returns the Component from corresponding dictionary of parameters.""" # what is passed in json: config_params = {k: _resolve(v) for k, v in params.items()} @@ -91,25 +92,29 @@ def from_params(params: Dict, mode: str = 'infer', serialized: Any = None, **kwa e = ConfigError('Component config has no `class_name` nor `ref` fields') log.exception(e) raise e - cls = get_model(cls_name) + obj = get_model(cls_name) - # find the submodels params recursively - config_params = {k: _init_param(v, mode) for k, v in config_params.items()} + if inspect.isclass(obj): + # find the submodels params recursively + config_params = {k: _init_param(v, mode) for k, v in config_params.items()} - try: - spec = inspect.getfullargspec(cls) - if 'mode' in spec.args+spec.kwonlyargs or spec.varkw is not None: - kwargs['mode'] = mode - - component = cls(**dict(config_params, **kwargs)) try: - _refs[config_params['id']] = component - except KeyError: - pass - except Exception: - log.exception("Exception in {}".format(cls)) - raise + spec = inspect.getfullargspec(obj) + if 'mode' in spec.args+spec.kwonlyargs or spec.varkw is not None: + kwargs['mode'] = mode + + component = obj(**dict(config_params, **kwargs)) + try: + _refs[config_params['id']] = component + except KeyError: + pass + except Exception: + log.exception("Exception in {}".format(obj)) + raise + + if serialized is not None: + component.deserialize(serialized) + else: + component = obj - if serialized is not None: - component.deserialize(serialized) return component diff --git a/deeppavlov/core/common/registry.json b/deeppavlov/core/common/registry.json index 0d50ff6de9..e630f80b42 100644 --- a/deeppavlov/core/common/registry.json +++ b/deeppavlov/core/common/registry.json @@ -137,7 +137,7 @@ "squad_preprocessor": "deeppavlov.models.preprocessors.squad_preprocessor:SquadPreprocessor", "squad_vocab_embedder": "deeppavlov.models.preprocessors.squad_preprocessor:SquadVocabEmbedder", "static_dictionary": "deeppavlov.vocabs.typos:StaticDictionary", - "str_lower": "deeppavlov.models.preprocessors.str_lower:StrLower", + "str_lower": "deeppavlov.models.preprocessors.str_lower:str_lower", "str_token_reverser": "deeppavlov.models.preprocessors.str_token_reverser:StrTokenReverser", "str_utf8_encoder": "deeppavlov.models.preprocessors.str_utf8_encoder:StrUTF8Encoder", "stream_spacy_tokenizer": "deeppavlov.models.tokenizers.spacy_tokenizer:StreamSpacyTokenizer", diff --git a/deeppavlov/models/morpho_tagger/cells.py b/deeppavlov/models/morpho_tagger/cells.py index d887003b07..b1fc40e72c 100644 --- a/deeppavlov/models/morpho_tagger/cells.py +++ b/deeppavlov/models/morpho_tagger/cells.py @@ -1,3 +1,17 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import keras.activations as kact import keras.backend as kb import keras.initializers as kinit diff --git a/deeppavlov/models/morpho_tagger/common.py b/deeppavlov/models/morpho_tagger/common.py index c72f8a7b9e..9abf0bc06e 100644 --- a/deeppavlov/models/morpho_tagger/common.py +++ b/deeppavlov/models/morpho_tagger/common.py @@ -1,14 +1,25 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys from pathlib import Path -from typing import List, Dict, Union, Optional +from typing import List, Union, Optional from deeppavlov.core.commands.infer import build_model from deeppavlov.core.commands.utils import expand_path, parse_config -from deeppavlov.core.common.params import from_params -from deeppavlov.core.common.registry import get_model from deeppavlov.core.common.registry import register from deeppavlov.core.models.component import Component -from deeppavlov.dataset_iterators.morphotagger_iterator import MorphoTaggerDatasetIterator from deeppavlov.dataset_readers.morphotagging_dataset_reader import read_infile from deeppavlov.models.morpho_tagger.common_tagger import make_pos_and_tag diff --git a/deeppavlov/models/morpho_tagger/common_tagger.py b/deeppavlov/models/morpho_tagger/common_tagger.py index 1b0d9731c8..6b7b905b39 100644 --- a/deeppavlov/models/morpho_tagger/common_tagger.py +++ b/deeppavlov/models/morpho_tagger/common_tagger.py @@ -1,6 +1,18 @@ -""" -File containing common operation with keras.backend objects -""" +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""File containing common operation with keras.backend objects""" from typing import Union, Optional, Tuple diff --git a/deeppavlov/models/morpho_tagger/lemmatizer.py b/deeppavlov/models/morpho_tagger/lemmatizer.py index 44e9b1be78..2f0450ab51 100644 --- a/deeppavlov/models/morpho_tagger/lemmatizer.py +++ b/deeppavlov/models/morpho_tagger/lemmatizer.py @@ -1,3 +1,17 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import abstractmethod from typing import List, Optional diff --git a/deeppavlov/models/preprocessors/str_lower.py b/deeppavlov/models/preprocessors/str_lower.py index 11f5aa68cc..41de4726b0 100644 --- a/deeppavlov/models/preprocessors/str_lower.py +++ b/deeppavlov/models/preprocessors/str_lower.py @@ -15,27 +15,19 @@ from typing import Union from deeppavlov.core.common.registry import register -from deeppavlov.core.models.component import Component @register('str_lower') -class StrLower(Component): - """Component for converting strings to lowercase at any level of lists nesting +def str_lower(batch: Union[str, list, tuple]): + """Recursively search for strings in a list and convert them to lowercase - """ - def __init__(self, *args, **kwargs): - pass - - def __call__(self, batch: Union[str, list, tuple]): - """Recursively search for strings in a list and convert them to lowercase + Args: + batch: a string or a list containing strings at some level of nesting - Args: - batch: a string or a list containing strings at some level of nesting - - Returns: - the same structure where all strings are converted to lowercase - """ - if isinstance(batch, (list, tuple)): - return [self(line) for line in batch] - else: - return batch.lower() + Returns: + the same structure where all strings are converted to lowercase + """ + if isinstance(batch, str): + return batch.lower() + else: + return list(map(str_lower, batch)) diff --git a/docs/apiref/models/preprocessors.rst b/docs/apiref/models/preprocessors.rst index f35c31e904..5561f511db 100644 --- a/docs/apiref/models/preprocessors.rst +++ b/docs/apiref/models/preprocessors.rst @@ -27,9 +27,7 @@ deeppavlov.models.preprocessors .. autoclass:: deeppavlov.models.preprocessors.siamese_preprocessor.SiamesePreprocessor -.. autoclass:: deeppavlov.models.preprocessors.str_lower.StrLower - - .. automethod:: __call__ +.. autofunction:: deeppavlov.models.preprocessors.str_lower.str_lower .. autoclass:: deeppavlov.models.preprocessors.str_token_reverser.StrTokenReverser diff --git a/docs/intro/configuration.rst b/docs/intro/configuration.rst index c828409f8a..9f873c5e9c 100644 --- a/docs/intro/configuration.rst +++ b/docs/intro/configuration.rst @@ -25,7 +25,7 @@ components: .. code:: python { - "class_name": "deeppavlov.models.preprocessors.str_lower:StrLower", + "class_name": "deeppavlov.models.preprocessors.str_lower:str_lower", "in": ["x"], "out": ["x_lower"] }, @@ -35,6 +35,8 @@ components: "out": ["x_tokens"] }, +Pipeline elements could be child classes of :class:`~deeppavlov.core.models.component.Component` or functions. + Each :class:`~deeppavlov.core.models.component.Component` in the pipeline must implement method :meth:`__call__` and has ``class_name`` parameter, which is its registered codename, or full name of any python class in the form of ``"module_name:ClassName"``. It can also have any other parameters which repeat its :meth:`__init__` method arguments. @@ -307,8 +309,7 @@ Preprocessor is a component that processes batch of samples. apostrophe ``'``, transforming more than three the same symbols to two symbols. - - :class:`~deeppavlov.models.preprocessors.str_lower.StrLower` (registered - as ``str_lower``) converts samples to lowercase. + - :meth:`~deeppavlov.models.preprocessors.str_lower.str_lower` converts samples to lowercase. * Already implemented universal preprocessors of another type of features: diff --git a/examples/classification_tutorial.ipynb b/examples/classification_tutorial.ipynb index e0ab7dccba..55d312e3c1 100644 --- a/examples/classification_tutorial.ipynb +++ b/examples/classification_tutorial.ipynb @@ -317,7 +317,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "`StrLower` lowercases texts." + "`str_lower` lowercases texts." ] }, { @@ -343,7 +343,7 @@ } ], "source": [ - "from deeppavlov.models.preprocessors.str_lower import StrLower" + "from deeppavlov.models.preprocessors.str_lower import str_lower" ] }, { @@ -363,7 +363,6 @@ } ], "source": [ - "str_lower = StrLower()\n", "str_lower(['Is it freezing in Offerman, California?'])" ] }, From 2c4482d010d59cc770cb15425874083b6b16f006 Mon Sep 17 00:00:00 2001 From: Fedor Ignatov Date: Tue, 22 Oct 2019 10:45:10 +0300 Subject: [PATCH 15/28] fix: fix socket test freeze in case if model sends back nothing (#1045) * fix: fixed socket test freeze in case if model sends back nothing * refactor: socket api infer timeout increased from 20 s to 60 s * feat: removed timeout error handling * feat: one test model replaced by original one in tests --- .../ranking_ubuntu_v2_interact_test.json | 148 ------------------ tests/test_quick_start.py | 8 +- 2 files changed, 6 insertions(+), 150 deletions(-) delete mode 100644 tests/test_configs/ranking/ranking_ubuntu_v2_interact_test.json diff --git a/tests/test_configs/ranking/ranking_ubuntu_v2_interact_test.json b/tests/test_configs/ranking/ranking_ubuntu_v2_interact_test.json deleted file mode 100644 index 708f1434f2..0000000000 --- a/tests/test_configs/ranking/ranking_ubuntu_v2_interact_test.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "dataset_reader": { - "class_name": "ubuntu_v2_reader", - "data_path": "{DOWNLOADS_PATH}/ubuntu_v2_data" - }, - "dataset_iterator": { - "class_name": "siamese_iterator", - "num_samples": 1024, - "seed": 243 - }, - "chainer": { - "in": [ - "x" - ], - "in_y": [ - "y" - ], - "pipe": [ - { - "id": "preproc", - "class_name": "siamese_preprocessor", - "use_matrix": true, - "num_ranking_samples": 2, - "max_sequence_length": 50, - "fit_on": [ - "x" - ], - "in": [ - "x" - ], - "out": [ - "x_proc" - ], - "sent_vocab": { - "id": "siam_sent_vocab", - "class_name": "simple_vocab", - "save_path": "{MODELS_PATH}/ubuntu_v2_vocabs/sent.dict", - "load_path": "{MODELS_PATH}/ubuntu_v2_vocabs/sent.dict" - }, - "tokenizer": { - "class_name": "split_tokenizer" - }, - "vocab": { - "id": "siam_vocab", - "class_name": "simple_vocab", - "save_path": "{MODELS_PATH}/ubuntu_v2_vocabs/tok.dict", - "load_path": "{MODELS_PATH}/ubuntu_v2_vocabs/tok.dict" - }, - "embedder": { - "id": "siam_embedder", - "class_name": "fasttext", - "load_path": "{DOWNLOADS_PATH}/embeddings/ft_dummy_300.bin", - "dim": 300 - } - }, - { - "id": "embeddings", - "class_name": "emb_mat_assembler", - "embedder": "#siam_embedder", - "vocab": "#siam_vocab" - }, - { - "id": "model", - "class_name": "bilstm_nn", - "len_vocab": "#siam_vocab.len", - "use_matrix": "#preproc.use_matrix", - "max_sequence_length": "#preproc.max_sequence_length", - "embedding_dim": "#siam_embedder.dim", - "seed": 243, - "hidden_dim": 300, - "emb_matrix": "#embeddings.emb_mat", - "learning_rate": 0.001, - "triplet_loss": false, - "batch_size": 256, - "save_path": "{MODELS_PATH}/ubuntu_v2_model/model_weights.h5", - "load_path": "{MODELS_PATH}/ubuntu_v2_model/model_weights.h5" - }, - { - "in": [ - "x_proc" - ], - "in_y": [ - "y" - ], - "out": [ - "y_predicted" - ], - "class_name": "siamese_predictor", - "model": "#model", - "batch_size": "#model.batch_size", - "interact_pred_num": 3, - "attention": true, - "responses": "#siam_sent_vocab", - "preproc_func": "#preproc.__call__" - } - ], - "out": [ - "y_predicted" - ] - }, - "train": { - "epochs": 200, - "batch_size": 256, - "pytest_max_batches": 2, - "train_metrics": [], - "metrics": [ - "r@1", - "rank_response" - ], - "validation_patience": 10, - "val_every_n_epochs": 1, - "log_every_n_batches": 1000, - "class_name": "nn_trainer", - "evaluation_targets": [ - "valid", - "test" - ] - }, - "metadata": { - "variables": { - "ROOT_PATH": "~/.deeppavlov", - "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", - "MODELS_PATH": "{ROOT_PATH}/models" - }, - "requirements": [ - "{DEEPPAVLOV_PATH}/requirements/tf.txt", - "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" - ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, - "download": [ - { - "url": "http://files.deeppavlov.ai/deeppavlov_data/ubuntu_v2_ranking.tar.gz", - "subdir": "{MODELS_PATH}" - }, - { - "url": "http://files.deeppavlov.ai/datasets/ubuntu_v2_data.tar.gz", - "subdir": "{DOWNLOADS_PATH}/ubuntu_v2_data" - }, - { - "url": "http://files.deeppavlov.ai/deeppavlov_data/embeddings/ft_dummy_300.bin", - "subdir": "{DOWNLOADS_PATH}/embeddings" - } - ] - } -} \ No newline at end of file diff --git a/tests/test_quick_start.py b/tests/test_quick_start.py index 1093ca7194..396798c4a0 100644 --- a/tests/test_quick_start.py +++ b/tests/test_quick_start.py @@ -163,7 +163,7 @@ ("ranking/ranking_insurance_test.json", "ranking", ('TI',)): [ONE_ARGUMENT_INFER_CHECK], ("ranking/ranking_insurance_interact_test.json", "ranking", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], ("ranking/ranking_ubuntu_v2_test.json", "ranking", ('TI',)): [ONE_ARGUMENT_INFER_CHECK], - ("ranking/ranking_ubuntu_v2_interact_test.json", "ranking", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], + ("ranking/ranking_ubuntu_v2_interact.json", "ranking", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], ("ranking/ranking_ubuntu_v2_mt_test.json", "ranking", ('TI',)): [ONE_ARGUMENT_INFER_CHECK], ("ranking/ranking_ubuntu_v2_mt_interact_test.json", "ranking", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], ("ranking/paraphrase_ident_paraphraser_test.json", "ranking", ('TI',)): [ONE_ARGUMENT_INFER_CHECK], @@ -460,6 +460,7 @@ def interact_socket(config_path, socket_type): with socket.socket(address_family, socket.SOCK_STREAM) as s: s.connect(connect_arg) s.sendall(dumped_socket_payload.encode('utf-8')) + s.settimeout(60) data = b'' try: while True: @@ -471,7 +472,10 @@ def interact_socket(config_path, socket_type): break except BlockingIOError: pass - resp = json.loads(data) + try: + resp = json.loads(data) + except json.decoder.JSONDecodeError: + raise ValueError(f"Can't decode model response {data}") assert resp['status'] == 'OK', f"{socket_type} socket request returned status: {resp['status']}"\ f" with {config_path}\n{logfile.getvalue().decode()}" From 242055644d9e215ef7d01a3c4e997e2645fcc701 Mon Sep 17 00:00:00 2001 From: Fedor Ignatov Date: Fri, 25 Oct 2019 16:17:42 +0300 Subject: [PATCH 16/28] feat: Skills and agent moved to deprecated (#1040) * feat: Flask was completely replaced by Fastapi - Alexa, Alice and ms_bot_framework modes now on FastAPI - Flask removed from requirements - Docs endpoint name was removed from REST server config file * feat: added async to riseapi model and probe endpoints and to alice * feat: added async to riseapi * docs: added license and docstrings to data models of Alice ans Alexa * refactor: skills and agent moved to utils.deprecated * fix: removed data_model files form Alice and Alexa * refactor: deprecated moved from utils to deeppavlov Aiml and rasa skills moved from deprecated * refactor: ms_bot's framework and alexa bots got parent class bot agent generator moved to bot parent class * feat: removed multi-instance bot option, run method moved to BaseBot * refactor: OutGateway class removed, methods put to ms_bot_framework conv * feat: bot attribute removed from ms_bot conversation class * feat: common parent class for msbot and alexa conversations timers moved to BaseConversation class * refactor: removed multiple sessions from ms_bot * ms_bot auth params moved to server config file * fix: type `redirect_root_do_docs` in server.server * refactor: removed `stateful` param from everywhere * feat: removed default_skill_wrap parameter from everywhere * feat: removed deprecated from Alexa connector * feat: alexa works * refactor: bots and conversations moved to utils.connector dir * refactor: Bot added to Alice (removed skills, agent) * refactor: removed deprecated structures from telegram connector * refactor: configs refactored, server code simplified * refactor: configs in bots and conversations * refactor: conversations, bots * refactor: dsl_skill moved from deprecated * refactor: DialoLogger removed from deprecated Dialog logging added to channel connectors * refatcor: removed deprecated components from not deprecated skills * refactor: Connectors docs and interfaces * fix: fixed ConnectionResetError for requests in one session * fix: fixed aiml and dsl tests removed unnecessary imports * refactor: deep.py, fix small typos * docs: added breaking changes for 0.7.0 * docs: added links to readme.md Co-Authored-By: Aleksei Lymar * docs: changes in breaking changes * docs: removed irrelevant breaking change * fix: wrong argument parsing in deep.py * refactor: reduced line length --- README.md | 12 + deeppavlov/__init__.py | 2 +- deeppavlov/agents/hello_bot_agent/__init__.py | 0 deeppavlov/agents/processors/__init__.py | 0 deeppavlov/agents/rich_content/__init__.py | 0 deeppavlov/configs/skills/aiml_skill.json | 2 - .../{dsl_skill => skills}/dsl_skill.json | 0 deeppavlov/contrib/__init__.py | 0 deeppavlov/contrib/skills/__init__.py | 0 deeppavlov/core/agent/__init__.py | 1 - deeppavlov/core/common/chainer.py | 2 +- deeppavlov/core/common/registry.json | 4 +- deeppavlov/core/skill/__init__.py | 0 deeppavlov/deep.py | 113 ++-- deeppavlov/{agents => deprecated}/__init__.py | 0 deeppavlov/deprecated/agent/__init__.py | 4 + .../{core => deprecated}/agent/agent.py | 4 +- .../{core => deprecated}/agent/filter.py | 0 .../{core => deprecated}/agent/processor.py | 0 .../agent/rich_content.py | 0 .../agents}/__init__.py | 0 .../agents/default_agent/__init__.py | 1 + .../agents/default_agent/default_agent.py | 13 +- .../agents/ecommerce_agent/__init__.py | 1 + .../agents/ecommerce_agent/ecommerce_agent.py | 10 +- .../deprecated/agents/filters/__init__.py | 1 + .../agents/filters/transparent_filter.py | 2 +- .../agents/hello_bot_agent}/__init__.py | 0 .../agents/hello_bot_agent/hello_bot_agent.py | 6 +- .../deprecated/agents/processors/__init__.py | 3 + .../default_rich_content_processor.py | 5 +- .../processors/highest_confidence_selector.py | 2 +- .../agents/processors/random_selector.py | 2 +- .../agents/rich_content/__init__.py | 1 + .../rich_content/default_rich_content.py | 2 +- deeppavlov/deprecated/skill/__init__.py | 1 + .../{core => deprecated}/skill/skill.py | 0 .../filters => deprecated/skills}/__init__.py | 0 .../skills/default_skill/__init__.py | 1 + .../skills/default_skill/default_skill.py | 2 +- .../skills/ecommerce_skill/__init__.py | 3 + .../skills/ecommerce_skill/bleu_retrieve.py | 5 +- .../skills/ecommerce_skill/tfidf_retrieve.py | 3 + .../skills/pattern_matching_skill/__init__.py | 1 + .../pattern_matching_skill.py | 16 +- .../similarity_matching_skill/__init__.py | 0 .../similarity_matching_skill.py | 18 +- deeppavlov/evolve.py | 28 +- deeppavlov/models/bert/bert_ner.py | 6 +- .../models/preprocessors/bert_preprocessor.py | 2 +- deeppavlov/skills/aiml_skill/__init__.py | 1 + deeppavlov/skills/aiml_skill/aiml_skill.py | 34 +- deeppavlov/skills/default_skill/__init__.py | 0 deeppavlov/skills/dsl_skill/__init__.py | 3 + deeppavlov/skills/dsl_skill/context.py | 1 - deeppavlov/skills/dsl_skill/dsl_skill.py | 3 +- .../skills/dsl_skill/handlers/__init__.py | 2 - .../dsl_skill/handlers/regex_handler.py | 3 +- deeppavlov/skills/dsl_skill/utils.py | 1 - deeppavlov/skills/ecommerce_skill/__init__.py | 1 - .../skills/pattern_matching_skill/__init__.py | 1 - deeppavlov/skills/rasa_skill/__init__.py | 1 + deeppavlov/skills/rasa_skill/rasa_skill.py | 39 +- deeppavlov/utils/alexa/__init__.py | 1 + deeppavlov/utils/alexa/bot.py | 208 ------- deeppavlov/utils/alexa/conversation.py | 238 -------- deeppavlov/utils/alexa/server.py | 91 +-- deeppavlov/utils/alice/__init__.py | 2 +- deeppavlov/utils/alice/alice.py | 126 ---- deeppavlov/utils/alice/server.py | 65 +++ deeppavlov/utils/connector/__init__.py | 2 + deeppavlov/utils/connector/bot.py | 550 ++++++++++++++++++ deeppavlov/utils/connector/conversation.py | 465 +++++++++++++++ .../connector}/dialog_logger.py | 17 +- .../utils/{alexa => connector}/ssl_tools.py | 5 +- deeppavlov/utils/ms_bot_framework/__init__.py | 1 + deeppavlov/utils/ms_bot_framework/bot.py | 110 ---- .../utils/ms_bot_framework/conversation.py | 156 ----- deeppavlov/utils/ms_bot_framework/server.py | 102 +--- deeppavlov/utils/server/__init__.py | 1 + deeppavlov/utils/server/server.py | 20 +- .../utils/settings/connector_config.json | 59 ++ .../utils/settings/dialog_logger_config.json | 2 +- deeppavlov/utils/settings/models_info.json | 26 - deeppavlov/utils/settings/server_config.json | 20 +- deeppavlov/utils/socket/__init__.py | 1 + deeppavlov/utils/socket/socket.py | 13 +- deeppavlov/utils/telegram/__init__.py | 1 + deeppavlov/utils/telegram/telegram_ui.py | 98 +--- docs/apiref/agents.rst | 12 - docs/apiref/agents/default_agent.rst | 6 - docs/apiref/agents/filters.rst | 6 - docs/apiref/agents/hello_bot_agent.rst | 6 - docs/apiref/agents/processors.rst | 12 - docs/apiref/agents/rich_content.rst | 6 - docs/apiref/core/agent.rst | 18 - docs/apiref/core/skill.rst | 6 - docs/apiref/skills/aiml_skill.rst | 2 +- docs/apiref/skills/default_skill.rst | 6 - docs/apiref/skills/dsl_skill.rst | 2 +- docs/apiref/skills/ecommerce_skill.rst | 5 - docs/apiref/skills/pattern_matching_skill.rst | 5 - docs/apiref/skills/rasa_skill.rst | 2 +- docs/features/overview.rst | 10 - docs/features/skills/aiml_skill.rst | 12 +- docs/features/skills/dsl_skill.rst | 6 +- docs/features/skills/ecommerce.rst | 182 ------ docs/features/skills/pattern_matching.rst | 7 - docs/features/skills/rasa_skill.rst | 12 +- docs/index.rst | 2 - docs/integrations/amazon_alexa.rst | 9 +- docs/integrations/ms_bot.rst | 9 +- docs/integrations/settings.rst | 5 +- docs/integrations/telegram.rst | 38 +- docs/integrations/yandex_alice.rst | 47 +- docs/intro/overview.rst | 10 +- examples/README.md | 2 - examples/yandex_faq_ru.ipynb | 202 ------- tests/test_aiml_skill.py | 27 +- tests/test_dsl_skill.py | 4 +- tests/test_quick_start.py | 12 +- tests/test_rasa_skill.py | 32 +- 122 files changed, 1518 insertions(+), 1953 deletions(-) delete mode 100644 deeppavlov/agents/hello_bot_agent/__init__.py delete mode 100644 deeppavlov/agents/processors/__init__.py delete mode 100644 deeppavlov/agents/rich_content/__init__.py rename deeppavlov/configs/{dsl_skill => skills}/dsl_skill.json (100%) delete mode 100644 deeppavlov/contrib/__init__.py delete mode 100644 deeppavlov/contrib/skills/__init__.py delete mode 100644 deeppavlov/core/agent/__init__.py delete mode 100644 deeppavlov/core/skill/__init__.py rename deeppavlov/{agents => deprecated}/__init__.py (100%) create mode 100644 deeppavlov/deprecated/agent/__init__.py rename deeppavlov/{core => deprecated}/agent/agent.py (98%) rename deeppavlov/{core => deprecated}/agent/filter.py (100%) rename deeppavlov/{core => deprecated}/agent/processor.py (100%) rename deeppavlov/{core => deprecated}/agent/rich_content.py (100%) rename deeppavlov/{agents/default_agent => deprecated/agents}/__init__.py (100%) create mode 100644 deeppavlov/deprecated/agents/default_agent/__init__.py rename deeppavlov/{ => deprecated}/agents/default_agent/default_agent.py (88%) create mode 100644 deeppavlov/deprecated/agents/ecommerce_agent/__init__.py rename deeppavlov/{ => deprecated}/agents/ecommerce_agent/ecommerce_agent.py (95%) create mode 100644 deeppavlov/deprecated/agents/filters/__init__.py rename deeppavlov/{ => deprecated}/agents/filters/transparent_filter.py (96%) rename deeppavlov/{agents/ecommerce_agent => deprecated/agents/hello_bot_agent}/__init__.py (100%) rename deeppavlov/{ => deprecated}/agents/hello_bot_agent/hello_bot_agent.py (86%) create mode 100644 deeppavlov/deprecated/agents/processors/__init__.py rename deeppavlov/{ => deprecated}/agents/processors/default_rich_content_processor.py (90%) rename deeppavlov/{ => deprecated}/agents/processors/highest_confidence_selector.py (96%) rename deeppavlov/{ => deprecated}/agents/processors/random_selector.py (96%) create mode 100644 deeppavlov/deprecated/agents/rich_content/__init__.py rename deeppavlov/{ => deprecated}/agents/rich_content/default_rich_content.py (99%) create mode 100644 deeppavlov/deprecated/skill/__init__.py rename deeppavlov/{core => deprecated}/skill/skill.py (100%) rename deeppavlov/{agents/filters => deprecated/skills}/__init__.py (100%) create mode 100644 deeppavlov/deprecated/skills/default_skill/__init__.py rename deeppavlov/{ => deprecated}/skills/default_skill/default_skill.py (98%) create mode 100644 deeppavlov/deprecated/skills/ecommerce_skill/__init__.py rename deeppavlov/{ => deprecated}/skills/ecommerce_skill/bleu_retrieve.py (99%) rename deeppavlov/{ => deprecated}/skills/ecommerce_skill/tfidf_retrieve.py (99%) create mode 100644 deeppavlov/deprecated/skills/pattern_matching_skill/__init__.py rename deeppavlov/{ => deprecated}/skills/pattern_matching_skill/pattern_matching_skill.py (84%) rename deeppavlov/{contrib => deprecated}/skills/similarity_matching_skill/__init__.py (100%) rename deeppavlov/{contrib => deprecated}/skills/similarity_matching_skill/similarity_matching_skill.py (86%) delete mode 100644 deeppavlov/skills/default_skill/__init__.py delete mode 100644 deeppavlov/skills/ecommerce_skill/__init__.py delete mode 100644 deeppavlov/skills/pattern_matching_skill/__init__.py delete mode 100644 deeppavlov/utils/alexa/bot.py delete mode 100644 deeppavlov/utils/alexa/conversation.py delete mode 100644 deeppavlov/utils/alice/alice.py create mode 100644 deeppavlov/utils/alice/server.py create mode 100644 deeppavlov/utils/connector/__init__.py create mode 100644 deeppavlov/utils/connector/bot.py create mode 100644 deeppavlov/utils/connector/conversation.py rename deeppavlov/{core/agent => utils/connector}/dialog_logger.py (88%) rename deeppavlov/utils/{alexa => connector}/ssl_tools.py (96%) delete mode 100644 deeppavlov/utils/ms_bot_framework/bot.py delete mode 100644 deeppavlov/utils/ms_bot_framework/conversation.py create mode 100644 deeppavlov/utils/settings/connector_config.json delete mode 100644 deeppavlov/utils/settings/models_info.json delete mode 100644 docs/apiref/agents.rst delete mode 100644 docs/apiref/agents/default_agent.rst delete mode 100644 docs/apiref/agents/filters.rst delete mode 100644 docs/apiref/agents/hello_bot_agent.rst delete mode 100644 docs/apiref/agents/processors.rst delete mode 100644 docs/apiref/agents/rich_content.rst delete mode 100644 docs/apiref/core/agent.rst delete mode 100644 docs/apiref/core/skill.rst delete mode 100644 docs/apiref/skills/default_skill.rst delete mode 100644 docs/apiref/skills/ecommerce_skill.rst delete mode 100644 docs/apiref/skills/pattern_matching_skill.rst delete mode 100644 docs/features/skills/ecommerce.rst delete mode 100644 docs/features/skills/pattern_matching.rst delete mode 100644 examples/yandex_faq_ru.ipynb diff --git a/README.md b/README.md index 8e3aaf1d2a..cf4ca3c8b4 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,18 @@ and others in the Integrations section for more info. ## Breaking Changes +**Breaking changes in version 0.7.0** +- in dialog logger config file [dialog_logger_config.json](deeppavlov/utils/settings/dialog_logger_config.json) `agent_name` parameter was renamed to `logger_name`, + the default value was changed +- Agent, Skill, eCommerce Bot and Pattern Matching classes were moved to [deeppavlov.deprecated](deeppavlov/deprecated/) +- [AIML Skill](http://docs.deeppavlov.ai/en/0.7.0/features/skills/aiml_skill.html), + [RASA Skill](http://docs.deeppavlov.ai/en/0.7.0/features/skills/rasa_skill.html), + [Yandex Alice](http://docs.deeppavlov.ai/en/0.7.0/integrations/yandex_alice.html), + [Amazon Alexa](http://docs.deeppavlov.ai/en/0.7.0/integrations/amazon_alexa.html), + [Microsoft Bot Framework](http://docs.deeppavlov.ai/en/0.7.0/integrations/ms_bot.html) and + [Telegram integration](http://docs.deeppavlov.ai/en/0.7.0/integrations/telegram.html) interfaces were changed +- `/start` and `/help` Telegram messages were moved from `models_info.json` to [connector_config.json](/deeppavlov/utils/settings/connector_config.json) + **Breaking changes in version 0.6.0** - [REST API](http://docs.deeppavlov.ai/en/0.6.0/integrations/rest_api.html): - all models default endpoints were renamed to `/model` diff --git a/deeppavlov/__init__.py b/deeppavlov/__init__.py index 9b773df304..6765198e9f 100644 --- a/deeppavlov/__init__.py +++ b/deeppavlov/__init__.py @@ -37,7 +37,7 @@ def evaluate_model(config: [str, Path, dict], download: bool = False, recursive: except ImportError: 'Assuming that requirements are not yet installed' -__version__ = '0.6.1' +__version__ = '0.7.0' __author__ = 'Neural Networks and Deep Learning lab, MIPT' __description__ = 'An open source library for building end-to-end dialog systems and training chatbots.' __keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot'] diff --git a/deeppavlov/agents/hello_bot_agent/__init__.py b/deeppavlov/agents/hello_bot_agent/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deeppavlov/agents/processors/__init__.py b/deeppavlov/agents/processors/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deeppavlov/agents/rich_content/__init__.py b/deeppavlov/agents/rich_content/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deeppavlov/configs/skills/aiml_skill.json b/deeppavlov/configs/skills/aiml_skill.json index be18512610..a45b80cb1f 100644 --- a/deeppavlov/configs/skills/aiml_skill.json +++ b/deeppavlov/configs/skills/aiml_skill.json @@ -2,7 +2,6 @@ "chainer": { "in": [ "utterances_batch", - "history_batch", "states_batch" ], "out": [ @@ -19,7 +18,6 @@ "null_confidence": 0.33, "in": [ "utterances_batch", - "history_batch", "states_batch" ], "out": [ diff --git a/deeppavlov/configs/dsl_skill/dsl_skill.json b/deeppavlov/configs/skills/dsl_skill.json similarity index 100% rename from deeppavlov/configs/dsl_skill/dsl_skill.json rename to deeppavlov/configs/skills/dsl_skill.json diff --git a/deeppavlov/contrib/__init__.py b/deeppavlov/contrib/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deeppavlov/contrib/skills/__init__.py b/deeppavlov/contrib/skills/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deeppavlov/core/agent/__init__.py b/deeppavlov/core/agent/__init__.py deleted file mode 100644 index dfb5373f19..0000000000 --- a/deeppavlov/core/agent/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .agent import * diff --git a/deeppavlov/core/common/chainer.py b/deeppavlov/core/common/chainer.py index 9f8a2d7f01..26f157d96f 100644 --- a/deeppavlov/core/common/chainer.py +++ b/deeppavlov/core/common/chainer.py @@ -28,7 +28,7 @@ class Chainer(Component): """ - Builds an agent/component pipeline from heterogeneous components (Rule-based/ML/DL). It allows to train + Builds a component pipeline from heterogeneous components (Rule-based/ML/DL). It allows to train and infer models in a pipeline as a whole. Attributes: diff --git a/deeppavlov/core/common/registry.json b/deeppavlov/core/common/registry.json index e630f80b42..49e94c6ed4 100644 --- a/deeppavlov/core/common/registry.json +++ b/deeppavlov/core/common/registry.json @@ -40,8 +40,8 @@ "dstc2_reader": "deeppavlov.dataset_readers.dstc2_reader:DSTC2DatasetReader", "dstc_slotfilling": "deeppavlov.models.slotfill.slotfill:DstcSlotFillingNetwork", "ecommerce_preprocess": "deeppavlov.models.preprocessors.ecommerce_preprocess:EcommercePreprocess", - "ecommerce_skill_bleu": "deeppavlov.skills.ecommerce_skill.bleu_retrieve:EcommerceSkillBleu", - "ecommerce_skill_tfidf": "deeppavlov.skills.ecommerce_skill.tfidf_retrieve:EcommerceSkillTfidf", + "ecommerce_skill_bleu": "deeppavlov.deprecated.skills.ecommerce_skill.bleu_retrieve:EcommerceSkillBleu", + "ecommerce_skill_tfidf": "deeppavlov.deprecated.skills.ecommerce_skill.tfidf_retrieve:EcommerceSkillTfidf", "elmo_embedder": "deeppavlov.models.embedders.elmo_embedder:ELMoEmbedder", "elmo_file_paths_iterator": "deeppavlov.dataset_iterators.elmo_file_paths_iterator:ELMoFilePathsIterator", "elmo_model": "deeppavlov.models.elmo.elmo:ELMo", diff --git a/deeppavlov/core/skill/__init__.py b/deeppavlov/core/skill/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deeppavlov/deep.py b/deeppavlov/deep.py index 3439935343..b0533f5095 100644 --- a/deeppavlov/deep.py +++ b/deeppavlov/deep.py @@ -1,18 +1,16 @@ -""" -Copyright 2017 Neural Networks and Deep Learning lab, MIPT - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import argparse from logging import getLogger @@ -22,13 +20,13 @@ from deeppavlov.core.common.cross_validation import calc_cv_score from deeppavlov.core.common.file import find_config from deeppavlov.download import deep_download -from deeppavlov.utils.alexa.server import run_alexa_default_agent +from deeppavlov.utils.alexa import start_alexa_server from deeppavlov.utils.alice import start_alice_server -from deeppavlov.utils.ms_bot_framework.server import run_ms_bf_default_agent +from deeppavlov.utils.ms_bot_framework import start_ms_bf_server from deeppavlov.utils.pip_wrapper import install_from_config -from deeppavlov.utils.server.server import start_model_server -from deeppavlov.utils.socket.socket import start_socket_server -from deeppavlov.utils.telegram.telegram_ui import interact_model_by_telegram +from deeppavlov.utils.server import start_model_server +from deeppavlov.utils.socket import start_socket_server +from deeppavlov.utils.telegram import interact_model_by_telegram log = getLogger(__name__) @@ -36,7 +34,7 @@ parser.add_argument("mode", help="select a mode, train or interact", type=str, choices={'train', 'evaluate', 'interact', 'predict', 'interactbot', 'interactmsbot', - 'alexa', 'riseapi', 'risesocket', 'download', 'install', 'crossval'}) + 'alexa', 'alice', 'riseapi', 'risesocket', 'download', 'install', 'crossval'}) parser.add_argument("config_path", help="path to a pipeline json config", type=str) parser.add_argument("-e", "--start-epoch-num", dest="start_epoch_num", default=None, @@ -50,79 +48,59 @@ parser.add_argument("--folds", help="number of folds", type=int, default=5) parser.add_argument("-t", "--token", default=None, help="telegram bot token", type=str) + parser.add_argument("-i", "--ms-id", default=None, help="microsoft bot framework app id", type=str) parser.add_argument("-s", "--ms-secret", default=None, help="microsoft bot framework app secret", type=str) -parser.add_argument("--multi-instance", action="store_true", help="allow rising of several instances of the model") -parser.add_argument("--stateful", action="store_true", help="interact with a stateful model") -parser.add_argument("--no-default-skill", action="store_true", help="do not wrap with default skill") - -parser.add_argument("--https", action="store_true", help="run model in https mode") +parser.add_argument("--https", action="store_true", default=None, help="run model in https mode") parser.add_argument("--key", default=None, help="ssl key", type=str) parser.add_argument("--cert", default=None, help="ssl certificate", type=str) parser.add_argument("-p", "--port", default=None, help="api port", type=int) + parser.add_argument("--socket-type", default='TCP', type=str, choices={"TCP", "UNIX"}) parser.add_argument("--socket-file", default="/tmp/deeppavlov_socket.s", type=str) -parser.add_argument("--api-mode", help="rest api mode: 'basic' with batches or 'alice' for Yandex.Dialogs format", - type=str, default='basic', choices={'basic', 'alice'}) - def main(): args = parser.parse_args() - pipeline_config_path = find_config(args.config_path) - https = args.https - ssl_key = args.key - ssl_cert = args.cert if args.download or args.mode == 'download': deep_download(pipeline_config_path) - multi_instance = args.multi_instance - stateful = args.stateful - if args.mode == 'train': - train_evaluate_model_from_config(pipeline_config_path, recursive=args.recursive, + train_evaluate_model_from_config(pipeline_config_path, + recursive=args.recursive, start_epoch_num=args.start_epoch_num) elif args.mode == 'evaluate': train_evaluate_model_from_config(pipeline_config_path, to_train=False, start_epoch_num=args.start_epoch_num) elif args.mode == 'interact': interact_model(pipeline_config_path) elif args.mode == 'interactbot': - token = args.token - interact_model_by_telegram(model_config=pipeline_config_path, - token=token, - default_skill_wrap=not args.no_default_skill) + interact_model_by_telegram(model_config=pipeline_config_path, token=args.token) elif args.mode == 'interactmsbot': - ms_id = args.ms_id - ms_secret = args.ms_secret - run_ms_bf_default_agent(model_config=pipeline_config_path, - app_id=ms_id, - app_secret=ms_secret, - multi_instance=multi_instance, - stateful=stateful, - port=args.port, - https=https, - ssl_key=ssl_key, - ssl_cert=ssl_cert, - default_skill_wrap=not args.no_default_skill) + start_ms_bf_server(model_config=pipeline_config_path, + app_id=args.ms_id, + app_secret=args.ms_secret, + port=args.port, + https=args.https, + ssl_key=args.key, + ssl_cert=args.cert) elif args.mode == 'alexa': - run_alexa_default_agent(model_config=pipeline_config_path, - multi_instance=multi_instance, - stateful=stateful, - port=args.port, - https=https, - ssl_key=ssl_key, - ssl_cert=ssl_cert, - default_skill_wrap=not args.no_default_skill) + start_alexa_server(model_config=pipeline_config_path, + port=args.port, + https=args.https, + ssl_key=args.key, + ssl_cert=args.cert) + elif args.mode == 'alice': + start_alice_server(model_config=pipeline_config_path, + port=args.port, + https=args.https, + ssl_key=args.key, + ssl_cert=args.cert) elif args.mode == 'riseapi': - alice = args.api_mode == 'alice' - if alice: - start_alice_server(pipeline_config_path, https, ssl_key, ssl_cert, port=args.port) - else: - start_model_server(pipeline_config_path, https, ssl_key, ssl_cert, port=args.port) + start_model_server(pipeline_config_path, args.https, args.key, args.cert, port=args.port) elif args.mode == 'risesocket': start_socket_server(pipeline_config_path, args.socket_type, port=args.port, socket_file=args.socket_file) elif args.mode == 'predict': @@ -133,8 +111,7 @@ def main(): if args.folds < 2: log.error('Minimum number of Folds is 2') else: - n_folds = args.folds - calc_cv_score(pipeline_config_path, n_folds=n_folds, is_loo=False) + calc_cv_score(pipeline_config_path, n_folds=args.folds, is_loo=False) if __name__ == "__main__": diff --git a/deeppavlov/agents/__init__.py b/deeppavlov/deprecated/__init__.py similarity index 100% rename from deeppavlov/agents/__init__.py rename to deeppavlov/deprecated/__init__.py diff --git a/deeppavlov/deprecated/agent/__init__.py b/deeppavlov/deprecated/agent/__init__.py new file mode 100644 index 0000000000..ae1ae69d49 --- /dev/null +++ b/deeppavlov/deprecated/agent/__init__.py @@ -0,0 +1,4 @@ +from .agent import Agent, SkillWrapper +from .filter import Filter +from .processor import Processor +from .rich_content import RichControl, RichMessage diff --git a/deeppavlov/core/agent/agent.py b/deeppavlov/deprecated/agent/agent.py similarity index 98% rename from deeppavlov/core/agent/agent.py rename to deeppavlov/deprecated/agent/agent.py index 773775d54b..9bec306f6c 100644 --- a/deeppavlov/core/agent/agent.py +++ b/deeppavlov/deprecated/agent/agent.py @@ -14,10 +14,10 @@ from abc import ABCMeta, abstractmethod from collections import defaultdict -from typing import List, Dict, Tuple, Optional, Union +from typing import List, Dict, Tuple, Optional -from deeppavlov.core.agent.dialog_logger import DialogLogger from deeppavlov.core.models.component import Component +from deeppavlov.utils.connector.dialog_logger import DialogLogger class Agent(Component, metaclass=ABCMeta): diff --git a/deeppavlov/core/agent/filter.py b/deeppavlov/deprecated/agent/filter.py similarity index 100% rename from deeppavlov/core/agent/filter.py rename to deeppavlov/deprecated/agent/filter.py diff --git a/deeppavlov/core/agent/processor.py b/deeppavlov/deprecated/agent/processor.py similarity index 100% rename from deeppavlov/core/agent/processor.py rename to deeppavlov/deprecated/agent/processor.py diff --git a/deeppavlov/core/agent/rich_content.py b/deeppavlov/deprecated/agent/rich_content.py similarity index 100% rename from deeppavlov/core/agent/rich_content.py rename to deeppavlov/deprecated/agent/rich_content.py diff --git a/deeppavlov/agents/default_agent/__init__.py b/deeppavlov/deprecated/agents/__init__.py similarity index 100% rename from deeppavlov/agents/default_agent/__init__.py rename to deeppavlov/deprecated/agents/__init__.py diff --git a/deeppavlov/deprecated/agents/default_agent/__init__.py b/deeppavlov/deprecated/agents/default_agent/__init__.py new file mode 100644 index 0000000000..f5b18b62e8 --- /dev/null +++ b/deeppavlov/deprecated/agents/default_agent/__init__.py @@ -0,0 +1 @@ +from .default_agent import DefaultAgent diff --git a/deeppavlov/agents/default_agent/default_agent.py b/deeppavlov/deprecated/agents/default_agent/default_agent.py similarity index 88% rename from deeppavlov/agents/default_agent/default_agent.py rename to deeppavlov/deprecated/agents/default_agent/default_agent.py index bdd6aaeac6..4737bb80c4 100644 --- a/deeppavlov/agents/default_agent/default_agent.py +++ b/deeppavlov/deprecated/agents/default_agent/default_agent.py @@ -14,11 +14,9 @@ from typing import List, Optional -from deeppavlov.agents.filters.transparent_filter import TransparentFilter -from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector -from deeppavlov.core.agent.agent import Agent -from deeppavlov.core.agent.filter import Filter -from deeppavlov.core.agent.processor import Processor +from deeppavlov.deprecated.agent import Agent, Filter, Processor +from deeppavlov.deprecated.agents.filters import TransparentFilter +from deeppavlov.deprecated.agents.processors import HighestConfidenceSelector from deeppavlov.core.models.component import Component @@ -35,7 +33,8 @@ class DefaultAgent(Agent): a) To define set of skills it uses; b) To implement skills Filter; c) To implement Processor. - You can refer to :class:`deeppavlov.core.skill.Skill`, :class:`deeppavlov.core.agent.Filter`, :class:`deeppavlov.core.agent.Processor` base classes to get more info. + You can refer to :class:`deeppavlov.deprecated.skill.Skill`, :class:`deeppavlov.deprecated.agent.Filter`, + :class:`deeppavlov.deprecated.agent.Processor` base classes to get more info. Args: skills: List of initiated agent skills or components instances. @@ -53,7 +52,7 @@ def __init__(self, skills: List[Component], skills_processor: Optional[Processor self.skills_filter = skills_filter or TransparentFilter(len(skills)) self.skills_processor = skills_processor or HighestConfidenceSelector() - def _call(self, utterances_batch: list, utterances_ids: Optional[list]=None) -> list: + def _call(self, utterances_batch: list, utterances_ids: Optional[list] = None) -> list: """ Processes batch of utterances and returns corresponding responses batch. diff --git a/deeppavlov/deprecated/agents/ecommerce_agent/__init__.py b/deeppavlov/deprecated/agents/ecommerce_agent/__init__.py new file mode 100644 index 0000000000..d83851d715 --- /dev/null +++ b/deeppavlov/deprecated/agents/ecommerce_agent/__init__.py @@ -0,0 +1 @@ +from .ecommerce_agent import EcommerceAgent diff --git a/deeppavlov/agents/ecommerce_agent/ecommerce_agent.py b/deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py similarity index 95% rename from deeppavlov/agents/ecommerce_agent/ecommerce_agent.py rename to deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py index 8f7fe615ea..5313235d5c 100644 --- a/deeppavlov/agents/ecommerce_agent/ecommerce_agent.py +++ b/deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py @@ -17,12 +17,11 @@ from logging import getLogger from typing import List, Dict, Any -from deeppavlov.agents.rich_content.default_rich_content import PlainText, ButtonsFrame, Button -from deeppavlov.core.agent.agent import Agent -from deeppavlov.core.agent.rich_content import RichMessage from deeppavlov.core.commands.infer import build_model -from deeppavlov.core.skill.skill import Skill from deeppavlov.deep import find_config +from deeppavlov.deprecated.agent import Agent, RichMessage +from deeppavlov.deprecated.agents.rich_content import PlainText, ButtonsFrame, Button +from deeppavlov.deprecated.skill import Skill from deeppavlov.utils.ms_bot_framework.server import run_ms_bot_framework_server parser = argparse.ArgumentParser() @@ -192,8 +191,7 @@ def main(): args = parser.parse_args() run_ms_bot_framework_server(agent_generator=make_agent, app_id=args.ms_id, - app_secret=args.ms_secret, - stateful=True) + app_secret=args.ms_secret) if __name__ == '__main__': diff --git a/deeppavlov/deprecated/agents/filters/__init__.py b/deeppavlov/deprecated/agents/filters/__init__.py new file mode 100644 index 0000000000..6535882349 --- /dev/null +++ b/deeppavlov/deprecated/agents/filters/__init__.py @@ -0,0 +1 @@ +from .transparent_filter import TransparentFilter diff --git a/deeppavlov/agents/filters/transparent_filter.py b/deeppavlov/deprecated/agents/filters/transparent_filter.py similarity index 96% rename from deeppavlov/agents/filters/transparent_filter.py rename to deeppavlov/deprecated/agents/filters/transparent_filter.py index 1ae9292422..494ed4b8c8 100644 --- a/deeppavlov/agents/filters/transparent_filter.py +++ b/deeppavlov/deprecated/agents/filters/transparent_filter.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from deeppavlov.core.agent.filter import Filter +from deeppavlov.deprecated.agent import Filter class TransparentFilter(Filter): diff --git a/deeppavlov/agents/ecommerce_agent/__init__.py b/deeppavlov/deprecated/agents/hello_bot_agent/__init__.py similarity index 100% rename from deeppavlov/agents/ecommerce_agent/__init__.py rename to deeppavlov/deprecated/agents/hello_bot_agent/__init__.py diff --git a/deeppavlov/agents/hello_bot_agent/hello_bot_agent.py b/deeppavlov/deprecated/agents/hello_bot_agent/hello_bot_agent.py similarity index 86% rename from deeppavlov/agents/hello_bot_agent/hello_bot_agent.py rename to deeppavlov/deprecated/agents/hello_bot_agent/hello_bot_agent.py index 0cef1e7d05..3d8adc8217 100644 --- a/deeppavlov/agents/hello_bot_agent/hello_bot_agent.py +++ b/deeppavlov/deprecated/agents/hello_bot_agent/hello_bot_agent.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from deeppavlov.agents.default_agent.default_agent import DefaultAgent -from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector -from deeppavlov.skills.pattern_matching_skill import PatternMatchingSkill +from deeppavlov.deprecated.agents.default_agent import DefaultAgent +from deeppavlov.deprecated.agents.processors import HighestConfidenceSelector +from deeppavlov.deprecated.skills.pattern_matching_skill import PatternMatchingSkill def make_hello_bot_agent() -> DefaultAgent: diff --git a/deeppavlov/deprecated/agents/processors/__init__.py b/deeppavlov/deprecated/agents/processors/__init__.py new file mode 100644 index 0000000000..5e7b8aeaca --- /dev/null +++ b/deeppavlov/deprecated/agents/processors/__init__.py @@ -0,0 +1,3 @@ +from .default_rich_content_processor import DefaultRichContentWrapper +from .highest_confidence_selector import HighestConfidenceSelector +from .random_selector import RandomSelector diff --git a/deeppavlov/agents/processors/default_rich_content_processor.py b/deeppavlov/deprecated/agents/processors/default_rich_content_processor.py similarity index 90% rename from deeppavlov/agents/processors/default_rich_content_processor.py rename to deeppavlov/deprecated/agents/processors/default_rich_content_processor.py index 529460286c..354fe16403 100644 --- a/deeppavlov/agents/processors/default_rich_content_processor.py +++ b/deeppavlov/deprecated/agents/processors/default_rich_content_processor.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from deeppavlov.agents.rich_content.default_rich_content import PlainText -from deeppavlov.core.agent.processor import Processor -from deeppavlov.core.agent.rich_content import RichMessage +from deeppavlov.deprecated.agent import Processor, RichMessage +from deeppavlov.deprecated.agents.rich_content import PlainText class DefaultRichContentWrapper(Processor): diff --git a/deeppavlov/agents/processors/highest_confidence_selector.py b/deeppavlov/deprecated/agents/processors/highest_confidence_selector.py similarity index 96% rename from deeppavlov/agents/processors/highest_confidence_selector.py rename to deeppavlov/deprecated/agents/processors/highest_confidence_selector.py index cc9d0be601..8493cc5da9 100644 --- a/deeppavlov/agents/processors/highest_confidence_selector.py +++ b/deeppavlov/deprecated/agents/processors/highest_confidence_selector.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from deeppavlov.core.agent.processor import Processor +from deeppavlov.deprecated.agent import Processor class HighestConfidenceSelector(Processor): diff --git a/deeppavlov/agents/processors/random_selector.py b/deeppavlov/deprecated/agents/processors/random_selector.py similarity index 96% rename from deeppavlov/agents/processors/random_selector.py rename to deeppavlov/deprecated/agents/processors/random_selector.py index 0fa54a3b33..1b70b58218 100644 --- a/deeppavlov/agents/processors/random_selector.py +++ b/deeppavlov/deprecated/agents/processors/random_selector.py @@ -14,7 +14,7 @@ import random -from deeppavlov.core.agent.processor import Processor +from deeppavlov.deprecated.agent import Processor class RandomSelector(Processor): diff --git a/deeppavlov/deprecated/agents/rich_content/__init__.py b/deeppavlov/deprecated/agents/rich_content/__init__.py new file mode 100644 index 0000000000..743e6566d1 --- /dev/null +++ b/deeppavlov/deprecated/agents/rich_content/__init__.py @@ -0,0 +1 @@ +from .default_rich_content import Button, ButtonsFrame, PlainText diff --git a/deeppavlov/agents/rich_content/default_rich_content.py b/deeppavlov/deprecated/agents/rich_content/default_rich_content.py similarity index 99% rename from deeppavlov/agents/rich_content/default_rich_content.py rename to deeppavlov/deprecated/agents/rich_content/default_rich_content.py index f841bd314d..5c55eee6ab 100644 --- a/deeppavlov/agents/rich_content/default_rich_content.py +++ b/deeppavlov/deprecated/agents/rich_content/default_rich_content.py @@ -14,7 +14,7 @@ from typing import Optional -from deeppavlov.core.agent.rich_content import RichControl +from deeppavlov.deprecated.agent import RichControl class PlainText(RichControl): diff --git a/deeppavlov/deprecated/skill/__init__.py b/deeppavlov/deprecated/skill/__init__.py new file mode 100644 index 0000000000..1949352372 --- /dev/null +++ b/deeppavlov/deprecated/skill/__init__.py @@ -0,0 +1 @@ +from .skill import Skill diff --git a/deeppavlov/core/skill/skill.py b/deeppavlov/deprecated/skill/skill.py similarity index 100% rename from deeppavlov/core/skill/skill.py rename to deeppavlov/deprecated/skill/skill.py diff --git a/deeppavlov/agents/filters/__init__.py b/deeppavlov/deprecated/skills/__init__.py similarity index 100% rename from deeppavlov/agents/filters/__init__.py rename to deeppavlov/deprecated/skills/__init__.py diff --git a/deeppavlov/deprecated/skills/default_skill/__init__.py b/deeppavlov/deprecated/skills/default_skill/__init__.py new file mode 100644 index 0000000000..579fd145fc --- /dev/null +++ b/deeppavlov/deprecated/skills/default_skill/__init__.py @@ -0,0 +1 @@ +from .default_skill import DefaultStatelessSkill diff --git a/deeppavlov/skills/default_skill/default_skill.py b/deeppavlov/deprecated/skills/default_skill/default_skill.py similarity index 98% rename from deeppavlov/skills/default_skill/default_skill.py rename to deeppavlov/deprecated/skills/default_skill/default_skill.py index f8fae2fc4d..6894031bc7 100644 --- a/deeppavlov/skills/default_skill/default_skill.py +++ b/deeppavlov/deprecated/skills/default_skill/default_skill.py @@ -15,7 +15,7 @@ from typing import Tuple, Optional, List from deeppavlov.core.common.chainer import Chainer -from deeppavlov.core.skill.skill import Skill +from deeppavlov.deprecated.skill import Skill proposals = { 'en': 'expecting_arg: {}', diff --git a/deeppavlov/deprecated/skills/ecommerce_skill/__init__.py b/deeppavlov/deprecated/skills/ecommerce_skill/__init__.py new file mode 100644 index 0000000000..3dd9ad83f1 --- /dev/null +++ b/deeppavlov/deprecated/skills/ecommerce_skill/__init__.py @@ -0,0 +1,3 @@ +from .bleu_retrieve import EcommerceSkillBleu +from .tfidf_retrieve import EcommerceSkillTfidf + diff --git a/deeppavlov/skills/ecommerce_skill/bleu_retrieve.py b/deeppavlov/deprecated/skills/ecommerce_skill/bleu_retrieve.py similarity index 99% rename from deeppavlov/skills/ecommerce_skill/bleu_retrieve.py rename to deeppavlov/deprecated/skills/ecommerce_skill/bleu_retrieve.py index 5c5cbe3e5b..00a05aaaef 100644 --- a/deeppavlov/skills/ecommerce_skill/bleu_retrieve.py +++ b/deeppavlov/deprecated/skills/ecommerce_skill/bleu_retrieve.py @@ -1,8 +1,11 @@ # Copyright 2018 Neural Networks and Deep Learning lab, MIPT +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at +# # http://www.apache.org/licenses/LICENSE-2.0 +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,7 +27,7 @@ from deeppavlov.core.common.file import save_pickle, load_pickle from deeppavlov.core.common.registry import register from deeppavlov.core.models.estimator import Component -from deeppavlov.core.skill.skill import Skill +from deeppavlov.deprecated.skill import Skill from deeppavlov.metrics.bleu import bleu_advanced log = getLogger(__name__) diff --git a/deeppavlov/skills/ecommerce_skill/tfidf_retrieve.py b/deeppavlov/deprecated/skills/ecommerce_skill/tfidf_retrieve.py similarity index 99% rename from deeppavlov/skills/ecommerce_skill/tfidf_retrieve.py rename to deeppavlov/deprecated/skills/ecommerce_skill/tfidf_retrieve.py index 938c66a8ed..a801f7e7ec 100644 --- a/deeppavlov/skills/ecommerce_skill/tfidf_retrieve.py +++ b/deeppavlov/deprecated/skills/ecommerce_skill/tfidf_retrieve.py @@ -1,8 +1,11 @@ # Copyright 2018 Neural Networks and Deep Learning lab, MIPT +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at +# # http://www.apache.org/licenses/LICENSE-2.0 +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/deeppavlov/deprecated/skills/pattern_matching_skill/__init__.py b/deeppavlov/deprecated/skills/pattern_matching_skill/__init__.py new file mode 100644 index 0000000000..b9d29d5ead --- /dev/null +++ b/deeppavlov/deprecated/skills/pattern_matching_skill/__init__.py @@ -0,0 +1 @@ +from .pattern_matching_skill import PatternMatchingSkill diff --git a/deeppavlov/skills/pattern_matching_skill/pattern_matching_skill.py b/deeppavlov/deprecated/skills/pattern_matching_skill/pattern_matching_skill.py similarity index 84% rename from deeppavlov/skills/pattern_matching_skill/pattern_matching_skill.py rename to deeppavlov/deprecated/skills/pattern_matching_skill/pattern_matching_skill.py index ce701140ce..04c0eabbf2 100644 --- a/deeppavlov/skills/pattern_matching_skill/pattern_matching_skill.py +++ b/deeppavlov/deprecated/skills/pattern_matching_skill/pattern_matching_skill.py @@ -1,8 +1,22 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import random import re from typing import List, Tuple, Optional -from deeppavlov.core.skill.skill import Skill +from deeppavlov.deprecated.skill import Skill class PatternMatchingSkill(Skill): diff --git a/deeppavlov/contrib/skills/similarity_matching_skill/__init__.py b/deeppavlov/deprecated/skills/similarity_matching_skill/__init__.py similarity index 100% rename from deeppavlov/contrib/skills/similarity_matching_skill/__init__.py rename to deeppavlov/deprecated/skills/similarity_matching_skill/__init__.py diff --git a/deeppavlov/contrib/skills/similarity_matching_skill/similarity_matching_skill.py b/deeppavlov/deprecated/skills/similarity_matching_skill/similarity_matching_skill.py similarity index 86% rename from deeppavlov/contrib/skills/similarity_matching_skill/similarity_matching_skill.py rename to deeppavlov/deprecated/skills/similarity_matching_skill/similarity_matching_skill.py index 459efa0d32..eb46832f93 100644 --- a/deeppavlov/contrib/skills/similarity_matching_skill/similarity_matching_skill.py +++ b/deeppavlov/deprecated/skills/similarity_matching_skill/similarity_matching_skill.py @@ -1,14 +1,26 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from logging import getLogger from typing import Tuple, Optional, List -import numpy as np - from deeppavlov import build_model, train_model from deeppavlov.configs import configs from deeppavlov.core.commands.utils import expand_path from deeppavlov.core.common.file import read_json from deeppavlov.core.data.utils import update_dict_recursive -from deeppavlov.core.skill.skill import Skill +from deeppavlov.deprecated.skill import Skill log = getLogger(__name__) diff --git a/deeppavlov/evolve.py b/deeppavlov/evolve.py index 0867c4eb5d..ed796d33a5 100644 --- a/deeppavlov/evolve.py +++ b/deeppavlov/evolve.py @@ -1,18 +1,16 @@ -""" -Copyright 2017 Neural Networks and Deep Learning lab, MIPT - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import argparse import json diff --git a/deeppavlov/models/bert/bert_ner.py b/deeppavlov/models/bert/bert_ner.py index d5c3524161..55171a92aa 100644 --- a/deeppavlov/models/bert/bert_ner.py +++ b/deeppavlov/models/bert/bert_ner.py @@ -11,19 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from logging import getLogger -from typing import List, Any, Tuple, Union, Dict +from typing import List, Union, Dict import numpy as np import tensorflow as tf -from tensorflow.python.ops import array_ops from bert_dp.modeling import BertConfig, BertModel from bert_dp.optimization import AdamWeightDecayOptimizer from deeppavlov.core.commands.utils import expand_path from deeppavlov.core.common.registry import register -from deeppavlov.core.data.utils import zero_pad -from deeppavlov.core.models.component import Component from deeppavlov.core.layers.tf_layers import bi_rnn from deeppavlov.core.models.tf_model import LRScheduledTFModel diff --git a/deeppavlov/models/preprocessors/bert_preprocessor.py b/deeppavlov/models/preprocessors/bert_preprocessor.py index 2889cd8710..86dfdd6df3 100644 --- a/deeppavlov/models/preprocessors/bert_preprocessor.py +++ b/deeppavlov/models/preprocessors/bert_preprocessor.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import random + import re from logging import getLogger from typing import Tuple, List, Optional, Union diff --git a/deeppavlov/skills/aiml_skill/__init__.py b/deeppavlov/skills/aiml_skill/__init__.py index e69de29bb2..e5b4b02f6b 100644 --- a/deeppavlov/skills/aiml_skill/__init__.py +++ b/deeppavlov/skills/aiml_skill/__init__.py @@ -0,0 +1 @@ +from .aiml_skill import AIMLSkill diff --git a/deeppavlov/skills/aiml_skill/aiml_skill.py b/deeppavlov/skills/aiml_skill/aiml_skill.py index 849464b5f8..384730ddff 100644 --- a/deeppavlov/skills/aiml_skill/aiml_skill.py +++ b/deeppavlov/skills/aiml_skill/aiml_skill.py @@ -1,18 +1,32 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import uuid from pathlib import Path -from typing import Tuple, Optional, List, Iterable +from typing import Tuple, Optional, List from logging import getLogger import aiml from deeppavlov.core.common.registry import register -from deeppavlov.core.skill.skill import Skill +from deeppavlov.core.models.component import Component log = getLogger(__name__) @register("aiml_skill") -class AIMLSkill(Skill): +class AIMLSkill(Component): """Skill wraps python-aiml library into DeepPavlov interfrace. AIML uses directory with AIML scripts which are loaded at initialization and used as patterns for answering at each step. @@ -78,18 +92,17 @@ def process_step(self, utterance_str: str, user_id: any) -> Tuple[str, float]: return response, confidence def _generate_user_id(self) -> str: - """ - Here you put user id generative logic if you want to implement it in the skill. + """Here you put user id generative logic if you want to implement it in the skill. - Although it is better to delegate user_id generation to Agent Layer - Returns: str + Returns: + user_id: Random generated user ID. """ return uuid.uuid1().hex - def __call__(self, utterances_batch: List[str], - history_batch: Optional[List]=None, - states_batch: Optional[List]=None) -> Tuple[List[str], List[float], list]: + def __call__(self, + utterances_batch: List[str], + states_batch: Optional[List] = None) -> Tuple[List[str], List[float], list]: """Returns skill inference result. Returns batches of skill inference results, estimated confidence @@ -98,7 +111,6 @@ def __call__(self, utterances_batch: List[str], Args: utterances_batch: A batch of utterances of str type. - history_batch: A batch of list typed histories for each utterance. states_batch: A batch of arbitrary typed states for each utterance. diff --git a/deeppavlov/skills/default_skill/__init__.py b/deeppavlov/skills/default_skill/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deeppavlov/skills/dsl_skill/__init__.py b/deeppavlov/skills/dsl_skill/__init__.py index e69de29bb2..d2b332d4b6 100644 --- a/deeppavlov/skills/dsl_skill/__init__.py +++ b/deeppavlov/skills/dsl_skill/__init__.py @@ -0,0 +1,3 @@ +from .context import UserContext +from .dsl_skill import DSLMeta +from .utils import SkillResponse, UserId diff --git a/deeppavlov/skills/dsl_skill/context.py b/deeppavlov/skills/dsl_skill/context.py index 7f79cc951f..b1dc1b3cf4 100644 --- a/deeppavlov/skills/dsl_skill/context.py +++ b/deeppavlov/skills/dsl_skill/context.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from typing import Optional, Union, Dict import json diff --git a/deeppavlov/skills/dsl_skill/dsl_skill.py b/deeppavlov/skills/dsl_skill/dsl_skill.py index ce2317ecdb..93e9f8544d 100644 --- a/deeppavlov/skills/dsl_skill/dsl_skill.py +++ b/deeppavlov/skills/dsl_skill/dsl_skill.py @@ -20,7 +20,8 @@ from deeppavlov.core.common.registry import register from deeppavlov.skills.dsl_skill.context import UserContext -from deeppavlov.skills.dsl_skill.handlers import Handler, RegexHandler +from deeppavlov.skills.dsl_skill.handlers.handler import Handler +from deeppavlov.skills.dsl_skill.handlers.regex_handler import RegexHandler from deeppavlov.skills.dsl_skill.utils import SkillResponse, UserId diff --git a/deeppavlov/skills/dsl_skill/handlers/__init__.py b/deeppavlov/skills/dsl_skill/handlers/__init__.py index 320eeddce9..e69de29bb2 100644 --- a/deeppavlov/skills/dsl_skill/handlers/__init__.py +++ b/deeppavlov/skills/dsl_skill/handlers/__init__.py @@ -1,2 +0,0 @@ -from .handler import * -from .regex_handler import * diff --git a/deeppavlov/skills/dsl_skill/handlers/regex_handler.py b/deeppavlov/skills/dsl_skill/handlers/regex_handler.py index f658ca427e..04cf171774 100644 --- a/deeppavlov/skills/dsl_skill/handlers/regex_handler.py +++ b/deeppavlov/skills/dsl_skill/handlers/regex_handler.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. - import re from typing import List, Callable, Optional from deeppavlov.skills.dsl_skill.context import UserContext -from .handler import Handler +from deeppavlov.skills.dsl_skill.handlers.handler import Handler class RegexHandler(Handler): diff --git a/deeppavlov/skills/dsl_skill/utils.py b/deeppavlov/skills/dsl_skill/utils.py index b2d1e9a7dc..717d52f637 100644 --- a/deeppavlov/skills/dsl_skill/utils.py +++ b/deeppavlov/skills/dsl_skill/utils.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from typing import Union, NamedTuple UserId = Union[str, int] diff --git a/deeppavlov/skills/ecommerce_skill/__init__.py b/deeppavlov/skills/ecommerce_skill/__init__.py deleted file mode 100644 index 8b13789179..0000000000 --- a/deeppavlov/skills/ecommerce_skill/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/deeppavlov/skills/pattern_matching_skill/__init__.py b/deeppavlov/skills/pattern_matching_skill/__init__.py deleted file mode 100644 index 837e9ce14e..0000000000 --- a/deeppavlov/skills/pattern_matching_skill/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .pattern_matching_skill import * diff --git a/deeppavlov/skills/rasa_skill/__init__.py b/deeppavlov/skills/rasa_skill/__init__.py index e69de29bb2..d694bafa04 100644 --- a/deeppavlov/skills/rasa_skill/__init__.py +++ b/deeppavlov/skills/rasa_skill/__init__.py @@ -0,0 +1 @@ +from .rasa_skill import RASASkill diff --git a/deeppavlov/skills/rasa_skill/rasa_skill.py b/deeppavlov/skills/rasa_skill/rasa_skill.py index d29350c881..e334f4b0ab 100644 --- a/deeppavlov/skills/rasa_skill/rasa_skill.py +++ b/deeppavlov/skills/rasa_skill/rasa_skill.py @@ -1,25 +1,39 @@ -import uuid +# Copyright 2019 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import asyncio import logging +import uuid +from functools import reduce from pathlib import Path from typing import Tuple, Optional, List -from functools import reduce - -from deeppavlov.core.common.registry import register -from deeppavlov.core.skill.skill import Skill +from rasa.cli.utils import get_validated_path +from rasa.constants import DEFAULT_MODELS_PATH from rasa.core.agent import Agent -from rasa.core.channels import UserMessage from rasa.core.channels import CollectingOutputChannel +from rasa.core.channels import UserMessage from rasa.model import get_model -from rasa.cli.utils import get_validated_path -from rasa.constants import DEFAULT_MODELS_PATH + +from deeppavlov.core.common.registry import register +from deeppavlov.core.models.component import Component logger = logging.getLogger(__name__) @register("rasa_skill") -class RASASkill(Skill): +class RASASkill(Component): """RASASkill lets you to wrap RASA Agent as a Skill within DeepPavlov environment. The component requires path to your RASA models (folder with timestamped tar.gz archieves) @@ -51,9 +65,9 @@ def __init__(self, path_to_models: str, **kwargs) -> None: self.ioloop = asyncio.new_event_loop() logger.info(f"path to RASA models is: `{self.path_to_models}`") - def __call__(self, utterances_batch: List[str], - history_batch: Optional[List]=None, - states_batch: Optional[List]=None) -> Tuple[List[str], List[float], list]: + def __call__(self, + utterances_batch: List[str], + states_batch: Optional[List] = None) -> Tuple[List[str], List[float], list]: """Returns skill inference result. Returns batches of skill inference results, estimated confidence @@ -62,7 +76,6 @@ def __call__(self, utterances_batch: List[str], Args: utterances_batch: A batch of utterances of str type. - history_batch: A batch of list typed histories for each utterance. states_batch: A batch of arbitrary typed states for each utterance. diff --git a/deeppavlov/utils/alexa/__init__.py b/deeppavlov/utils/alexa/__init__.py index e69de29bb2..7c5931bb83 100644 --- a/deeppavlov/utils/alexa/__init__.py +++ b/deeppavlov/utils/alexa/__init__.py @@ -0,0 +1 @@ +from .server import start_alexa_server diff --git a/deeppavlov/utils/alexa/bot.py b/deeppavlov/utils/alexa/bot.py deleted file mode 100644 index df883adf0e..0000000000 --- a/deeppavlov/utils/alexa/bot.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright 2017 Neural Networks and Deep Learning lab, MIPT -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import threading -from collections import namedtuple -from datetime import timedelta, datetime -from logging import getLogger -from queue import Empty, Queue -from threading import Timer, Thread -from typing import Optional, Dict - -from OpenSSL.crypto import X509 - -from deeppavlov.agents.default_agent.default_agent import DefaultAgent -from deeppavlov.utils.alexa.conversation import Conversation -from deeppavlov.utils.alexa.ssl_tools import verify_cert, verify_signature - -REQUEST_TIMESTAMP_TOLERANCE_SECS = 150 -REFRESH_VALID_CERTS_PERIOD_SECS = 120 - -log = getLogger(__name__) - -ValidatedCert = namedtuple('ValidatedCert', ['cert', 'expiration_timestamp']) - -# TODO: make common superclass with Bot from ms_bot_framework -class Bot(Thread): - """Contains agent (if not multi-instanced), conversations, validates Alexa requests and routes them to conversations. - - Args: - agent_generator: Callback which generates DefaultAgent instance with alexa skill. - config: Alexa skill configuration settings. - input_queue: Queue for incoming requests from Alexa. - output_queue: Queue for outcoming responses to Alexa. - - Attributes: - config: Alexa skill configuration settings. - conversations: Dict with current conversations, key - Alexa user ID, value - Conversation object. - input_queue: Queue for incoming requests from Alexa. - output_queue: Queue for outcoming responses to Alexa. - valid_certificates: Dict where key - signature chain url, value - ValidatedCert instance. - agent: Alexa skill agent if not multi-instance mode. - agent_generator: Callback which generates DefaultAgent instance with alexa skill. - timer: Timer which triggers periodical certificates with expired validation cleanup. - """ - def __init__(self, agent_generator: callable, config: dict, input_queue: Queue, output_queue: Queue) -> None: - super(Bot, self).__init__() - self.config = config - self.conversations: Dict[str, Conversation] = {} - self.input_queue = input_queue - self.output_queue = output_queue - self._run_flag = True - - self.valid_certificates: Dict[str, ValidatedCert] = {} - - self.agent: Optional[DefaultAgent] = None - self.agent_generator = agent_generator - - if not self.config['multi_instance']: - self.agent = self._init_agent() - log.info('New bot instance level agent initiated') - - self.timer = Timer(REFRESH_VALID_CERTS_PERIOD_SECS, self._refresh_valid_certs) - self.timer.start() - - def run(self) -> None: - """Thread run method implementation.""" - while self._run_flag: - try: - request = self.input_queue.get(timeout=1) - except Empty: - pass - else: - response = self._handle_request(request) - self.output_queue.put(response) - - def join(self, timeout=None): - """Thread join method implementation.""" - self._run_flag = False - for timer in threading.enumerate(): - if isinstance(timer, threading.Timer): - timer.cancel() - Thread.join(self, timeout) - - def _del_conversation(self, conversation_key: str) -> None: - """Deletes Conversation instance. - - Args: - conversation_key: Conversation key. - """ - if conversation_key in self.conversations.keys(): - del self.conversations[conversation_key] - log.info(f'Deleted conversation, key: {conversation_key}') - - def _init_agent(self) -> DefaultAgent: - """Initiates Alexa skill agent from agent generator""" - # TODO: Decide about multi-instance mode necessity. - # If model multi-instancing is still necessary - refactor and remove - agent = self.agent_generator() - return agent - - def _refresh_valid_certs(self) -> None: - """Conducts cleanup of periodical certificates with expired validation.""" - self.timer = Timer(REFRESH_VALID_CERTS_PERIOD_SECS, self._refresh_valid_certs) - self.timer.start() - - expired_certificates = [] - - for valid_cert_url, valid_cert in self.valid_certificates.items(): - valid_cert: ValidatedCert = valid_cert - cert_expiration_time: datetime = valid_cert.expiration_timestamp - if datetime.utcnow() > cert_expiration_time: - expired_certificates.append(valid_cert_url) - - for expired_cert_url in expired_certificates: - del self.valid_certificates[expired_cert_url] - log.info(f'Validation period of {expired_cert_url} certificate expired') - - def _verify_request(self, signature_chain_url: str, signature: str, request_body: bytes) -> bool: - """Conducts series of Alexa request verifications against Amazon Alexa requirements. - - Args: - signature_chain_url: Signature certificate URL from SignatureCertChainUrl HTTP header. - signature: Base64 decoded Alexa request signature from Signature HTTP header. - request_body: full HTTPS request body - Returns: - result: True if verification was successful, False if not. - """ - if signature_chain_url not in self.valid_certificates.keys(): - amazon_cert: X509 = verify_cert(signature_chain_url) - if amazon_cert: - amazon_cert_lifetime: timedelta = self.config['amazon_cert_lifetime'] - expiration_timestamp = datetime.utcnow() + amazon_cert_lifetime - validated_cert = ValidatedCert(cert=amazon_cert, expiration_timestamp=expiration_timestamp) - self.valid_certificates[signature_chain_url] = validated_cert - log.info(f'Certificate {signature_chain_url} validated') - else: - log.error(f'Certificate {signature_chain_url} validation failed') - return False - else: - validated_cert: ValidatedCert = self.valid_certificates[signature_chain_url] - amazon_cert: X509 = validated_cert.cert - - if verify_signature(amazon_cert, signature, request_body): - result = True - else: - log.error(f'Failed signature verification for request: {request_body.decode("utf-8", "replace")}') - result = False - - return result - - def _handle_request(self, request: dict) -> dict: - """Processes Alexa requests from skill server and returns responses to Alexa. - - Args: - request: Dict with Alexa request payload and metadata. - Returns: - result: Alexa formatted or error response. - """ - request_body: bytes = request['request_body'] - signature_chain_url: str = request['signature_chain_url'] - signature: str = request['signature'] - alexa_request: dict = request['alexa_request'] - - if not self._verify_request(signature_chain_url, signature, request_body): - return {'error': 'failed certificate/signature check'} - - timestamp_str = alexa_request['request']['timestamp'] - timestamp_datetime = datetime.strptime(timestamp_str, '%Y-%m-%dT%H:%M:%SZ') - now = datetime.utcnow() - - delta = now - timestamp_datetime if now >= timestamp_datetime else timestamp_datetime - now - - if abs(delta.seconds) > REQUEST_TIMESTAMP_TOLERANCE_SECS: - log.error(f'Failed timestamp check for request: {request_body.decode("utf-8", "replace")}') - return {'error': 'failed request timestamp check'} - - conversation_key = alexa_request['session']['user']['userId'] - - if conversation_key not in self.conversations.keys(): - if self.config['multi_instance']: - conv_agent = self._init_agent() - log.info('New conversation instance level agent initiated') - else: - conv_agent = self.agent - - self.conversations[conversation_key] = \ - Conversation(config=self.config, - agent=conv_agent, - conversation_key=conversation_key, - self_destruct_callback=lambda: self._del_conversation(conversation_key)) - - log.info(f'Created new conversation, key: {conversation_key}') - - conversation = self.conversations[conversation_key] - response = conversation.handle_request(alexa_request) - - return response diff --git a/deeppavlov/utils/alexa/conversation.py b/deeppavlov/utils/alexa/conversation.py deleted file mode 100644 index 61f02a8946..0000000000 --- a/deeppavlov/utils/alexa/conversation.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright 2017 Neural Networks and Deep Learning lab, MIPT -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from copy import deepcopy -from logging import getLogger -from threading import Timer -from typing import Optional - -from deeppavlov.agents.default_agent.default_agent import DefaultAgent -from deeppavlov.core.agent.rich_content import RichMessage - -log = getLogger(__name__) - - -class Conversation: - """Contains agent (if multi-instanced), receives requests, generates responses. - - Args: - config: Alexa skill configuration settings. - agent: DeepPavlov Agent instance. - conversation_key: Alexa conversation ID. - self_destruct_callback: Conversation instance deletion callback function. - - Attributes: - config: Alexa skill configuration settings. - agent: Alexa skill agent. - key: Alexa conversation ID. - stateful: Stateful mode flag. - timer: Conversation self-destruct timer. - handled_requests: Mapping of Alexa requests types to requests handlers. - response_template: Alexa response template. - """ - def __init__(self, config: dict, agent: DefaultAgent, conversation_key: str, - self_destruct_callback: callable) -> None: - self.config = config - self.agent = agent - self.key = conversation_key - self.self_destruct_callback = self_destruct_callback - self.stateful: bool = self.config['stateful'] - self.timer: Optional[Timer] = None - - self.handled_requests = { - 'LaunchRequest': self._handle_launch, - 'IntentRequest': self._handle_intent, - 'SessionEndedRequest': self._handle_end, - '_unsupported': self._handle_unsupported - } - - self.response_template = { - 'version': '1.0', - 'sessionAttributes': { - 'sessionId': None - } - } - - self._start_timer() - - def _start_timer(self) -> None: - """Initiates self-destruct timer.""" - self.timer = Timer(self.config['conversation_lifetime'], self.self_destruct_callback) - self.timer.start() - - def _rearm_self_destruct(self) -> None: - """Rearms self-destruct timer.""" - self.timer.cancel() - self._start_timer() - - def handle_request(self, request: dict) -> dict: - """Routes Alexa requests to appropriate handlers. - - Args: - request: Alexa request. - Returns: - response: Response conforming Alexa response specification. - """ - request_type = request['request']['type'] - request_id = request['request']['requestId'] - log.debug(f'Received request. Type: {request_type}, id: {request_id}') - - if request_type in self.handled_requests.keys(): - response: dict = self.handled_requests[request_type](request) - else: - response: dict = self.handled_requests['_unsupported'](request) - log.warning(f'Unsupported request type: {request_type}, request id: {request_id}') - - self._rearm_self_destruct() - - return response - - def _act(self, utterance: str) -> list: - """Infers DeepPavlov agent with raw user input extracted from Alexa request. - - Args: - utterance: Raw user input extracted from Alexa request. - Returns: - response: DeepPavlov agent response. - """ - if self.stateful: - utterance = [[utterance], [self.key]] - else: - utterance = [[utterance]] - - agent_response: list = self.agent(*utterance) - - return agent_response - - def _generate_response(self, response: dict, request: dict) -> dict: - """Populates generated response with additional data conforming Alexa response specification. - - Args: - response: Raw user input extracted from Alexa request. - request: Alexa request. - Returns: - response: Response conforming Alexa response specification. - """ - response_template = deepcopy(self.response_template) - response_template['sessionAttributes']['sessionId'] = request['session']['sessionId'] - - for key, value in response_template.items(): - if key not in response.keys(): - response[key] = value - - return response - - def _handle_intent(self, request: dict) -> dict: - """Handles IntentRequest Alexa request. - - Args: - request: Alexa request. - Returns: - response: "response" part of response dict conforming Alexa specification. - """ - intent_name = self.config['intent_name'] - slot_name = self.config['slot_name'] - - request_id = request['request']['requestId'] - request_intent: dict = request['request']['intent'] - - if intent_name != request_intent['name']: - log.error(f"Wrong intent name received: {request_intent['name']} in request {request_id}") - return {'error': 'wrong intent name'} - - if slot_name not in request_intent['slots'].keys(): - log.error(f'No slot named {slot_name} found in request {request_id}') - return {'error': 'no slot found'} - - utterance = request_intent['slots'][slot_name]['value'] - agent_response = self._act(utterance) - - if not agent_response: - log.error(f'Some error during response generation for request {request_id}') - return {'error': 'error during response generation'} - - prediction: RichMessage = agent_response[0] - prediction: list = prediction.alexa() - - if not prediction: - log.error(f'Some error during response generation for request {request_id}') - return {'error': 'error during response generation'} - - response = self._generate_response(prediction[0], request) - - return response - - def _handle_launch(self, request: dict) -> dict: - """Handles LaunchRequest Alexa request. - - Args: - request: Alexa request. - Returns: - response: "response" part of response dict conforming Alexa specification. - """ - response = { - 'response': { - 'shouldEndSession': False, - 'outputSpeech': { - 'type': 'PlainText', - 'text': self.config['start_message'] - }, - 'card': { - 'type': 'Simple', - 'content': self.config['start_message'] - } - } - } - - response = self._generate_response(response, request) - - return response - - def _handle_end(self, request: dict) -> dict: - """Handles SessionEndedRequest Alexa request and deletes Conversation instance. - - Args: - request: Alexa request. - Returns: - response: Dummy empty response dict. - """ - response = {} - self.self_destruct_callback() - return response - - def _handle_unsupported(self, request: dict) -> dict: - """Handles all unsupported types of Alexa requests. Returns standard message. - - Args: - request: Alexa request. - Returns: - response: "response" part of response dict conforming Alexa specification. - """ - response = { - 'response': { - 'shouldEndSession': False, - 'outputSpeech': { - 'type': 'PlainText', - 'text': self.config['unsupported_message'] - }, - 'card': { - 'type': 'Simple', - 'content': self.config['unsupported_message'] - } - } - } - - response = self._generate_response(response, request) - - return response diff --git a/deeppavlov/utils/alexa/server.py b/deeppavlov/utils/alexa/server.py index d569621664..70a60851a3 100644 --- a/deeppavlov/utils/alexa/server.py +++ b/deeppavlov/utils/alexa/server.py @@ -13,115 +13,57 @@ # limitations under the License. import asyncio -from datetime import timedelta +import json from logging import getLogger from pathlib import Path from queue import Queue from typing import Union, Optional import uvicorn -import json from fastapi import FastAPI from starlette.responses import JSONResponse -from deeppavlov.agents.default_agent.default_agent import DefaultAgent -from deeppavlov.agents.processors.default_rich_content_processor import DefaultRichContentWrapper -from deeppavlov.core.commands.infer import build_model -from deeppavlov.core.common.file import read_json -from deeppavlov.core.common.paths import get_settings_path -from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill -from deeppavlov.utils.alexa.bot import Bot from deeppavlov.utils.alexa.request_parameters import data_body, cert_chain_url_header, signature_header -from deeppavlov.utils.server.server import get_ssl_params, redirect_root_do_docs - -SERVER_CONFIG_FILENAME = 'server_config.json' - -AMAZON_CERTIFICATE_LIFETIME = timedelta(hours=1) +from deeppavlov.utils.connector import AlexaBot +from deeppavlov.utils.server import get_ssl_params, redirect_root_to_docs, get_server_params log = getLogger(__name__) uvicorn_log = getLogger('uvicorn') app = FastAPI() -def run_alexa_default_agent(model_config: Union[str, Path, dict], - multi_instance: bool = False, - stateful: bool = False, - port: Optional[int] = None, - https: bool = False, - ssl_key: Optional[str] = None, - ssl_cert: Optional[str] = None, - default_skill_wrap: bool = True) -> None: - """Creates Alexa agents factory and initiates Alexa web service. +def start_alexa_server(model_config: Union[str, Path, dict], + port: Optional[int] = None, + https: bool = False, + ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None) -> None: + """Initiates FastAPI web service with Alexa skill. - Wrapper around run_alexa_server. Allows raise Alexa web service with - DeepPavlov config in backend. + Allows raise Alexa web service with DeepPavlov config in backend. Args: model_config: DeepPavlov config path. - multi_instance: Multi instance mode flag. - stateful: Stateful mode flag. - port: FastAPI web service port. - https: Flag for running Alexa skill service in https mode. - ssl_key: SSL key file path. - ssl_cert: SSL certificate file path. - default_skill_wrap: Wrap with default skill flag. - - """ - def get_default_agent() -> DefaultAgent: - model = build_model(model_config) - skill = DefaultStatelessSkill(model) if default_skill_wrap else model - agent = DefaultAgent([skill], skills_processor=DefaultRichContentWrapper()) - return agent - - run_alexa_server(agent_generator=get_default_agent, - multi_instance=multi_instance, - stateful=stateful, - port=port, - https=https, - ssl_key=ssl_key, - ssl_cert=ssl_cert) - - -def run_alexa_server(agent_generator: callable, - multi_instance: bool = False, - stateful: bool = False, - port: Optional[int] = None, - https: bool = False, - ssl_key: Optional[str] = None, - ssl_cert: Optional[str] = None) -> None: - """Initiates FastAPI web service with Alexa skill. - - Args: - agent_generator: Callback Alexa agents factory. - multi_instance: Multi instance mode flag. - stateful: Stateful mode flag. port: FastAPI web service port. https: Flag for running Alexa skill service in https mode. ssl_key: SSL key file path. ssl_cert: SSL certificate file path. """ - server_config_path = Path(get_settings_path(), SERVER_CONFIG_FILENAME).resolve() - server_params = read_json(server_config_path) + server_params = get_server_params(model_config) - host = server_params['common_defaults']['host'] - port = port or server_params['common_defaults']['port'] - alexa_server_params = server_params['alexa_defaults'] + host = server_params['host'] + port = port or server_params['port'] - alexa_server_params['multi_instance'] = multi_instance or server_params['common_defaults']['multi_instance'] - alexa_server_params['stateful'] = stateful or server_params['common_defaults']['stateful'] - alexa_server_params['amazon_cert_lifetime'] = AMAZON_CERTIFICATE_LIFETIME - - ssl_config = get_ssl_params(server_params['common_defaults'], https, ssl_key=ssl_key, ssl_cert=ssl_cert) + ssl_config = get_ssl_params(server_params, https, ssl_key=ssl_key, ssl_cert=ssl_cert) input_q = Queue() output_q = Queue() - bot = Bot(agent_generator, alexa_server_params, input_q, output_q) + bot = AlexaBot(model_config, input_q, output_q) bot.start() endpoint = '/interact' - redirect_root_do_docs(app, 'interact', endpoint, 'post') + redirect_root_to_docs(app, 'interact', endpoint, 'post') @app.post(endpoint, summary='Amazon Alexa custom service endpoint', response_description='A model response') async def interact(data: dict = data_body, @@ -139,7 +81,6 @@ async def interact(data: dict = data_body, loop = asyncio.get_event_loop() response: dict = await loop.run_in_executor(None, bot.output_queue.get) response_code = 400 if 'error' in response.keys() else 200 - return JSONResponse(response, status_code=response_code) uvicorn.run(app, host=host, port=port, logger=uvicorn_log, ssl_version=ssl_config.version, diff --git a/deeppavlov/utils/alice/__init__.py b/deeppavlov/utils/alice/__init__.py index b1b1d00647..02434309b8 100644 --- a/deeppavlov/utils/alice/__init__.py +++ b/deeppavlov/utils/alice/__init__.py @@ -1 +1 @@ -from .alice import start_agent_server, start_alice_server +from .server import start_alice_server diff --git a/deeppavlov/utils/alice/alice.py b/deeppavlov/utils/alice/alice.py deleted file mode 100644 index fb2062a1b6..0000000000 --- a/deeppavlov/utils/alice/alice.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2017 Neural Networks and Deep Learning lab, MIPT -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -from collections import namedtuple -from logging import getLogger -from pathlib import Path -from typing import Dict, Union, Optional - -import uvicorn -from fastapi import FastAPI - -from deeppavlov import build_model -from deeppavlov.agents.default_agent.default_agent import DefaultAgent -from deeppavlov.agents.processors.default_rich_content_processor import DefaultRichContentWrapper -from deeppavlov.core.agent import Agent -from deeppavlov.core.agent.rich_content import RichMessage -from deeppavlov.core.common.paths import get_settings_path -from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill -from deeppavlov.utils.server.server import SSLConfig, get_server_params, get_ssl_params, redirect_root_do_docs -from deeppavlov.utils.alice.request_parameters import data_body - -SERVER_CONFIG_FILENAME = 'server_config.json' - -log = getLogger(__name__) -uvicorn_log = getLogger('uvicorn') -app = FastAPI() - -DialogID = namedtuple('DialogID', ['user_id', 'session_id']) - - -def interact_alice(agent: Agent, data: Dict) -> Dict: - """ - Exchange messages between basic pipelines and the Yandex.Dialogs service. - If the pipeline returns multiple values, only the first one is forwarded to Yandex. - """ - text = data['request'].get('command', '').strip() - payload = data['request'].get('payload') - - session_id = data['session']['session_id'] - user_id = data['session']['user_id'] - message_id = data['session']['message_id'] - - dialog_id = DialogID(user_id, session_id) - - agent_response: Union[str, RichMessage] = agent([payload or text], [dialog_id])[0] - if isinstance(agent_response, RichMessage): - response_text = '\n'.join([j['content'] for j in agent_response.json() if j['type'] == 'plain_text']) - else: - response_text = str(agent_response) - - response = { - 'response': { - 'end_session': False, - 'text': response_text - }, - 'session': { - 'session_id': session_id, - 'message_id': message_id, - 'user_id': user_id - }, - 'version': '1.0' - } - - return response - - -def start_alice_server(model_config: Path, - https: bool = False, - ssl_key: Optional[str] = None, - ssl_cert: Optional[str] = None, - port: Optional[int] = None) -> None: - server_config_path = get_settings_path() / SERVER_CONFIG_FILENAME - server_params = get_server_params(server_config_path, model_config) - - https = https or server_params['https'] - host = server_params['host'] - port = port or server_params['port'] - model_endpoint = server_params['model_endpoint'] - - ssl_config = get_ssl_params(server_params, https, ssl_key=ssl_key, ssl_cert=ssl_cert) - - model = build_model(model_config) - skill = DefaultStatelessSkill(model, lang='ru') - agent = DefaultAgent([skill], skills_processor=DefaultRichContentWrapper()) - - start_agent_server(agent, host, port, model_endpoint, ssl_config=ssl_config) - - -def start_agent_server(agent: Agent, - host: str, - port: int, - endpoint: str, - ssl_key: Optional[str] = None, - ssl_cert: Optional[str] = None, - ssl_config: Optional[SSLConfig] = None) -> None: - - if ssl_key and ssl_cert and ssl_config: - raise ValueError('ssl_key, ssl_cert, ssl_config was assigned at the same time. Please, use either' - 'ssl_config or ssl_key and ssl_cert') - - if ssl_key and ssl_cert: - ssl_config = get_ssl_params({}, True, ssl_key=ssl_key, ssl_cert=ssl_cert) - else: - ssl_config = ssl_config or get_ssl_params({}, False, ssl_key=None, ssl_cert=None) - - redirect_root_do_docs(app, 'answer', endpoint, 'post') - - @app.post(endpoint, summary='A model endpoint', response_description='A model response') - async def answer(data: dict = data_body) -> dict: - loop = asyncio.get_event_loop() - return await loop.run_in_executor(None, interact_alice, agent, data) - - uvicorn.run(app, host=host, port=port, logger=uvicorn_log, ssl_version=ssl_config.version, - ssl_keyfile=ssl_config.keyfile, ssl_certfile=ssl_config.certfile) diff --git a/deeppavlov/utils/alice/server.py b/deeppavlov/utils/alice/server.py new file mode 100644 index 0000000000..7c5a49f138 --- /dev/null +++ b/deeppavlov/utils/alice/server.py @@ -0,0 +1,65 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from logging import getLogger +from pathlib import Path +from queue import Queue +from typing import Optional, Union + +import uvicorn +from fastapi import FastAPI + +from deeppavlov.utils.alice.request_parameters import data_body +from deeppavlov.utils.connector import AliceBot +from deeppavlov.utils.server import get_server_params, get_ssl_params, redirect_root_to_docs + +log = getLogger(__name__) +uvicorn_log = getLogger('uvicorn') +app = FastAPI() + + +def start_alice_server(model_config: Union[str, Path], + host: Optional[str] = None, + port: Optional[int] = None, + endpoint: Optional[str] = None, + https: Optional[bool] = None, + ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None) -> None: + server_params = get_server_params(model_config) + + host = host or server_params['host'] + port = port or server_params['port'] + endpoint = endpoint or server_params['model_endpoint'] + + ssl_config = get_ssl_params(server_params, https, ssl_key=ssl_key, ssl_cert=ssl_cert) + + input_q = Queue() + output_q = Queue() + + bot = AliceBot(model_config, input_q, output_q) + bot.start() + + redirect_root_to_docs(app, 'answer', endpoint, 'post') + + @app.post(endpoint, summary='A model endpoint', response_description='A model response') + async def answer(data: dict = data_body) -> dict: + loop = asyncio.get_event_loop() + bot.input_queue.put(data) + response: dict = await loop.run_in_executor(None, bot.output_queue.get) + return response + + uvicorn.run(app, host=host, port=port, logger=uvicorn_log, ssl_version=ssl_config.version, + ssl_keyfile=ssl_config.keyfile, ssl_certfile=ssl_config.certfile) + bot.join() diff --git a/deeppavlov/utils/connector/__init__.py b/deeppavlov/utils/connector/__init__.py new file mode 100644 index 0000000000..6adbf146d9 --- /dev/null +++ b/deeppavlov/utils/connector/__init__.py @@ -0,0 +1,2 @@ +from .bot import AlexaBot, AliceBot, MSBot, TelegramBot +from .dialog_logger import DialogLogger diff --git a/deeppavlov/utils/connector/bot.py b/deeppavlov/utils/connector/bot.py new file mode 100644 index 0000000000..eec9ca8429 --- /dev/null +++ b/deeppavlov/utils/connector/bot.py @@ -0,0 +1,550 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading +from collections import namedtuple +from datetime import datetime, timedelta +from logging import getLogger +from pathlib import Path +from queue import Empty, Queue +from threading import Thread, Timer +from typing import Dict, Optional, Union + +import requests +import telebot +from OpenSSL.crypto import X509 +from requests.exceptions import HTTPError + +from deeppavlov.core.commands.infer import build_model +from deeppavlov.core.common.chainer import Chainer +from deeppavlov.core.common.file import read_json +from deeppavlov.core.common.paths import get_settings_path +from deeppavlov.utils.connector.conversation import AlexaConversation, AliceConversation, BaseConversation +from deeppavlov.utils.connector.conversation import MSConversation, TelegramConversation +from deeppavlov.utils.connector.ssl_tools import verify_cert, verify_signature + +CONNECTOR_CONFIG_FILENAME = 'connector_config.json' +INPUT_QUEUE_TIMEOUT = 1 + +log = getLogger(__name__) + +ValidatedCert = namedtuple('ValidatedCert', ['cert', 'expiration_timestamp']) + + +class BaseBot(Thread): + """Routes requests to conversations, sends responses to channel. + + Attributes: + input_queue: Queue for incoming requests from the channel. + + """ + input_queue: Queue + _run_flag: bool + _model: Chainer + _conversations: Dict[str, BaseConversation] + + def __init__(self, + model_config: Union[str, Path, dict], + input_queue: Queue) -> None: + """Builds DeepPavlov model, initiates class attributes. + + Args: + model_config: Path to DeepPavlov model config file. + input_queue: Queue for incoming requests from channel. + + """ + super(BaseBot, self).__init__() + self.input_queue = input_queue + self._run_flag = True + self._model = build_model(model_config) + self._conversations = dict() + log.info('Bot initiated') + + def run(self) -> None: + """Thread run method implementation. Routes requests from ``input_queue`` to request handler.""" + while self._run_flag: + try: + request: dict = self.input_queue.get(timeout=INPUT_QUEUE_TIMEOUT) + except Empty: + pass + else: + response = self._handle_request(request) + self._send_response(response) + + def join(self, timeout: Optional[float] = None) -> None: + """Thread join method implementation. Stops reading requests from ``input_queue``, cancels all timers. + + Args: + timeout: Timeout for join operation in seconds. If the timeout argument is not present or None, + the operation will block until the thread terminates. + + """ + self._run_flag = False + for timer in threading.enumerate(): + if isinstance(timer, Timer): + timer.cancel() + Thread.join(self, timeout) + + def _del_conversation(self, conversation_key: Union[int, str]) -> None: + """Deletes Conversation instance. + + Args: + conversation_key: Conversation key. + + """ + if conversation_key in self._conversations.keys(): + del self._conversations[conversation_key] + log.info(f'Deleted conversation, key: {conversation_key}') + + def _handle_request(self, request: dict) -> Optional[dict]: + """Routes the request to the appropriate conversation. + + Args: + request: Request from the channel. + + Returns: + response: Corresponding response to the channel request if replies are sent via bot, None otherwise. + + """ + raise NotImplementedError + + def _send_response(self, response: Optional[dict]) -> None: + """Sends response to the request back to the channel. + + Args: + response: Corresponding response to the channel request if replies are sent via bot, None otherwise. + + """ + raise NotImplementedError + + def _get_connector_params(self) -> dict: + """Reads bot and conversation default params from connector config file. + + Returns: + connector_defaults: Dictionary containing bot defaults and conversation defaults dicts. + + """ + connector_config_path = get_settings_path() / CONNECTOR_CONFIG_FILENAME + connector_config: dict = read_json(connector_config_path) + + bot_name = type(self).__name__ + bot_defaults = connector_config['bot_defaults'].get(bot_name, {}) + + model_name = type(self._model.get_main_component()).__name__ + models_info = connector_config['models_info'] + model_msgs = models_info.get(model_name, models_info['@default']) + conversation_defaults = connector_config['conversation_defaults'] + conversation_defaults['start_message'] = model_msgs['start_message'] + conversation_defaults['help_message'] = model_msgs['help_message'] + + connector_defaults = {'bot_defaults': bot_defaults, + 'conversation_defaults': conversation_defaults} + + return connector_defaults + + +class AlexaBot(BaseBot): + """Validates Alexa requests and routes them to conversations, sends responses to Alexa. + + Attributes: + input_queue: Queue for incoming requests from Alexa. + output_queue: Queue for outgoing responses to Alexa. + + """ + output_queue: Queue + _conversation_config: dict + _amazon_cert_lifetime: timedelta + _request_timestamp_tolerance_secs: int + _refresh_valid_certs_period_secs: int + _valid_certificates: Dict[str, ValidatedCert] + _timer: Timer + + def __init__(self, + model_config: Union[str, Path, dict], + input_queue: Queue, + output_queue: Queue) -> None: + """Initiates class attributes. + + Args: + model_config: Path to DeepPavlov model config file. + input_queue: Queue for incoming requests from Alexa. + output_queue: Queue for outgoing responses to Alexa. + + """ + super(AlexaBot, self).__init__(model_config, input_queue) + self.output_queue = output_queue + + connector_config: dict = self._get_connector_params() + self._conversation_config: dict = connector_config['conversation_defaults'] + bot_config: dict = connector_config['bot_defaults'] + + self._conversation_config['intent_name'] = bot_config['intent_name'] + self._conversation_config['slot_name'] = bot_config['slot_name'] + + self._amazon_cert_lifetime = timedelta(seconds=bot_config['amazon_cert_lifetime_secs']) + self._request_timestamp_tolerance_secs = bot_config['request_timestamp_tolerance_secs'] + self._refresh_valid_certs_period_secs = bot_config['refresh_valid_certs_period_secs'] + self._valid_certificates = {} + self._refresh_valid_certs() + + def _refresh_valid_certs(self) -> None: + """Provides cleanup of periodical certificates with expired validation.""" + self._timer = Timer(self._refresh_valid_certs_period_secs, self._refresh_valid_certs) + self._timer.start() + + expired_certificates = [] + + for valid_cert_url, valid_cert in self._valid_certificates.items(): + valid_cert: ValidatedCert = valid_cert + cert_expiration_time: datetime = valid_cert.expiration_timestamp + if datetime.utcnow() > cert_expiration_time: + expired_certificates.append(valid_cert_url) + + for expired_cert_url in expired_certificates: + del self._valid_certificates[expired_cert_url] + log.info(f'Validation period of {expired_cert_url} certificate expired') + + def _verify_request(self, signature_chain_url: str, signature: str, request_body: bytes) -> bool: + """Provides series of Alexa request verifications against Amazon Alexa requirements. + + Args: + signature_chain_url: Signature certificate URL from SignatureCertChainUrl HTTP header. + signature: Base64 decoded Alexa request signature from Signature HTTP header. + request_body: full HTTPS request body + + Returns: + result: True if verification was successful, False otherwise. + + """ + if signature_chain_url not in self._valid_certificates.keys(): + amazon_cert: X509 = verify_cert(signature_chain_url) + if amazon_cert: + expiration_timestamp = datetime.utcnow() + self._amazon_cert_lifetime + validated_cert = ValidatedCert(cert=amazon_cert, expiration_timestamp=expiration_timestamp) + self._valid_certificates[signature_chain_url] = validated_cert + log.info(f'Certificate {signature_chain_url} validated') + else: + log.error(f'Certificate {signature_chain_url} validation failed') + return False + else: + validated_cert: ValidatedCert = self._valid_certificates[signature_chain_url] + amazon_cert: X509 = validated_cert.cert + + if verify_signature(amazon_cert, signature, request_body): + result = True + else: + log.error(f'Failed signature verification for request: {request_body.decode("utf-8", "replace")}') + result = False + + return result + + def _handle_request(self, request: dict) -> dict: + """Processes Alexa request and returns response. + + Args: + request: Dict with Alexa request payload and metadata. + + Returns: + result: Alexa formatted or error response. + + """ + request_body: bytes = request['request_body'] + signature_chain_url: str = request['signature_chain_url'] + signature: str = request['signature'] + alexa_request: dict = request['alexa_request'] + + if not self._verify_request(signature_chain_url, signature, request_body): + return {'error': 'failed certificate/signature check'} + + timestamp_str = alexa_request['request']['timestamp'] + timestamp_datetime = datetime.strptime(timestamp_str, '%Y-%m-%dT%H:%M:%SZ') + now = datetime.utcnow() + + delta = now - timestamp_datetime if now >= timestamp_datetime else timestamp_datetime - now + + if abs(delta.seconds) > self._request_timestamp_tolerance_secs: + log.error(f'Failed timestamp check for request: {request_body.decode("utf-8", "replace")}') + return {'error': 'failed request timestamp check'} + + conversation_key = alexa_request['session']['sessionId'] + + if conversation_key not in self._conversations: + self._conversations[conversation_key] = \ + AlexaConversation(config=self._conversation_config, + model=self._model, + self_destruct_callback=self._del_conversation, + conversation_id=conversation_key) + + log.info(f'Created new conversation, key: {conversation_key}') + + conversation = self._conversations[conversation_key] + response = conversation.handle_request(alexa_request) + + return response + + def _send_response(self, response: dict) -> None: + """Sends response to Alexa. + + Args: + response: Alexa formatted or error response. + + """ + self.output_queue.put(response) + + +class AliceBot(BaseBot): + """Processes Alice requests and routes them to conversations, returns responses to Alice. + + Attributes: + input_queue: Queue for incoming requests from Alice. + output_queue: Queue for outgoing responses to Alice. + + """ + output_queue: Queue + _conversation_config: dict + + def __init__(self, + model_config: Union[str, Path, dict], + input_queue: Queue, + output_queue: Queue) -> None: + """Initiates class attributes. + + Args: + model_config: Path to DeepPavlov model config file. + input_queue: Queue for incoming requests from Alice. + output_queue: Queue for outgoing responses to Alice. + + """ + super(AliceBot, self).__init__(model_config, input_queue) + self.output_queue = output_queue + connector_config: dict = self._get_connector_params() + self._conversation_config = connector_config['conversation_defaults'] + + def _handle_request(self, request: dict) -> dict: + """Processes Alice request and returns response. + + Args: + request: Dict with Alice request payload and metadata. + + Returns: + result: Alice formatted response. + + """ + conversation_key = request['session']['session_id'] + + if conversation_key not in self._conversations: + self._conversations[conversation_key] = \ + AliceConversation(config=self._conversation_config, + model=self._model, + self_destruct_callback=self._del_conversation, + conversation_id=conversation_key) + log.info(f'Created new conversation, key: {conversation_key}') + conversation = self._conversations[conversation_key] + response = conversation.handle_request(request) + + return response + + def _send_response(self, response: dict) -> None: + """Sends response to Alice. + + Args: + response: Alice formatted response. + + """ + self.output_queue.put(response) + + +class MSBot(BaseBot): + """Routes Microsoft Bot Framework requests to conversations, sends responses to Bot Framework. + + Attributes: + input_queue: Queue for incoming requests from Microsoft Bot Framework. + + """ + _conversation_config: dict + _auth_polling_interval: int + _auth_url: str + _auth_headers: dict + _auth_payload: dict + _http_session: requests.Session + + def __init__(self, + model_config: Union[str, Path, dict], + input_queue: Queue, + client_id: Optional[str], + client_secret: Optional[str]) -> None: + """Initiates class attributes. + + Args: + model_config: Path to DeepPavlov model config file. + input_queue: Queue for incoming requests from Microsoft Bot Framework. + client_id: Microsoft App ID. + client_secret: Microsoft App Secret. + + Raises: + ValueError: If ``client_id`` or ``client_secret`` were not set neither in the configuration file nor + in method arguments. + + """ + super(MSBot, self).__init__(model_config, input_queue) + connector_config: dict = self._get_connector_params() + bot_config: dict = connector_config['bot_defaults'] + bot_config['auth_payload']['client_id'] = client_id or bot_config['auth_payload']['client_id'] + bot_config['auth_payload']['client_secret'] = client_secret or bot_config['auth_payload']['client_secret'] + + if not bot_config['auth_payload']['client_id']: + e = ValueError('Microsoft Bot Framework app id required: initiate -i param ' + 'or auth_payload.client_id param in server configuration file') + log.error(e) + raise e + + if not bot_config['auth_payload']['client_secret']: + e = ValueError('Microsoft Bot Framework app secret required: initiate -s param ' + 'or auth_payload.client_secret param in server configuration file') + log.error(e) + raise e + + self._conversation_config = connector_config['conversation_defaults'] + self._auth_polling_interval = bot_config['auth_polling_interval'] + self._auth_url = bot_config['auth_url'] + self._auth_headers = bot_config['auth_headers'] + self._auth_payload = bot_config['auth_payload'] + self._http_session = requests.Session() + self._update_access_info() + + def _update_access_info(self) -> None: + """Updates headers for http_session used to send responses to Bot Framework. + + Raises: + HTTPError: If authentication token request returned other than 200 status code. + + """ + self._timer = threading.Timer(self._auth_polling_interval, self._update_access_info) + self._timer.start() + + result = requests.post(url=self._auth_url, + headers=self._auth_headers, + data=self._auth_payload) + + status_code = result.status_code + if status_code != 200: + raise HTTPError(f'Authentication token request returned wrong HTTP status code: {status_code}') + + access_info = result.json() + headers = { + 'Authorization': f"{access_info['token_type']} {access_info['access_token']}", + 'Content-Type': 'application/json' + } + + self._http_session.headers.update(headers) + + log.info(f'Obtained authentication information from Microsoft Bot Framework: {str(access_info)}') + + def _handle_request(self, request: dict) -> None: + """Routes MS Bot Framework request to conversation. + + Args: + request: Dict with MS Bot Framework request payload and metadata. + + """ + conversation_key = request['conversation']['id'] + + if conversation_key not in self._conversations: + self._conversations[conversation_key] = \ + MSConversation(config=self._conversation_config, + model=self._model, + self_destruct_callback=self._del_conversation, + conversation_id=conversation_key, + http_session=self._http_session) + + log.info(f'Created new conversation, key: {conversation_key}') + + conversation = self._conversations[conversation_key] + conversation.handle_request(request) + + def _send_response(self, response: dict) -> None: + """Dummy method to match ``run`` method body.""" + pass + + +class TelegramBot(BaseBot): + """Routes messages from Telegram to conversations, sends responses back.""" + _conversation_config: dict + _token: str + + def __init__(self, model_config: Union[str, Path, dict], token: Optional[str]) -> None: + """Initiates and validates class attributes. + + Args: + model_config: Path to DeepPavlov model config file. + token: Telegram bot token. + + Raises: + ValueError: If telegram token was not set neither in config file nor in method arguments. + + """ + super(TelegramBot, self).__init__(model_config, Queue()) + connector_config: dict = self._get_connector_params() + bot_config: dict = connector_config['bot_defaults'] + self._conversation_config = connector_config['conversation_defaults'] + self._token = token or bot_config['token'] + + if not self._token: + e = ValueError('Telegram token required: initiate -t param or telegram_defaults/token ' + 'in server configuration file') + log.error(e) + raise e + + def start(self) -> None: + """Starts polling messages from Telegram, routes messages to handlers.""" + bot = telebot.TeleBot(self._token) + bot.remove_webhook() + + @bot.message_handler(commands=['start']) + def send_start_message(message: telebot.types.Message) -> None: + chat_id = message.chat.id + out_message = self._conversation_config['start_message'] + bot.send_message(chat_id, out_message) + + @bot.message_handler(commands=['help']) + def send_help_message(message: telebot.types.Message) -> None: + chat_id = message.chat.id + out_message = self._conversation_config['help_message'] + bot.send_message(chat_id, out_message) + + @bot.message_handler() + def handle_inference(message: telebot.types.Message) -> None: + chat_id = message.chat.id + context = message.text + + if chat_id not in self._conversations: + self._conversations[chat_id] = \ + TelegramConversation(config=self._conversation_config, + model=self._model, + self_destruct_callback=self._del_conversation, + conversation_id=chat_id) + + conversation = self._conversations[chat_id] + response = conversation.handle_request(context) + bot.send_message(chat_id, response) + + bot.polling() + + def _handle_request(self, request: dict) -> None: + """Dummy method to match ``run`` method body.""" + pass + + def _send_response(self, response: Optional[dict]) -> None: + """Dummy method to match ``run`` method body.""" + pass diff --git a/deeppavlov/utils/connector/conversation.py b/deeppavlov/utils/connector/conversation.py new file mode 100644 index 0000000000..09129ed70c --- /dev/null +++ b/deeppavlov/utils/connector/conversation.py @@ -0,0 +1,465 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from logging import getLogger +from threading import Timer +from typing import Dict, Optional, Union +from urllib.parse import urljoin + +from requests import Session + +from deeppavlov.core.common.chainer import Chainer +from deeppavlov.utils.connector.dialog_logger import DialogLogger + +log = getLogger(__name__) + +DIALOG_LOGGER_NAME_MAPPING = { + 'AlexaConversation': 'alexa', + 'AliceConversation': 'alice', + 'MSConversation': 'ms_bot_framework', + 'TelegramConversation': 'telegram', + '_unsupported': 'new_conversation' +} + + +class BaseConversation: + """Receives requests, generates responses.""" + _model: Chainer + _self_destruct_callback: callable + _conversation_id: Union[int, str] + _timer: Timer + _infer_utterances: list + _conversation_lifetime: int + _next_utter_msg: str + _start_message: str + + def __init__(self, + config: dict, + model: Chainer, + self_destruct_callback: callable, + conversation_id: Union[int, str]) -> None: + """Initiates instance properties and starts self-destruct timer. + + Args: + config: Dictionary containing base conversation parameters. + model: Model that infered with user messages. + self_destruct_callback: Function that removes this Conversation instance. + conversation_id: Conversation ID. + + """ + self._model = model + self._self_destruct_callback = self_destruct_callback + self._conversation_id = conversation_id + self._infer_utterances = list() + self._conversation_lifetime = config['conversation_lifetime'] + self._next_utter_msg = config['next_utter_msg'] + self._start_message = config['start_message'] + self._unsupported_message = config['unsupported_msg'] + logger_name: str = DIALOG_LOGGER_NAME_MAPPING.get(type(self).__name__, + DIALOG_LOGGER_NAME_MAPPING['_unsupported']) + self._dialog_logger = DialogLogger(logger_name=logger_name) + self._start_timer() + + def handle_request(self, request: dict) -> Optional[dict]: + """Rearms self-destruct timer and sends the request to a processing. + + Args: + request: Request from the channel. + + Returns: + response: Corresponding to the channel response to the request from the channel if replies are sent via bot, + None otherwise. + + """ + self._rearm_self_destruct() + return self._handle_request(request) + + def _start_timer(self) -> None: + """Initiates self-destruct timer.""" + self._timer = Timer(self._conversation_lifetime, self._self_destruct_callback, [self._conversation_id]) + self._timer.start() + + def _rearm_self_destruct(self) -> None: + """Rearms self-destruct timer.""" + self._timer.cancel() + self._start_timer() + + def _handle_request(self, request: dict) -> Optional[dict]: + """Routes the request to the appropriate handler. + + Args: + request: Request from the channel. + + Returns: + response: Corresponding response to the channel request if replies are sent via bot, None otherwise. + + """ + raise NotImplementedError + + def _handle_launch(self, request: dict) -> Optional[dict]: + """Handles launch request. + + Args: + request: Start request from channel. + + Returns: + response: Greeting message wrapped in the appropriate to the channel structure if replies are sent via bot, + None otherwise. + + """ + response = self._generate_response(self._start_message, request) + + return response + + def _handle_unsupported(self, request: dict) -> Optional[dict]: + """Handles all unsupported request types. + + Args: + request: Request from channel for which a separate handler was not defined. + + Returns: + response: Message that request type is not supported wrapped in the appropriate to the channel data + structure if replies are sent via bot, None otherwise. + + """ + response = self._generate_response(self._unsupported_message, request) + log.warning(f'Unsupported request: {request}') + + return response + + def _generate_response(self, message: str, request: dict) -> Optional[dict]: + """Wraps message in the appropriate to the channel data structure. + + Args: + message: Raw message to be sent to the channel. + request: Request from the channel to which the ``message`` replies. + + Returns: + response: Data structure to be sent to the channel if replies are sent via bot, None otherwise. + + """ + raise NotImplementedError + + def _act(self, utterance: str) -> str: + """Infers DeepPavlov model with utterance. + + If DeepPavlov model requires more than one argument, utterances are accumulated until reaching required + arguments amount to infer. + + Args: + utterance: Text to be processed by DeepPavlov model. + + Returns: + response: Model response if enough model arguments have been accumulated, message prompting for the next + model argument otherwise. + + """ + self._infer_utterances.append([utterance]) + + if len(self._infer_utterances) == len(self._model.in_x): + self._dialog_logger.log_in(self._infer_utterances, self._conversation_id) + prediction = self._model(*self._infer_utterances) + self._infer_utterances = list() + if len(self._model.out_params) == 1: + prediction = [prediction] + prediction = '; '.join([str(output[0]) for output in prediction]) + response = prediction + self._dialog_logger.log_out(response, self._conversation_id) + else: + response = self._next_utter_msg.format(self._model.in_x[len(self._infer_utterances)]) + + return response + + +class AlexaConversation(BaseConversation): + """Receives requests from Amazon Alexa and generates responses.""" + _intent_name: str + _slot_name: str + _handled_requests: Dict[str, callable] + + def __init__(self, config: dict, model, self_destruct_callback: callable, conversation_id: str) -> None: + super(AlexaConversation, self).__init__(config, model, self_destruct_callback, conversation_id) + self._intent_name = config['intent_name'] + self._slot_name = config['slot_name'] + + self._handled_requests = { + 'LaunchRequest': self._handle_launch, + 'IntentRequest': self._handle_intent, + 'SessionEndedRequest': self._handle_end, + '_unsupported': self._handle_unsupported + } + + def _handle_request(self, request: dict) -> dict: + """Routes Alexa requests to the appropriate handler. + + Args: + request: Alexa request. + + Returns: + response: Response conforming to the Alexa response specification. + + """ + request_type = request['request']['type'] + request_id = request['request']['requestId'] + log.debug(f'Received request. Type: {request_type}, id: {request_id}') + + if request_type in self._handled_requests: + response = self._handled_requests[request_type](request) + else: + response = self._handled_requests['_unsupported'](request) + + return response + + def _generate_response(self, message: str, request: dict) -> dict: + """Wraps message in the conforming to the Alexa data structure. + + Args: + message: Raw message to be sent to Alexa. + request: Request from the channel to which the ``message`` replies. + + Returns: + response: Data structure conforming to the Alexa response specification. + + """ + response = { + 'version': '1.0', + 'sessionAttributes': { + 'sessionId': request['session']['sessionId'] + }, + 'response': { + 'shouldEndSession': False, + 'outputSpeech': { + 'type': 'PlainText', + 'text': message + }, + 'card': { + 'type': 'Simple', + 'content': message + } + } + } + + return response + + def _handle_intent(self, request: dict) -> dict: + """Handles IntentRequest Alexa request. + + Args: + request: Alexa request. + + Returns: + response: Data structure conforming to the Alexa response specification. + + """ + request_id = request['request']['requestId'] + request_intent: dict = request['request']['intent'] + + if self._intent_name != request_intent['name']: + log.error(f"Wrong intent name received: {request_intent['name']} in request {request_id}") + return {'error': 'wrong intent name'} + + if self._slot_name not in request_intent['slots'].keys(): + log.error(f'No slot named {self._slot_name} found in request {request_id}') + return {'error': 'no slot found'} + + utterance = request_intent['slots'][self._slot_name]['value'] + model_response = self._act(utterance) + + if not model_response: + log.error(f'Some error during response generation for request {request_id}') + return {'error': 'error during response generation'} + + response = self._generate_response(model_response, request) + + return response + + def _handle_end(self, request: dict) -> dict: + """Handles SessionEndedRequest Alexa request and deletes Conversation instance. + + Args: + request: Alexa request. + + Returns: + response: Dummy empty response dict. + + """ + response = {} + self._self_destruct_callback(self._conversation_id) + return response + + +class AliceConversation(BaseConversation): + """Receives requests from Yandex.Alice and generates responses.""" + def _handle_request(self, request: dict) -> dict: + """Routes Alice requests to the appropriate handler. + + Args: + request: Alice request. + + Returns: + response: Response conforming to the Alice response specification. + + """ + message_id = request['session']['message_id'] + session_id = request['session']['session_id'] + log.debug(f'Received message. Session: {session_id}, message_id: {message_id}') + + if request['session']['new']: + response = self._handle_launch(request) + elif request['request']['command'].strip(): + text = request['request']['command'].strip() + model_response = self._act(text) + response = self._generate_response(model_response, request) + else: + response = self._handle_unsupported(request) + + return response + + def _generate_response(self, message: str, request: dict) -> dict: + """Wraps message in the conforming to the Alice data structure. + + Args: + message: Raw message to be sent to Alice. + request: Request from the channel to which the ``message`` replies. + + Returns: + response: Data structure conforming to the Alice response specification. + + """ + response = { + 'response': { + 'end_session': False, + 'text': message + }, + 'session': { + 'session_id': request['session']['session_id'], + 'message_id': request['session']['message_id'], + 'user_id': request['session']['user_id'] + }, + 'version': '1.0' + } + + return response + + +class MSConversation(BaseConversation): + """Receives requests from Microsoft Bot Framework and generates responses.""" + def __init__(self, + config: dict, + model: Chainer, + self_destruct_callback: callable, + conversation_id: str, + http_session: Session) -> None: + """Initiates instance properties and starts self-destruct timer. + + Args: + config: Dictionary containing base conversation parameters. + model: Model that infered with user messages. + self_destruct_callback: Function that removes this Conversation instance. + conversation_id: Conversation ID. + http_session: Session used to send responses to Bot Framework. + + """ + super(MSConversation, self).__init__(config, model, self_destruct_callback, conversation_id) + self._http_session = http_session + + self._handled_activities = { + 'message': self._handle_message, + 'conversationUpdate': self._handle_launch, + '_unsupported': self._handle_unsupported + } + + def _handle_request(self, request: dict) -> None: + """Routes MS Bot requests to the appropriate handler. Returns None since handlers send responses themselves. + + Args: + request: MS Bot request. + + """ + activity_type = request['type'] + activity_id = request['id'] + log.debug(f'Received activity. Type: {activity_type}, id: {activity_id}') + + if activity_type in self._handled_activities.keys(): + self._handled_activities[activity_type](request) + else: + self._handled_activities['_unsupported'](request) + + self._rearm_self_destruct() + + def _handle_message(self, request: dict) -> None: + """Handles MS Bot message request. + + Request redirected to ``_unsupported`` handler if ms bot message does not contain raw text. + + Args: + request: MS Bot request. + + """ + if 'text' in request: + in_text = request['text'] + model_response = self._act(in_text) + if model_response: + self._generate_response(model_response, request) + else: + self._handled_activities['_unsupported'](request) + + def _generate_response(self, message: str, request: dict) -> None: + """Wraps message in the conforming to the MS Bot data structure and sends it to MS Bot via HTTP session. + + Args: + message: Raw message to be sent to MS Bot. + request: Request from the channel to which the ``message`` replies. + + """ + response = { + "type": "message", + "from": request['recipient'], + "recipient": request['from'], + 'conversation': request['conversation'], + 'text': message + } + + url = urljoin(request['serviceUrl'], f"v3/conversations/{request['conversation']['id']}/activities") + + response = self._http_session.post(url=url, json=response) + + try: + response_json_str = str(response.json()) + except ValueError as e: + response_json_str = repr(e) + + log.debug(f'Sent activity to the MSBotFramework server. ' + f'Response code: {response.status_code}, response contents: {response_json_str}') + + +class TelegramConversation(BaseConversation): + """Receives requests from Telegram bot and generates responses.""" + def _handle_request(self, message: str) -> str: + """Handles raw text message from Telegram bot. + + Args: + message: Message from Telegram bot. + + Returns: + response: Response to a ``message``. + + """ + response = self._act(message) + + return response + + def _generate_response(self, message: str, request: dict) -> None: + """Does nothing.""" + pass diff --git a/deeppavlov/core/agent/dialog_logger.py b/deeppavlov/utils/connector/dialog_logger.py similarity index 88% rename from deeppavlov/core/agent/dialog_logger.py rename to deeppavlov/utils/connector/dialog_logger.py index 21e1d56c6d..bdd4ae0182 100644 --- a/deeppavlov/core/agent/dialog_logger.py +++ b/deeppavlov/utils/connector/dialog_logger.py @@ -18,7 +18,6 @@ from pathlib import Path from typing import Any, Optional, Hashable -from deeppavlov.core.agent.rich_content import RichMessage from deeppavlov.core.common.file import read_json from deeppavlov.core.common.paths import get_settings_path from deeppavlov.core.data.utils import jsonify_data @@ -36,22 +35,22 @@ class DialogLogger: Args: enabled: DialogLogger on/off flag. - agent_name: Agent name which is used for organising log files. + logger_name: Dialog logger name that is used for organising log files. Attributes: - agent_name: Agent name which is used for organising log files. + logger_name: Dialog logger name which is used for organising log files. log_max_size: Maximum size of log file, kb. self.log_file: Current log file object. """ - def __init__(self, enabled: bool = False, agent_name: Optional[str] = None) -> None: + def __init__(self, enabled: bool = False, logger_name: Optional[str] = None) -> None: self.config: dict = read_json(get_settings_path() / LOGGER_CONFIG_FILENAME) self.enabled: bool = enabled or self.config['enabled'] if self.enabled: - self.agent_name: str = agent_name or self.config['agent_name'] + self.logger_name: str = logger_name or self.config['logger_name'] self.log_max_size: int = self.config['logfile_max_size_kb'] self.log_file = self._get_log_file() - self.log_file.writelines('"Agent initiated"\n') + self.log_file.writelines('"Dialog logger initiated"\n') @staticmethod def _get_timestamp_utc_str() -> str: @@ -69,9 +68,9 @@ def _get_log_file(self): Returns: log_file: opened Python file object. """ - log_dir: Path = Path(self.config['log_path']).expanduser().resolve() / self.agent_name + log_dir: Path = Path(self.config['log_path']).expanduser().resolve() / self.logger_name log_dir.mkdir(parents=True, exist_ok=True) - log_file_path = Path(log_dir, f'{self._get_timestamp_utc_str()}_{self.agent_name}.log') + log_file_path = Path(log_dir, f'{self._get_timestamp_utc_str()}_{self.logger_name}.log') log_file = open(log_file_path, 'a', buffering=1, encoding='utf8') return log_file @@ -85,8 +84,6 @@ def _log(self, utterance: Any, direction: str, dialog_id: Optional[Hashable]=Non """ if isinstance(utterance, str): pass - elif isinstance(utterance, RichMessage): - utterance = utterance.json() elif isinstance(utterance, (list, dict)): utterance = jsonify_data(utterance) else: diff --git a/deeppavlov/utils/alexa/ssl_tools.py b/deeppavlov/utils/connector/ssl_tools.py similarity index 96% rename from deeppavlov/utils/alexa/ssl_tools.py rename to deeppavlov/utils/connector/ssl_tools.py index 7f0b528276..572ff56d92 100644 --- a/deeppavlov/utils/alexa/ssl_tools.py +++ b/deeppavlov/utils/connector/ssl_tools.py @@ -29,9 +29,8 @@ def verify_sc_url(url: str) -> bool: """Verify signature certificate URL against Amazon Alexa requirements. - Each call of Agent passes incoming utterances batch through skills filter, - agent skills, skills processor. Batch of dialog IDs can be provided, in - other case utterances indexes in incoming batch are used as dialog IDs. + Batch of dialog IDs can be provided, in other case utterances indexes in + incoming batch are used as dialog IDs. Args: url: Signature certificate URL from SignatureCertChainUrl HTTP header. diff --git a/deeppavlov/utils/ms_bot_framework/__init__.py b/deeppavlov/utils/ms_bot_framework/__init__.py index e69de29bb2..fdd5c51faf 100644 --- a/deeppavlov/utils/ms_bot_framework/__init__.py +++ b/deeppavlov/utils/ms_bot_framework/__init__.py @@ -0,0 +1 @@ +from .server import start_ms_bf_server diff --git a/deeppavlov/utils/ms_bot_framework/bot.py b/deeppavlov/utils/ms_bot_framework/bot.py deleted file mode 100644 index 811ea61cf3..0000000000 --- a/deeppavlov/utils/ms_bot_framework/bot.py +++ /dev/null @@ -1,110 +0,0 @@ -import threading -from collections import namedtuple -from logging import getLogger -from queue import Empty, Queue -from threading import Thread - -import requests -from requests.exceptions import HTTPError - -from .conversation import Conversation - -log = getLogger(__name__) - -ConvKey = namedtuple('ConvKey', ['channel_id', 'conversation_id']) - - -class Bot(Thread): - def __init__(self, agent_generator: callable, config: dict, input_queue: Queue): - super(Bot, self).__init__() - self.config = config - self._run_flag = True - - self.conversations = {} - self.access_info = {} - self.http_sessions = {} - self.input_queue = input_queue - - self.agent = None - self.agent_generator = agent_generator - - if not self.config['multi_instance']: - self.agent = self._init_agent() - log.info('New bot instance level agent initiated') - - polling_interval = self.config['auth_polling_interval'] - self.timer = threading.Timer(polling_interval, self._update_access_info) - self._request_access_info() - self.timer.start() - - def run(self): - while self._run_flag: - try: - activity = self.input_queue.get(timeout=1) - except Empty: - pass - else: - self._handle_activity(activity) - - def join(self, timeout=None): - self._run_flag = False - for timer in threading.enumerate(): - if isinstance(timer, threading.Timer): - timer.cancel() - Thread.join(self, timeout) - - def del_conversation(self, conversation_key: ConvKey): - del self.conversations[conversation_key] - log.info(f'Deleted conversation, key: {str(conversation_key)}') - - def _init_agent(self): - # TODO: Decide about multi-instance mode necessity. - # If model multi-instancing is still necessary - refactor and remove - agent = self.agent_generator() - return agent - - def _update_access_info(self): - polling_interval = self.config['auth_polling_interval'] - self.timer = threading.Timer(polling_interval, self._update_access_info) - self.timer.start() - self._request_access_info() - - def _request_access_info(self): - headers = {'Host': self.config['auth_host'], - 'Content-Type': self.config['auth_content_type']} - - payload = {'grant_type': self.config['auth_grant_type'], - 'scope': self.config['auth_scope'], - 'client_id': self.config['auth_app_id'], - 'client_secret': self.config['auth_app_secret']} - - result = requests.post(url=self.config['auth_url'], - headers=headers, - data=payload) - - status_code = result.status_code - if status_code != 200: - raise HTTPError(f'Authentication token request returned wrong HTTP status code: {status_code}') - - self.access_info = result.json() - log.info(f'Obtained authentication information from Microsoft Bot Framework: {str(self.access_info)}') - - def _handle_activity(self, activity: dict): - conversation_key = ConvKey(activity['channelId'], activity['conversation']['id']) - - if conversation_key not in self.conversations.keys(): - if self.config['multi_instance']: - conv_agent = self._init_agent() - log.info('New conversation instance level agent initiated') - else: - conv_agent = self.agent - - self.conversations[conversation_key] = Conversation(bot=self, - agent=conv_agent, - activity=activity, - conversation_key=conversation_key) - - log.info(f'Created new conversation, key: {str(conversation_key)}') - - conversation = self.conversations[conversation_key] - conversation.handle_activity(activity) diff --git a/deeppavlov/utils/ms_bot_framework/conversation.py b/deeppavlov/utils/ms_bot_framework/conversation.py deleted file mode 100644 index a24b7e1e86..0000000000 --- a/deeppavlov/utils/ms_bot_framework/conversation.py +++ /dev/null @@ -1,156 +0,0 @@ -import threading -from logging import getLogger -from urllib.parse import urljoin - -import requests - -from deeppavlov.core.agent.rich_content import RichMessage - -log = getLogger(__name__) - - -class Conversation: - def __init__(self, bot, agent, activity: dict, conversation_key): - self.bot = bot - self.agent = agent - self.key = conversation_key - - self.bot_id = activity['recipient']['id'] - self.bot_name = activity['recipient']['name'] - self.service_url = activity['serviceUrl'] - self.channel_id = activity['channelId'] - self.conversation_id = activity['conversation']['id'] - - self.out_gateway = OutGateway(self) - self.stateful = self.bot.config['stateful'] - - self.conversation_lifetime = self.bot.config['conversation_lifetime'] - self.timer = None - self._start_timer() - - if self.channel_id not in self.bot.http_sessions.keys() or not self.bot.http_sessions['self.channel_id']: - self.bot.http_sessions['self.channel_id'] = requests.Session() - - self.http_session = self.bot.http_sessions['self.channel_id'] - - self.handled_activities = { - 'message': self._handle_message - } - - def _start_timer(self): - self.timer = threading.Timer(self.conversation_lifetime, self._self_destruct) - self.timer.start() - - def _rearm_self_destruct(self): - self.timer.cancel() - self._start_timer() - - def _self_destruct(self): - self.bot.del_conversation(self.key) - - def handle_activity(self, activity: dict): - activity_type = activity['type'] - activity_id = activity['id'] - log.debug(f'Received activity. Type: {activity_type}, id: {activity_id}') - - if activity_type in self.handled_activities.keys(): - self.handled_activities[activity_type](activity) - else: - log.warning(f'Unsupported activity type: {activity_type}, activity id: {activity_id}') - - self._rearm_self_destruct() - - def _act(self, utterance: str): - if self.stateful: - utterance = [[utterance], [self.key]] - else: - utterance = [[utterance]] - - prediction = self.agent(*utterance) - - return prediction - - def _send_infer_results(self, response: RichMessage, in_activity: dict): - ms_bf_response = response.ms_bot_framework() - for out_activity in ms_bf_response: - if out_activity: - self.out_gateway.send_activity(out_activity, in_activity) - - def _handle_usupported(self, in_activity: dict): - activity_type = in_activity['type'] - self.out_gateway.send_plain_text(f'Unsupported kind of {activity_type} activity!') - log.warn(f'Received message with unsupported type: {str(in_activity)}') - - def _handle_message(self, in_activity: dict): - if 'text' in in_activity.keys(): - in_text = in_activity['text'] - agent_response = self._act(in_text) - if agent_response: - response = agent_response[0] - self._send_infer_results(response, in_activity) - else: - self._handle_usupported(in_activity) - - -class OutGateway: - def __init__(self, conversation: Conversation): - self.conversation = conversation - self.service_url = self.conversation.service_url - self.activity_template = { - 'from': { - 'id': self.conversation.bot_id, - 'name': self.conversation.bot_name - }, - 'conversation': { - 'id': self.conversation.conversation_id - } - } - - def send_activity(self, out_activity: dict, in_activity: dict = None): - service_url = self.service_url - - for key, value in self.activity_template.items(): - out_activity[key] = value - - if in_activity: - try: - service_url = in_activity['serviceUrl'] - except KeyError: - pass - - try: - out_activity['recepient']['id'] = in_activity['from']['id'] - except KeyError: - pass - - try: - out_activity['conversation']['name'] = in_activity['conversation']['name'] - except KeyError: - pass - - try: - out_activity['recepient']['name'] = in_activity['from']['name'] - except KeyError: - pass - - url = urljoin(service_url, f"v3/conversations/{self.conversation.conversation_id}/activities") - - authorization = f"{self.conversation.bot.access_info['token_type']} " \ - f"{self.conversation.bot.access_info['access_token']}" - headers = { - 'Authorization': authorization, - 'Content-Type': 'application/json' - } - - response = self.conversation.http_session.post( - url=url, - json=out_activity, - headers=headers) - - try: - response_json_str = str(response.json()) - except Exception: - response_json_str = '' - - log.debug(f'Sent activity to the MSBotFramework server. ' - f'Response code: {response.status_code}, response contents: {response_json_str}') diff --git a/deeppavlov/utils/ms_bot_framework/server.py b/deeppavlov/utils/ms_bot_framework/server.py index f19df071ea..c79e14f60d 100644 --- a/deeppavlov/utils/ms_bot_framework/server.py +++ b/deeppavlov/utils/ms_bot_framework/server.py @@ -15,110 +15,40 @@ from logging import getLogger from pathlib import Path from queue import Queue -from typing import Union, Optional +from typing import Optional import uvicorn from fastapi import FastAPI -from deeppavlov.agents.default_agent.default_agent import DefaultAgent -from deeppavlov.agents.processors.default_rich_content_processor import DefaultRichContentWrapper -from deeppavlov.core.commands.infer import build_model -from deeppavlov.core.common.file import read_json -from deeppavlov.core.common.paths import get_settings_path -from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill -from deeppavlov.utils.ms_bot_framework.bot import Bot -from deeppavlov.utils.server.server import get_ssl_params, redirect_root_do_docs - -SERVER_CONFIG_FILENAME = 'server_config.json' - -AUTH_URL = "https://login.microsoftonline.com/botframework.com/oauth2/v2.0/token" -AUTH_HOST = "login.microsoftonline.com" -AUTH_CONTENT_TYPE = "application/x-www-form-urlencoded" -AUTH_GRANT_TYPE = "client_credentials" -AUTH_SCOPE = "https://api.botframework.com/.default" +from deeppavlov.utils.connector import MSBot +from deeppavlov.utils.server import get_server_params, get_ssl_params, redirect_root_to_docs log = getLogger(__name__) uvicorn_log = getLogger('uvicorn') app = FastAPI() -def run_ms_bf_default_agent(model_config: Union[str, Path, dict], - app_id: str, - app_secret: str, - multi_instance: bool = False, - stateful: bool = False, - port: Optional[int] = None, - https: bool = False, - ssl_key: Optional[str] = None, - ssl_cert: Optional[str] = None, - default_skill_wrap: bool = True) -> None: - - def get_default_agent() -> DefaultAgent: - model = build_model(model_config) - skill = DefaultStatelessSkill(model) if default_skill_wrap else model - agent = DefaultAgent([skill], skills_processor=DefaultRichContentWrapper()) - return agent - - run_ms_bot_framework_server(agent_generator=get_default_agent, - app_id=app_id, - app_secret=app_secret, - multi_instance=multi_instance, - stateful=stateful, - port=port, - https=https, - ssl_key=ssl_key, - ssl_cert=ssl_cert) - - -def run_ms_bot_framework_server(agent_generator: callable, - app_id: str, - app_secret: str, - multi_instance: bool = False, - stateful: bool = False, - port: Optional[int] = None, - https: bool = False, - ssl_key: Optional[str] = None, - ssl_cert: Optional[str] = None) -> None: - - server_config_path = Path(get_settings_path(), SERVER_CONFIG_FILENAME).resolve() - server_params = read_json(server_config_path) - - host = server_params['common_defaults']['host'] - port = port or server_params['common_defaults']['port'] - - ms_bf_server_params = server_params['ms_bot_framework_defaults'] - - ms_bf_server_params['multi_instance'] = multi_instance or server_params['common_defaults']['multi_instance'] - ms_bf_server_params['stateful'] = stateful or server_params['common_defaults']['stateful'] - - ms_bf_server_params['auth_url'] = AUTH_URL - ms_bf_server_params['auth_host'] = AUTH_HOST - ms_bf_server_params['auth_content_type'] = AUTH_CONTENT_TYPE - ms_bf_server_params['auth_grant_type'] = AUTH_GRANT_TYPE - ms_bf_server_params['auth_scope'] = AUTH_SCOPE +def start_ms_bf_server(model_config: Path, + app_id: Optional[str], + app_secret: Optional[str], + port: Optional[int] = None, + https: bool = False, + ssl_key: Optional[str] = None, + ssl_cert: Optional[str] = None) -> None: - ms_bf_server_params['auth_app_id'] = app_id or ms_bf_server_params['auth_app_id'] - if not ms_bf_server_params['auth_app_id']: - e = ValueError('Microsoft Bot Framework app id required: initiate -i param ' - 'or auth_app_id param in server configuration file') - log.error(e) - raise e + server_params = get_server_params(model_config) - ms_bf_server_params['auth_app_secret'] = app_secret or ms_bf_server_params['auth_app_secret'] - if not ms_bf_server_params['auth_app_secret']: - e = ValueError('Microsoft Bot Framework app secret required: initiate -s param ' - 'or auth_app_secret param in server configuration file') - log.error(e) - raise e + host = server_params['host'] + port = port or server_params['port'] - ssl_config = get_ssl_params(server_params['common_defaults'], https, ssl_key=ssl_key, ssl_cert=ssl_cert) + ssl_config = get_ssl_params(server_params, https, ssl_key=ssl_key, ssl_cert=ssl_cert) input_q = Queue() - bot = Bot(agent_generator, ms_bf_server_params, input_q) + bot = MSBot(model_config, input_q, app_id, app_secret) bot.start() endpoint = '/v3/conversations' - redirect_root_do_docs(app, 'answer', endpoint, 'post') + redirect_root_to_docs(app, 'answer', endpoint, 'post') @app.post(endpoint) async def answer(activity: dict) -> dict: diff --git a/deeppavlov/utils/server/__init__.py b/deeppavlov/utils/server/__init__.py index e69de29bb2..412d18a2cb 100644 --- a/deeppavlov/utils/server/__init__.py +++ b/deeppavlov/utils/server/__init__.py @@ -0,0 +1 @@ +from .server import get_server_params, get_ssl_params, redirect_root_to_docs, start_model_server diff --git a/deeppavlov/utils/server/server.py b/deeppavlov/utils/server/server.py index 92a0c3e1d2..fb8e12333f 100644 --- a/deeppavlov/utils/server/server.py +++ b/deeppavlov/utils/server/server.py @@ -17,7 +17,7 @@ from collections import namedtuple from pathlib import Path from ssl import PROTOCOL_TLSv1_2 -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union import uvicorn from fastapi import Body, FastAPI, HTTPException @@ -28,15 +28,15 @@ from starlette.responses import RedirectResponse from starlette.middleware.cors import CORSMiddleware -from deeppavlov.core.agent.dialog_logger import DialogLogger from deeppavlov.core.commands.infer import build_model from deeppavlov.core.commands.utils import parse_config from deeppavlov.core.common.chainer import Chainer from deeppavlov.core.common.file import read_json from deeppavlov.core.common.paths import get_settings_path from deeppavlov.core.data.utils import check_nested_dict_keys, jsonify_data +from deeppavlov.utils.connector import DialogLogger -SERVER_CONFIG_FILENAME = 'server_config.json' +SERVER_CONFIG_PATH = get_settings_path() / 'server_config.json' SSLConfig = namedtuple('SSLConfig', ['version', 'keyfile', 'certfile']) @@ -60,10 +60,10 @@ def filter(self, record: logging.LogRecord) -> bool: allow_headers=['*'] ) -dialog_logger = DialogLogger(agent_name='dp_api') +dialog_logger = DialogLogger(logger_name='rest_api') -def get_server_params(server_config_path: Path, model_config: Path) -> Dict: +def get_server_params(model_config: Union[str, Path], server_config_path: Path = SERVER_CONFIG_PATH) -> Dict: server_config = read_json(server_config_path) model_config = parse_config(model_config) @@ -114,7 +114,7 @@ def get_ssl_params(server_params: dict, return ssl_config -def redirect_root_do_docs(fast_app: FastAPI, func_name: str, endpoint: str, method: str) -> None: +def redirect_root_to_docs(fast_app: FastAPI, func_name: str, endpoint: str, method: str) -> None: """Adds api route to server that redirects user from root to docs with opened `endpoint` description.""" @fast_app.get('/', include_in_schema=False) async def redirect_to_docs() -> RedirectResponse: @@ -166,8 +166,8 @@ def start_model_server(model_config: Path, ssl_key: Optional[str] = None, ssl_cert: Optional[str] = None, port: Optional[int] = None) -> None: - server_config_path = get_settings_path() / SERVER_CONFIG_FILENAME - server_params = get_server_params(server_config_path, model_config) + + server_params = get_server_params(model_config) host = server_params['host'] port = port or server_params['port'] @@ -189,7 +189,7 @@ def batch_decorator(cls: MetaModel) -> MetaModel: class Batch(BaseModel): pass - redirect_root_do_docs(app, 'answer', model_endpoint, 'post') + redirect_root_to_docs(app, 'answer', model_endpoint, 'post') model_endpoint_post_example = {arg_name: ['string'] for arg_name in model_args_names} @app.post(model_endpoint, summary='A model endpoint') @@ -207,4 +207,4 @@ async def api() -> List[str]: return model_args_names uvicorn.run(app, host=host, port=port, logger=uvicorn_log, ssl_version=ssl_config.version, - ssl_keyfile=ssl_config.keyfile, ssl_certfile=ssl_config.certfile) + ssl_keyfile=ssl_config.keyfile, ssl_certfile=ssl_config.certfile, timeout_keep_alive=20) diff --git a/deeppavlov/utils/settings/connector_config.json b/deeppavlov/utils/settings/connector_config.json new file mode 100644 index 0000000000..1779438538 --- /dev/null +++ b/deeppavlov/utils/settings/connector_config.json @@ -0,0 +1,59 @@ +{ + "bot_defaults": { + "AlexaBot": { + "amazon_cert_lifetime_secs": 3600, + "request_timestamp_tolerance_secs": 150, + "refresh_valid_certs_period_secs": 120, + "intent_name": "AskDeepPavlov", + "slot_name": "raw_input" + }, + "MSBot": { + "auth_polling_interval": 3500, + "auth_url": "https://login.microsoftonline.com/botframework.com/oauth2/v2.0/token", + "auth_headers": { + "Host": "login.microsoftonline.com", + "Content-Type": "application/x-www-form-urlencoded" + }, + "auth_payload": { + "grant_type": "client_credentials", + "scope": "https://api.botframework.com/.default", + "client_id": "", + "client_secret": "" + } + }, + "TelegramBot": { + "token": "" + } + }, + "conversation_defaults": { + "conversation_lifetime": 3600, + "next_utter_msg": "Please enter an argument '{}'", + "unsupported_msg": "Unsupported message received." + }, + "models_info": { + "@default": { + "start_message": "Welcome to DeepPavlov inference bot!", + "help_message": "Welcome to DeepPavlov inference bot!" + }, + "DstcSlotFillingNetwork": { + "start_message": "Welcome to the UI of the DSTC slot filling model. You can enter a text string and get identified slots with values. Example:\n\nFrench cusine\n\n{'food': 'french'}", + "help_message": "Welcome to the UI of the DSTC slot filling model. You can enter a text string and get identified slots with values. Example:\n\nFrench cusine\n\n{'food': 'french'}" + }, + "ErrorModel": { + "start_message": "Welcome to the automatic spelling correction component. You can enter a phrase with typos and get it back corrected. Example:\n\nhelllo\n\nhello", + "help_message": "Welcome to the automatic spelling correction component. You can enter a phrase with typos and get it back corrected. Example:\n\nhelllo\n\nhello" + }, + "GoalOrientedBot": { + "start_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in a restaurant advisory domain. Enter any phrase to start interacting with model.", + "help_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in a restaurant advisory domain. Enter any phrase to start interacting with model." + }, + "Seq2SeqGoalOrientedBot": { + "start_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in three domains: calendar scheduling, weather information retrieval, and point-of-interest navigation. Enter any phrase to start interacting with model. Example:\n\nWhere's the nearest parking garage?\n\nthe nearest is dish parking at dish_parking_address. is that okay?\n\nYes, please set directions via a route that avoids all heavy traffic if possible.\n\nit looks like there is a road block being reported on the route but i will still find the quickest route to dish_parking_address.\n\nThanks so much for your help.\n\nyou're welcome. have a good day. end_of_dialogue", + "help_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in three domains: calendar scheduling, weather information retrieval, and point-of-interest navigation. Enter any phrase to start interacting with model. Example:\n\nWhere's the nearest parking garage?\n\nthe nearest is dish parking at dish_parking_address. is that okay?\n\nYes, please set directions via a route that avoids all heavy traffic if possible.\n\nit looks like there is a road block being reported on the route but i will still find the quickest route to dish_parking_address.\n\nThanks so much for your help.\n\nyou're welcome. have a good day. end_of_dialogue" + }, + "KerasIntentModel": { + "start_message": "Welcome to the UI for the user intent classification model. You can enter enter a text string and get intents (classes which a request belongs with).", + "help_message": "Welcome to the UI for the user intent classification model. You can enter enter a text string and get intents (classes which a request belongs with)." + } + } +} \ No newline at end of file diff --git a/deeppavlov/utils/settings/dialog_logger_config.json b/deeppavlov/utils/settings/dialog_logger_config.json index fd4105d965..625d990c86 100644 --- a/deeppavlov/utils/settings/dialog_logger_config.json +++ b/deeppavlov/utils/settings/dialog_logger_config.json @@ -1,6 +1,6 @@ { "enabled": false, - "agent_name": "dp_agent", + "logger_name": "default", "log_path": "~/.deeppavlov/dialog_logs", "logfile_max_size_kb": 10240, "ensure_ascii": false diff --git a/deeppavlov/utils/settings/models_info.json b/deeppavlov/utils/settings/models_info.json deleted file mode 100644 index bbef74d626..0000000000 --- a/deeppavlov/utils/settings/models_info.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "@default": { - "start_message": "Welcome to DeepPavlov inference bot!", - "help_message": "Welcome to DeepPavlov inference bot!" - }, - "DstcSlotFillingNetwork": { - "start_message": "Welcome to the UI of the DSTC slot filling model. You can enter a text string and get identified slots with values. Example:\n\nFrench cusine\n\n{'food': 'french'}", - "help_message": "Welcome to the UI of the DSTC slot filling model. You can enter a text string and get identified slots with values. Example:\n\nFrench cusine\n\n{'food': 'french'}" - }, - "ErrorModel": { - "start_message": "Welcome to the automatic spelling correction component. You can enter a phrase with typos and get it back corrected. Example:\n\nhelllo\n\nhello", - "help_message": "Welcome to the automatic spelling correction component. You can enter a phrase with typos and get it back corrected. Example:\n\nhelllo\n\nhello" - }, - "GoalOrientedBot": { - "start_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in a restaurant advisory domain. Enter any phrase to start interacting with model.", - "help_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in a restaurant advisory domain. Enter any phrase to start interacting with model." - }, - "Seq2SeqGoalOrientedBot": { - "start_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in three domains: calendar scheduling, weather information retrieval, and point-of-interest navigation. Enter any phrase to start interacting with model. Example:\n\nWhere's the nearest parking garage?\n\nthe nearest is dish parking at dish_parking_address. is that okay?\n\nYes, please set directions via a route that avoids all heavy traffic if possible.\n\nit looks like there is a road block being reported on the route but i will still find the quickest route to dish_parking_address.\n\nThanks so much for your help.\n\nyou're welcome. have a good day. end_of_dialogue", - "help_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in three domains: calendar scheduling, weather information retrieval, and point-of-interest navigation. Enter any phrase to start interacting with model. Example:\n\nWhere's the nearest parking garage?\n\nthe nearest is dish parking at dish_parking_address. is that okay?\n\nYes, please set directions via a route that avoids all heavy traffic if possible.\n\nit looks like there is a road block being reported on the route but i will still find the quickest route to dish_parking_address.\n\nThanks so much for your help.\n\nyou're welcome. have a good day. end_of_dialogue" - }, - "KerasIntentModel": { - "start_message": "Welcome to the UI for the user intent classification model. You can enter enter a text string and get intents (classes which a request belongs with).", - "help_message": "Welcome to the UI for the user intent classification model. You can enter enter a text string and get intents (classes which a request belongs with)." - } -} diff --git a/deeppavlov/utils/settings/server_config.json b/deeppavlov/utils/settings/server_config.json index 67920833d2..a9f315182b 100644 --- a/deeppavlov/utils/settings/server_config.json +++ b/deeppavlov/utils/settings/server_config.json @@ -5,25 +5,7 @@ "model_args_names": "", "https": false, "https_cert_path": "", - "https_key_path": "", - "stateful": false, - "multi_instance": false - }, - "telegram_defaults": { - "token": "" - }, - "ms_bot_framework_defaults": { - "auth_polling_interval": 3500, - "conversation_lifetime": 3600, - "auth_app_id": "", - "auth_app_secret": "" - }, - "alexa_defaults": { - "intent_name": "AskDeepPavlov", - "slot_name": "raw_input", - "start_message": "Welcome to DeepPavlov Alexa wrapper!", - "unsupported_message": "Sorry, DeepPavlov can't understand it.", - "conversation_lifetime": 3600 + "https_key_path": "" }, "model_defaults": { "DstcSlotFillingNetwork": { diff --git a/deeppavlov/utils/socket/__init__.py b/deeppavlov/utils/socket/__init__.py index e69de29bb2..843dfda563 100644 --- a/deeppavlov/utils/socket/__init__.py +++ b/deeppavlov/utils/socket/__init__.py @@ -0,0 +1 @@ +from .socket import start_socket_server, SOCKET_CONFIG_FILENAME diff --git a/deeppavlov/utils/socket/socket.py b/deeppavlov/utils/socket/socket.py index c877070d32..55c66a8020 100644 --- a/deeppavlov/utils/socket/socket.py +++ b/deeppavlov/utils/socket/socket.py @@ -19,15 +19,17 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple, Union -from deeppavlov.core.agent.dialog_logger import DialogLogger from deeppavlov.core.commands.infer import build_model from deeppavlov.core.common.chainer import Chainer from deeppavlov.core.common.paths import get_settings_path from deeppavlov.core.data.utils import jsonify_data -from deeppavlov.utils.server.server import get_server_params +from deeppavlov.utils.connector import DialogLogger +from deeppavlov.utils.server import get_server_params SOCKET_CONFIG_FILENAME = 'socket_config.json' +dialog_logger = DialogLogger(logger_name='socket_api') + class SocketServer: """Creates socket server that sends the received data to the DeepPavlov model and returns model response. @@ -67,7 +69,7 @@ def __init__(self, model_config: Path, socket_type: str, port: Optional[int] = N """ socket_config_path = get_settings_path() / SOCKET_CONFIG_FILENAME - self._params = get_server_params(socket_config_path, model_config) + self._params = get_server_params(model_config, socket_config_path) self._socket_type = socket_type or self._params['socket_type'] if self._socket_type == 'TCP': @@ -87,7 +89,6 @@ def __init__(self, model_config: Path, socket_type: str, port: Optional[int] = N else: raise ValueError(f'socket type "{self._socket_type}" is not supported') - self._dialog_logger = DialogLogger(agent_name='dp_api') self._log = getLogger(__name__) self._loop = asyncio.get_event_loop() self._model = build_model(model_config) @@ -132,7 +133,7 @@ async def _handle_connection(self, conn: socket.socket, addr: Tuple) -> None: except ValueError: await self._wrap_error(conn, f'request "{recv_data}" type is not json') return - self._dialog_logger.log_in(data) + dialog_logger.log_in(data) model_args = [] for param_name in self._params['model_args_names']: param_value = data.get(param_name) @@ -160,7 +161,7 @@ async def _handle_connection(self, conn: socket.socket, addr: Tuple) -> None: prediction = [prediction] prediction = list(zip(*prediction)) result = await self._response('OK', prediction) - self._dialog_logger.log_out(result) + dialog_logger.log_out(result) await self._loop.sock_sendall(conn, result) async def _wrap_error(self, conn: socket.socket, error: str) -> None: diff --git a/deeppavlov/utils/telegram/__init__.py b/deeppavlov/utils/telegram/__init__.py index e69de29bb2..c04a651276 100644 --- a/deeppavlov/utils/telegram/__init__.py +++ b/deeppavlov/utils/telegram/__init__.py @@ -0,0 +1 @@ +from .telegram_ui import interact_model_by_telegram diff --git a/deeppavlov/utils/telegram/telegram_ui.py b/deeppavlov/utils/telegram/telegram_ui.py index a09a342626..81b439d56c 100644 --- a/deeppavlov/utils/telegram/telegram_ui.py +++ b/deeppavlov/utils/telegram/telegram_ui.py @@ -1,87 +1,23 @@ -""" -Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from logging import getLogger from pathlib import Path from typing import Union -import telebot - -from deeppavlov.agents.default_agent.default_agent import DefaultAgent -from deeppavlov.agents.processors.default_rich_content_processor import DefaultRichContentWrapper -from deeppavlov.core.agent import Agent -from deeppavlov.core.agent.rich_content import RichMessage -from deeppavlov.core.commands.infer import build_model -from deeppavlov.core.common.file import read_json -from deeppavlov.core.common.paths import get_settings_path -from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill - -log = getLogger(__name__) - -SERVER_CONFIG_FILENAME = 'server_config.json' -TELEGRAM_MODELS_INFO_FILENAME = 'models_info.json' - - -def init_bot_for_model(agent: Agent, token: str, model_name: str): - bot = telebot.TeleBot(token) - - models_info_path = Path(get_settings_path(), TELEGRAM_MODELS_INFO_FILENAME).resolve() - models_info = read_json(str(models_info_path)) - model_info = models_info[model_name] if model_name in models_info else models_info['@default'] - - @bot.message_handler(commands=['start']) - def send_start_message(message): - chat_id = message.chat.id - out_message = model_info['start_message'] - bot.send_message(chat_id, out_message) - - @bot.message_handler(commands=['help']) - def send_help_message(message): - chat_id = message.chat.id - out_message = model_info['help_message'] - bot.send_message(chat_id, out_message) - - @bot.message_handler() - def handle_inference(message): - chat_id = message.chat.id - context = message.text - - response: RichMessage = agent([context], [chat_id])[0] - for message in response.json(): - message_text = message['content'] - bot.send_message(chat_id, message_text) - - bot.polling() - - -def interact_model_by_telegram(model_config: Union[str, Path, dict], - token=None, - default_skill_wrap: bool = True): - - server_config_path = Path(get_settings_path(), SERVER_CONFIG_FILENAME) - server_config = read_json(server_config_path) - token = token if token else server_config['telegram_defaults']['token'] +from deeppavlov.utils.connector import TelegramBot - if not token: - e = ValueError('Telegram token required: initiate -t param or telegram_defaults/token ' - 'in server configuration file') - log.error(e) - raise e - model = build_model(model_config) - model_name = type(model.get_main_component()).__name__ - skill = DefaultStatelessSkill(model) if default_skill_wrap else model - agent = DefaultAgent([skill], skills_processor=DefaultRichContentWrapper()) - init_bot_for_model(agent, token, model_name) +def interact_model_by_telegram(model_config: Union[str, Path, dict], token: str = None) -> None: + bot = TelegramBot(model_config, token) + bot.start() diff --git a/docs/apiref/agents.rst b/docs/apiref/agents.rst deleted file mode 100644 index 93cf6dacc5..0000000000 --- a/docs/apiref/agents.rst +++ /dev/null @@ -1,12 +0,0 @@ -agents -====== -Agent classes. - -.. automodule:: deeppavlov.agents - :members: - -.. toctree:: - :glob: - :caption: Agents - - agents/* diff --git a/docs/apiref/agents/default_agent.rst b/docs/apiref/agents/default_agent.rst deleted file mode 100644 index e666b4c010..0000000000 --- a/docs/apiref/agents/default_agent.rst +++ /dev/null @@ -1,6 +0,0 @@ -deeppavlov.agents.default_agent -=============================== -Default agent with filters and processors support. - -.. automodule:: deeppavlov.agents.default_agent.default_agent - :members: diff --git a/docs/apiref/agents/filters.rst b/docs/apiref/agents/filters.rst deleted file mode 100644 index c009f7acb6..0000000000 --- a/docs/apiref/agents/filters.rst +++ /dev/null @@ -1,6 +0,0 @@ -deeppavlov.agents.filters -========================= -Filters for default agent. - -.. automodule:: deeppavlov.agents.filters.transparent_filter - :members: diff --git a/docs/apiref/agents/hello_bot_agent.rst b/docs/apiref/agents/hello_bot_agent.rst deleted file mode 100644 index 3bdc6c040b..0000000000 --- a/docs/apiref/agents/hello_bot_agent.rst +++ /dev/null @@ -1,6 +0,0 @@ -deeppavlov.agents.hello_bot_agent -================================= -Simple hello bot agent built on pattern matching skill. - -.. automodule:: deeppavlov.agents.hello_bot_agent.hello_bot_agent - :members: diff --git a/docs/apiref/agents/processors.rst b/docs/apiref/agents/processors.rst deleted file mode 100644 index 4d4193a147..0000000000 --- a/docs/apiref/agents/processors.rst +++ /dev/null @@ -1,12 +0,0 @@ -deeppavlov.agents.processors -============================ -Processors for default agent. - -.. automodule:: deeppavlov.agents.processors.default_rich_content_processor - :members: - -.. automodule:: deeppavlov.agents.processors.highest_confidence_selector - :members: - -.. automodule:: deeppavlov.agents.processors.random_selector - :members: diff --git a/docs/apiref/agents/rich_content.rst b/docs/apiref/agents/rich_content.rst deleted file mode 100644 index 506a4af5a2..0000000000 --- a/docs/apiref/agents/rich_content.rst +++ /dev/null @@ -1,6 +0,0 @@ -deeppavlov.agents.rich_content -============================== -Rich content support implemented for various channels. - -.. automodule:: deeppavlov.agents.rich_content.default_rich_content - :members: diff --git a/docs/apiref/core/agent.rst b/docs/apiref/core/agent.rst deleted file mode 100644 index 8c776e540d..0000000000 --- a/docs/apiref/core/agent.rst +++ /dev/null @@ -1,18 +0,0 @@ -deeppavlov.core.agent -===================== -Basic classes for building DeepPavlov agents. - -.. automodule:: deeppavlov.core.agent.agent - :members: - -.. automodule:: deeppavlov.core.agent.dialog_logger - :members: - -.. automodule:: deeppavlov.core.agent.filter - :members: - -.. automodule:: deeppavlov.core.agent.processor - :members: - -.. automodule:: deeppavlov.core.agent.rich_content - :members: diff --git a/docs/apiref/core/skill.rst b/docs/apiref/core/skill.rst deleted file mode 100644 index 8574f4563e..0000000000 --- a/docs/apiref/core/skill.rst +++ /dev/null @@ -1,6 +0,0 @@ -deeppavlov.core.skill -===================== -Basic class for building DeepPavlov skills. - -.. automodule:: deeppavlov.core.skill.skill - :members: diff --git a/docs/apiref/skills/aiml_skill.rst b/docs/apiref/skills/aiml_skill.rst index ce4e121f9d..97e7e0ffce 100644 --- a/docs/apiref/skills/aiml_skill.rst +++ b/docs/apiref/skills/aiml_skill.rst @@ -1,5 +1,5 @@ deeppavlov.skills.aiml_skill -======================================== +============================ .. automodule:: deeppavlov.skills.aiml_skill.aiml_skill :members: diff --git a/docs/apiref/skills/default_skill.rst b/docs/apiref/skills/default_skill.rst deleted file mode 100644 index f6f954eb88..0000000000 --- a/docs/apiref/skills/default_skill.rst +++ /dev/null @@ -1,6 +0,0 @@ -deeppavlov.skills.default_skill -=============================== -Skill used for wrapping DeepPavlov models. - -.. automodule:: deeppavlov.skills.default_skill.default_skill - :members: diff --git a/docs/apiref/skills/dsl_skill.rst b/docs/apiref/skills/dsl_skill.rst index facaf57a38..7bc6cf2fed 100644 --- a/docs/apiref/skills/dsl_skill.rst +++ b/docs/apiref/skills/dsl_skill.rst @@ -1,5 +1,5 @@ deeppavlov.skills.dsl_skill -======================================== +============================================ .. automodule:: deeppavlov.skills.dsl_skill.dsl_skill :members: diff --git a/docs/apiref/skills/ecommerce_skill.rst b/docs/apiref/skills/ecommerce_skill.rst deleted file mode 100644 index 87b32a4a3b..0000000000 --- a/docs/apiref/skills/ecommerce_skill.rst +++ /dev/null @@ -1,5 +0,0 @@ -deeppavlov.skills.ecommerce_skill -================================= - -.. automodule:: deeppavlov.skills.ecommerce_skill.tfidf_retrieve - :members: diff --git a/docs/apiref/skills/pattern_matching_skill.rst b/docs/apiref/skills/pattern_matching_skill.rst deleted file mode 100644 index c5fc3001e5..0000000000 --- a/docs/apiref/skills/pattern_matching_skill.rst +++ /dev/null @@ -1,5 +0,0 @@ -deeppavlov.skills.pattern_matching_skill -======================================== - -.. automodule:: deeppavlov.skills.pattern_matching_skill.pattern_matching_skill - :members: diff --git a/docs/apiref/skills/rasa_skill.rst b/docs/apiref/skills/rasa_skill.rst index c4cef207fb..4dcd93f7ac 100644 --- a/docs/apiref/skills/rasa_skill.rst +++ b/docs/apiref/skills/rasa_skill.rst @@ -1,5 +1,5 @@ deeppavlov.skills.rasa_skill -======================================== +============================ .. automodule:: deeppavlov.skills.rasa_skill.rasa_skill :members: diff --git a/docs/features/overview.rst b/docs/features/overview.rst index c388f4e911..3d22cba3b6 100644 --- a/docs/features/overview.rst +++ b/docs/features/overview.rst @@ -501,16 +501,6 @@ Comparison of deeppavlov pretrained model with others: .. _`Stanford Kvret`: https://nlp.stanford.edu/blog/a-new-multi-turn-multi-domain-task-oriented-dialogue-dataset/ -eCommerce bot :doc:`[docs] ` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The eCommerce bot intends to retrieve product items from catalog in sorted order. In addition, it asks an user to provide additional information to specify the search. - -.. note:: - - About **130 Mb** on disc required for eCommerce bot with TfIdf-based ranker and **500 Mb** for BLEU-based ranker. - - ODQA :doc:`[docs] ` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/features/skills/aiml_skill.rst b/docs/features/skills/aiml_skill.rst index 2f2c09944e..ac385c7ea9 100644 --- a/docs/features/skills/aiml_skill.rst +++ b/docs/features/skills/aiml_skill.rst @@ -27,9 +27,7 @@ Usage .. code:: python - from deeppavlov.agents.default_agent.default_agent import DefaultAgent - from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector - from deeppavlov.skills.aiml_skill.aiml_skill import AIMLSkill + from deeppavlov.skills.aiml_skill import AIMLSkill aiml_skill_config = { 'positive_confidence': 0.66, @@ -39,6 +37,8 @@ Usage } aiml_skill = AIMLSkill(**aiml_skill_config) - agent = DefaultAgent([aiml_skill], skills_selector=HighestConfidenceSelector()) - responses = agent(["Hello"]) - print(responses) + + states_batch = None + for utterance in ["Hello", "Hello to the same user_id"]: + responses_batch, confidences_batch, states_batch = aiml_skill([utterance], states_batch) + print(responses_batch[0]) diff --git a/docs/features/skills/dsl_skill.rst b/docs/features/skills/dsl_skill.rst index 575558cfc6..919b1661a5 100644 --- a/docs/features/skills/dsl_skill.rst +++ b/docs/features/skills/dsl_skill.rst @@ -13,7 +13,7 @@ For the case when no match occurred DSL skill returns the argument `on_invalid_c Quick Start ----------- -DeepPavlov library has default config for DSLSkill here: :config:`configs/dsl_skill/dsl_skill.json ` +DeepPavlov library has default config for DSLSkill here: :config:`configs/skills/dsl_skill.json ` Usage ^^^^^^^^ @@ -22,7 +22,7 @@ Usage from deeppavlov import configs, build_model from deeppavlov.core.common.file import read_json - from deeppavlov.skills.dsl_skill.dsl_skill import DSLMeta + from deeppavlov.skills.dsl_skill import DSLMeta class DSLSkill(metaclass=DSLMeta): @@ -33,7 +33,7 @@ Usage return response, confidence - skill_config = read_json(configs.dsl_skill.dsl_skill) + skill_config = read_json(configs.skills.dsl_skill) skill = build_model(skill_config, download=True) utterance = "Hello" diff --git a/docs/features/skills/ecommerce.rst b/docs/features/skills/ecommerce.rst deleted file mode 100644 index 3e511fb8c5..0000000000 --- a/docs/features/skills/ecommerce.rst +++ /dev/null @@ -1,182 +0,0 @@ -eCommerce Bot -====================== - -The eCommerce bot helps you to identify the most relevant product items according to your search query. The retrieval is based on the list of the ranking measures. In addition, when retrieved candidates are not specific enough, the bot asks you to provide additional information to specify the search (as on the example below). - -Here is a simple example of interaction: - -.. code:: bash - - >> Hello, I am a new eCommerce bot. - I will help you to find products that you are looking for. - Please type your query in plain text. - - x::bluetooth speaker - >> This is what I found for you: - - - Bluetooth Speaker (Black & Red) - - Bose SoundLink Bluetooth Speaker III - - Bose SoundLink Mini Bluetooth Speaker - - Bose SoundLink Mini Bluetooth Speaker - - JBL Flip Wireless Bluetooth Speaker (Black) - - To specify the search, please choose a Brand: JBL, Soundsworks - - x::JBL - >> The following items satisfy your request - - - JBL Flip Wireless Bluetooth Speaker (Black) - - JBL Flip Wireless Bluetooth Speaker (Black) - - JBL Charge Portable Indoor/Outdoor Bluetooth Speaker | Black - - - - - -Quick Start ------------ - -Building -^^^^^^^^ - -.. code:: python - - from deeppavlov import configs - from deeppavlov.core.commands.infer import build_model - - ecommerce = build_model(configs.ecommerce_skill.tfidf_retrieve, load_trained=True) - - -Inference -^^^^^^^^^ - -.. code:: python - - result = ecommerce(['bluetooth speaker'], [[]], [{}]) - -If some required packages are missing, install all the requirements by running in command line: - -.. code:: bash - - python -m deeppavlov install tfidf_retrieve - python -m deeppavlov install bleu_retrieve - - -Usage ------ - -Config file -^^^^^^^^^^^ - -BLEU-based [1]_ eCommerce bot -:config:`ecommerce_skill/bleu_retrieve.json` - -TfIdf-based eCommerce bot -:config:`ecommerce_skill/tfidf_retrieve.json` - - -Usage example -^^^^^^^^^^^^^ - -To interact with a pretrained model run: - -.. code:: bash - - python -m deeppavlov interact [-d] - -where ```` is a path to config file. - -You can also train your own model by specifying config file and running: - -.. code:: bash - - python -m deeppavlov train - -Configuration settings ----------------------- - -The eCommerce bot configuration consists of the following parts: - -- **dataset_reader** -- **dataset_iterator** -- **chainer** - -You can use your own **dataset_reader**, **dataset_iterator** for specific data. - -eCommerce bot with BLEU-based ranker -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- **chainer** - pipeline manager - - - ``in`` - pipeline input data: an user ``query``, a dialog ``state`` and dialog history ``history``. - - ``out`` - pipeline output data: ``response`` the structure with retrieved product items. - -- **ecommerce_skill_bleu** - ranker - - - ``min_similarity``: lower boundary for textual similarity ranker (by default 0.5). - - ``min_entropy``: lower boundary for entropy (by default 0.5). If the entropy is less than ``min_entropy``, it's omitted from the specification list. - - ``entropy_fields``: the specification attributes of the catalog items (by default "Size", "Brand", "Author", "Color", "Genre"). - - ``preprocess``: text preprocessing component. - - - **Input:** - - - ``query``: a plain text user query. - - ``history``: dialog history. - - ``state``: dialog state. - - - **Returns:** - - - ``items``: product items in sorted order from ``start`` index till ``end`` index (taken from the dialog state). - - ``entropies``: specification attributes with corresponding values in sorted order. - - ``confidence``: similarity confidence. - - ``state``: dialog state. - - -.. note:: - - About **500 Mb** on disc required for eCommerce bot with BLEU-based ranker. - - -eCommerce bot with TfIdf-based ranker -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- **chainer** - pipeline manager - - - ``in`` - pipeline input data: an user ``query``, a dialog ``state`` and dialog history ``history``. - - ``out`` - pipeline output data: ``response`` the structure with retrieved product items. - -- **ecommerce_skill_tfidf** - ranker - - - ``min_similarity``: lower boundary for textual similarity ranker (by default 0.5). - - ``min_entropy``: lower boundary for entropy (by default 0.5). If the entropy is less than ``min_entropy``, it's omitted from the specification list. - - ``entropy_fields``: the specification attributes of the catalog items (by default "Size", "Brand", "Author", "Color", "Genre"). - - - **Input:** - - - ``query``: a plain text user query. - - ``history``: dialog history. - - ``state``: dialog state. - - - **Returns:** - - - ``items``: product items in sorted order from ``start`` index till ``end`` index (taken from the dialog state). - - ``entropies``: specification attributes with corresponding values in sorted order. - - ``confidence``: similarity confidence. - - ``state``: dialog state. - - -.. note:: - - About **130 Mb** on disc required for eCommerce bot with TfIdf-based ranker - - -References ----------- - -.. [1] Papineni, Kishore, et al. "BLEU: a method for automatic evaluation - of machine translation." Proceedings of the 40th annual meeting on association - for computational linguistics. Association for Computational Linguistics, 2002. diff --git a/docs/features/skills/pattern_matching.rst b/docs/features/skills/pattern_matching.rst deleted file mode 100644 index adc687d8e8..0000000000 --- a/docs/features/skills/pattern_matching.rst +++ /dev/null @@ -1,7 +0,0 @@ -Pattern Matching Skill -====================== - -A :doc:`basic skill implementation` that will always respond with -one of predefined responses chosen at random. Skill's confidence equals ``1`` for incoming utterances that match any -of predefined patterns or ``0`` for utterances that do not. If no patterns were defined for a skill, its confidence will -always be equal to ``0.5``. diff --git a/docs/features/skills/rasa_skill.rst b/docs/features/skills/rasa_skill.rst index 0e9427a119..fea68eb23c 100644 --- a/docs/features/skills/rasa_skill.rst +++ b/docs/features/skills/rasa_skill.rst @@ -37,15 +37,15 @@ Usage without DeepPavlov configuration files .. code:: python - from deeppavlov.agents.default_agent.default_agent import DefaultAgent - from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector - from deeppavlov.skills.rasa_skill.rasa_skill import RASASkill + from deeppavlov.skills.rasa_skill import RASASkill rasa_skill_config = { 'path_to_models': , } rasa_skill = RASASkill(**rasa_skill_config) - agent = DefaultAgent([rasa_skill], skills_selector=HighestConfidenceSelector()) - responses = agent(["Hello"]) - print(responses) + + states_batch = None + for utterance in ["Hello", "Hello to the same user_id"]: + responses_batch, confidences_batch, states_batch = rasa_skill([utterance], states_batch) + print(responses_batch[0]) diff --git a/docs/index.rst b/docs/index.rst index 6e2cef9165..fd1ae647b5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -46,10 +46,8 @@ Welcome to DeepPavlov's documentation! Goal-Oriented Dialogue Bot Open-Domain Question Answering - Pattern Matching Sequence-To-Sequence Dialogue Bot Frequently Asked Questions Answering - eCommerce Bot AIML Rasa DSL diff --git a/docs/integrations/amazon_alexa.rst b/docs/integrations/amazon_alexa.rst index 856511637b..10dabe3076 100644 --- a/docs/integrations/amazon_alexa.rst +++ b/docs/integrations/amazon_alexa.rst @@ -186,7 +186,7 @@ DeepPavlov skill/model can be made available for Amazon Alexa as a REST service .. code:: bash - python -m deeppavlov alexa --https --key --cert [-d] [-p ] [--stateful] [--multi-instance] [--no-default-skill] + python -m deeppavlov alexa --https --key --cert [-d] [-p ] If you redirect requests to your skills service from some https endpoint, you may want to run it in http mode by omitting ``--https``, ``--key``, ``--cert`` keys. @@ -195,14 +195,7 @@ Optional ``-d`` key can be provided for dependencies download before service start. Optional ``-p`` key can be provided to override the port value from a settings file. - -Optional ``--stateful`` flag should be provided for stateful skills/models. - -Optional ``--multi-instance`` can be provided if you wish to raise separate skill/model instance for **each** conversation. -You should use ``--no-default-skill`` optional flag if your model implements an interface of DeepPavlov *Skill* -to skip its wrapping with DeepPavlov *DefaultStatelessSkill*. - REST service properties (host, port, https options) are provided in ``deeppavlov/utils/settings/server_config.json``. Please note, that all command line parameters override corresponding config ones. diff --git a/docs/integrations/ms_bot.rst b/docs/integrations/ms_bot.rst index 67786bd1aa..7ec34ad366 100644 --- a/docs/integrations/ms_bot.rst +++ b/docs/integrations/ms_bot.rst @@ -83,7 +83,7 @@ as a REST service by: .. code:: bash - python -m deeppavlov interactmsbot -i -s --https --key --cert [-d] [-p ] [--stateful] [--multi-instance] [--no-default-skill] + python -m deeppavlov interactmsbot -i -s --https --key --cert [-d] [-p ] Use *Microsoft App ID* and *Microsoft App Secret* obtained in the **Web App Bot connection configuration** section. @@ -95,15 +95,8 @@ Optional ``-d`` key can be provided for dependencies download before service start. Optional ``-p`` key can be provided to override the port value from a settings file. - -Optional ``--stateful`` flag should be provided for stateful skills/models. - -Optional ``--multi-instance`` can be provided if you wish to raise separate skill/model instance for **each** conversation. -You should use ``--no-default-skill`` optional flag if your model implements an interface of DeepPavlov *Skill* -to skip its wrapping with DeepPavlov *DefaultStatelessSkill*. - REST service properties (host, port) are provided in ``deeppavlov/utils/settings/server_config.json``. You can also store your app id and app secret in appropriate section of ``server_config.json``. Please note, that all command line parameters override corresponding config ones. diff --git a/docs/integrations/settings.rst b/docs/integrations/settings.rst index 32646665f3..9d27edaf1d 100644 --- a/docs/integrations/settings.rst +++ b/docs/integrations/settings.rst @@ -16,13 +16,14 @@ To reset settings in the current settings directory one can use ``python -m deep 2. Dialog logging ----------------- -DeepPavlov supports logging of dialogs carried by Agent or ``riseapi`` instances. You can manage dialog logging by editing ``dialog_logger_config.json`` settings file in a settings directory. +DeepPavlov supports logging of infered utterances and DeepPavlov model responses. You can manage dialog logging by +editing ``dialog_logger_config.json`` file in a settings directory. Following dialog logging settings are available: 1. **enabled** (default: ``false``): turns on/off dialog logging for DeepPavlov instance; 2. **log_path** (default: ``~/.deeppavlov/dialog_logs``): sets directory where dialog logs are stored; -3. **agent_name** (default: ``dp_agent``): sets subdirectory name for storing dialog logs; +3. **logger_name** (default: ``default``): sets subdirectory name for storing dialog logs; 4. **logfile_max_size_kb** (default: ``10240``): sets logfile maximum size in kilobytes. If exceeded, new log file is created; 5. **ensure_ascii** (default: ``false``): If ``true``, converts all non-ASCII symbols in logged content to Unicode code points. diff --git a/docs/integrations/telegram.rst b/docs/integrations/telegram.rst index c5356650e4..bdf171c56e 100644 --- a/docs/integrations/telegram.rst +++ b/docs/integrations/telegram.rst @@ -8,21 +8,16 @@ You can do it using command line interface or using python. Command line interface ~~~~~~~~~~~~~~~~~~~~~~ -To run a model specified by the ```` config file as a telegram bot +To run a model specified by the ```` config file as a Telegram bot with a ````: .. code:: bash - python -m deeppavlov interactbot -t [-d] [--no-default-skill] + python -m deeppavlov interactbot -t [-d] * ``-t ``: specifies telegram token as ````. * ``-d``: downloads model specific data before starting the service. -* ``-no-default-skill``: states that your model is already implements an - interface of a :class:`~deeppavlov.core.skill.skill.Skill` and doesn't - need additional wrapping into a stateless skill - :class:`~deeppavlov.skills.default_skill.default_skill.DefaultStatelessSkill` (models from - Skills section require the flag). The command will print the used host and port. Default web service properties (host, port, model endpoint, GET request arguments) can be modified via changing @@ -31,36 +26,19 @@ configuration is described in :doc:`REST API ` section. If you want to get custom ``/start`` and ``/help`` Telegram messages for the running model you should: -* Add section to ``deeppavlov/utils/settings/models_info.json`` with your custom Telegram messages +* Add section to ``models_info`` section of ``deeppavlov/utils/settings/connector_config.json`` with your custom + Telegram messages * In model config file specify ``metadata.labels.telegram_utils`` parameter with name which - refers to the added section of ``deeppavlov/utils/settings/models_info.json`` + refers to the added section of ``deeppavlov/utils/settings/connector_config.json`` ``models_info`` section. Python ~~~~~~ To run a model specified by a DeepPavlov config ```` as as -Telegram bot, you have to turn it to a :class:`~deeppavlov.core.skill.skill.Skill` -and then make it an :class:`~deeppavlov.core.agent.agent.Agent`. +Telegram bot, you have to run following code: .. code:: python - from deeppavlov import build_model - from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill - from deeppavlov.agents.default_agent.default_agent import DefaultAgent - from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector - from deeppavlov.utils.telegram.telegram_ui import init_bot_for_model - - model = build_model("", download=True) - - # Step 1: make it a Skill - skill = DefaultStatelessSkill(model) - # Step 2: make it an Agent - agent = DefaultAgent(skills=[skill]) - # Step 3: run server - init_bot_for_model(agent, token="", name="my_model_name") - -If your model is already a subclass of :class:`~deeppavlov.core.skill.skill.Skill` -or a subclass of :class:`~deeppavlov.core.agent.agent.Agent` (see -:doc:`skills ` and :doc:`agents `) you can skip -corresponding steps. + from deeppavlov.utils.telegram import interact_model_by_telegram + interact_model_by_telegram(model_config=, token=) diff --git a/docs/integrations/yandex_alice.rst b/docs/integrations/yandex_alice.rst index 93e72b199e..a74c3b5ba6 100644 --- a/docs/integrations/yandex_alice.rst +++ b/docs/integrations/yandex_alice.rst @@ -14,16 +14,22 @@ a new one -- run: openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj "/CN=MY_DOMAIN_OR_IP" -keyout my.key -out my.crt -Then run: +To run a model specified by the ```` config file as an Alice +skill, run: :: - python -m deeppavlov riseapi --api-mode alice --https --key my.key --cert my.crt [-d] [-p ] - + python -m deeppavlov alice --https --key my.key --cert my.crt [-d] [-p ] * ``-d``: download model specific data before starting the service. -* ``-p ``: sets the port to ````. Overrides default - settings from ``deeppavlov/utils/settings/server_config.json``. + +The command will print the used host and port. Default web service properties +(host, port, model endpoint, GET request arguments, paths to ssl cert and key, +https mode) can be modified via changing +``deeppavlov/utils/settings/server_config.json`` file. ``--https``, ``--key``, +``--cert``, ``-p`` arguments override values from ``server_config.json``. +Advanced API configuration is described in +:doc:`REST API ` section. Now set up and test your dialog (https://dialogs.yandex.ru/developer/). Detailed documentation of the platform could be found on @@ -35,28 +41,19 @@ Python ~~~~~~ To run a model specified by a DeepPavlov config ```` as an Alice -skill, firstly, you have to turn it to a :class:`~deeppavlov.core.skill.skill.Skill` -and then make it an :class:`~deeppavlov.core.agent.agent.Agent`. +skill using python, you have to run following code: .. code:: python - from deeppavlov import build_model - from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill - from deeppavlov.agents.default_agent.default_agent import DefaultAgent - from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector - from deeppavlov.utils.alice import start_agent_server - - model = build_model("", download=True) - - # Step 1: make it a Skill - skill = DefaultStatelessSkill(model) - # Step 2: make it an Agent - agent = DefaultAgent(skills=[skill]) - # Step 3: run server - start_agent_server(agent, host='0.0.0.0', port=7051, endpoint='/agent', ssl_key='my.key', ssl_cert='my.crt') + from deeppavlov.utils.alice import start_alice_server -If your model is already a subclass of :class:`~deeppavlov.core.skill.skill.Skill` -or a subclass of :class:`~deeppavlov.core.agent.agent.Agent` (see -:doc:`skills ` and :doc:`agents `) you can skip -corresponding steps. + start_alice_server(, + host=, + port=, + endpoint=, + https=True, + ssl_key='my.key', + ssl_cert='my.crt') +All arguments except ```` are optional. Optional arguments override +corresponding values from ``deeppavlov/utils/settings/server_config.json``. diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index e4ed9e0158..e1f76b382b 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -17,8 +17,6 @@ Our goal is to enable AI-application developers and researchers with: Key Concepts ------------ -- ``Agent`` is a conversational agent communicating with users in - natural language (text). - ``Skill`` fulfills user’s goal in some domain. Typically, this is accomplished by presenting information or completing transaction (e.g. answer question by FAQ, booking tickets etc.). However, for @@ -33,7 +31,7 @@ Key Concepts end-to-end mode being joined in a chain. - ``Skill Manager`` performs selection of the ``Skill`` to generate response. -- ``Chainer`` builds an agent/model pipeline from heterogeneous +- ``Chainer`` builds a model pipeline from heterogeneous components (Rule-based/ML/DL). It allows to train and infer models in a pipeline as a whole. @@ -49,12 +47,6 @@ difference of a ``Skill`` from a ``Model`` is that its input and output should both be strings. Therefore, ``Skill``\ s are usually associated with dialogue tasks. -``Agent`` is supposed to be a multi-purpose dialogue system that -comprises several ``Skill``\ s and can switch between them. It can be a -dialogue system that contains a goal-oriented and chatbot skills and -chooses which one to use for generating the answer depending on user -input. - DeepPavlov is built on top of machine learning frameworks `TensorFlow `__ and `Keras `__. Other external libraries can be used to diff --git a/examples/README.md b/examples/README.md index e3da3b553d..ec710904aa 100644 --- a/examples/README.md +++ b/examples/README.md @@ -14,8 +14,6 @@ * Optimal learning rate search in DeepPavlov [[notebook]](super_convergence_tutorial.ipynb) [[colab]](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/super_convergence_tutorial.ipynb) -* Integration of FAQ skill with Yandex Alice (RU) [[notebook]](yandex_faq_ru.ipynb) [[colab]](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/yandex_faq_ru.ipynb) - # Links More examples are available: diff --git a/examples/yandex_faq_ru.ipynb b/examples/yandex_faq_ru.ipynb deleted file mode 100644 index 3b87b2eccf..0000000000 --- a/examples/yandex_faq_ru.ipynb +++ /dev/null @@ -1,202 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Подключение **DeepPavlov** чатбота к **Яндекс.Алиса**\n", - "\n", - "В этом туториале я расскажу как разработать и подключить чатбота на основе библиотеки **DeepPavlov** к **Яндекс.Алиса**. Наш бот сможет приветствовать, прощаться и отвечать на вопросы (на основе списка FAQ - часто задаваемых вопросов). Более детальное руководство по созданию autoFAQ ботов на основе библиотеки DeepPavlov вы сможете найти в статье [Simple intent recognition and question answering with DeepPavlov](https://medium.com/deeppavlov)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Установка\n", - "\n", - "Для начала установите Python 3.6 и активируйте виртуальную среду разработки (`source activate py36`). Затем скачайте библиотеку DeepPavlov." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install -q deeppavlov" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Разработка" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Создайте скиллы для приветствия, прощания, и скилл на случай если запрос пользователя не удовлетворяет ни одной из категорий. Параметр responses определяет варианты ответа чатбота на запросы из параметра patters. Если при определении скилла параметр `pattens` не задан, то этот скилл становится скиллом \"заглушкой\", то есть он вызывается в том случае если ни один из скиллов не сработал. Параметр `default_confidence` задает минимальный порог уверенности при котором скилл-заглушка сработает." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from deeppavlov.skills.pattern_matching_skill import PatternMatchingSkill\n", - "\n", - "hello = PatternMatchingSkill(responses=['Привет!', 'Здравствуйте', 'Добрый день'], patterns=['Привет', 'Здравствуйте', 'Добрый день'])\n", - "bye = PatternMatchingSkill(responses=['Пока!', 'До свидания, надеюсь смог вам помочь', 'До встречи!'], patterns=['До свидания', 'Пока', 'Спасибо за помощь'])\n", - "fallback = PatternMatchingSkill(responses=['Я не понял, но могу попробовать ответить на другой вопрос', 'Я не понял, задайте другой вопрос'], default_confidence = 0.5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Создайте объект класса `SimilarityMatchingSkill`, который отвечает на запрос пользователя на основе списка часто задаваемых вопросов. Объект имеет следующие параметры\n", - "* `data_path` - путь к csv файлу с данными\n", - "* `x_col_name` - имя колонки с вопросами в csv файле (Question, по умолчанию)\n", - "* `y_col_name` - имя колонки с ответами в csv файле (Answer, по умолчанию)\n", - "* `edit_dict` - `dict` с параметрами конфигурации\n", - "* `save_load_path` - путь куда сохранить натренированную модель\n", - "* `train` - тренировать ли модель?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from deeppavlov.contrib.skills.similarity_matching_skill import SimilarityMatchingSkill\n", - "\n", - "faq = SimilarityMatchingSkill(data_path = 'http://files.deeppavlov.ai/faq/dataset.csv',\n", - " x_col_name = 'Question', \n", - " y_col_name = 'Answer',\n", - " save_load_path = './model',\n", - " config_type = 'tfidf_autofaq',\n", - " train = True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`SimilarityMatchingSkill` это скилл который позволяет использовать модели классификации интентов пользователя. Подробнее о этих моделях вы сможете узнать из нашего [блога](https://medium.com/deeppavlov).\n", - "Натренировав модель и сохранив в `save_load_path`, вы можете использовать ее указав путь загрузки `faq = SimilarityMatchingSkill(load_path='./model')`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Последний шаг объединить все скиллы в агент, и настроить параметр выбора скилла. Параметр `HighestConfidenceSelector` определяет, что будет вызван скилл с наивысшей уверенностью (confidence)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from deeppavlov.agents.default_agent.default_agent import DefaultAgent\n", - "from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector\n", - "\n", - "agent = DefaultAgent([hello, bye, faq, fallback], skills_selector=HighestConfidenceSelector())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Убедитесь, что агент корректо отвечает на запросы." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(agent(['Привет', 'Мне нужен личный кабинет', 'Пока']))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Далее запустите сервер с указанием пути для запросов `endpoint='faq'` и порта подключения `port=5000`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from deeppavlov.utils.alice import start_agent_server\n", - "\n", - "start_agent_server(agent, host='0.0.0.0', port=5000, endpoint='/faq')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Обратите внимание, что Яндекс.Диалоги в качестве **Webhook URL** требует указывать сервер с внешним IP адресом и доступом по протоколу *https*. \n", - "Для быстрого прототипирования вы можете использовать [ngrok](https://ngrok.com/). **Ngrok** позволит вам создавать туннель для доступа к вашему серверу с **DeepPavlov** в локальной сети, для этого запустите *ngrok http 5000* на вашем сервере с DeepPavlov. В ответ на это будет создано два туннеля, по одному на протоколы http и https.\n", - "Скопируйте адрес туннеля для https, добавьте к линку эндпоинт */faq*, итоговый линк будет **Webhook URL** для нашего Яндекс.Диалога." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Ссылки\n", - "\n", - "[DeepPavlov repository](https://github.com/deepmipt/DeepPavlov)\n", - "\n", - "[DeepPavlov demo page](https://demo.deeppavlov.ai)\n", - "\n", - "[DeepPavlov documentation](https://docs.deeppavlov.ai)\n", - "\n", - "[DeepPavlov blog](https://medium.com/deeppavlov)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "source": [], - "metadata": { - "collapsed": false - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/test_aiml_skill.py b/tests/test_aiml_skill.py index a62049bb63..ffa08a3634 100644 --- a/tests/test_aiml_skill.py +++ b/tests/test_aiml_skill.py @@ -1,10 +1,5 @@ -from pathlib import Path from logging import getLogger -import pytest - -from deeppavlov.agents.default_agent.default_agent import DefaultAgent -from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector from deeppavlov import configs, build_model from deeppavlov.utils.pip_wrapper.pip_wrapper import install_from_config @@ -15,24 +10,24 @@ class TestAIMLSkill: def setup(self): config_ref = configs.skills.aiml_skill install_from_config(config_ref) - aiml_skill = build_model(config_ref, download=True) - self.agent = DefaultAgent([aiml_skill], skills_selector=HighestConfidenceSelector()) + self.aiml_skill = build_model(config_ref, download=True) def test_simple_reaction(self): - user_messages_sequence = ["Hello", - "What s up?", - "Tell me a joke", - "Learn my pants are Red", - "LET DISCUSS MOVIES", - "Comedy movies are nice to watch", - "I LIKE WATCHING COMEDY!", - "Ok, goodbye" + user_messages_sequence = [ + "Hello", + "What s up?", + "Tell me a joke", + "Learn my pants are Red", + "LET DISCUSS MOVIES", + "Comedy movies are nice to watch", + "I LIKE WATCHING COMEDY!", + "Ok, goodbye" ] history_of_responses = [] for each_utt in user_messages_sequence: log.info(f"User says: {each_utt}") - responses_batch = self.agent([each_utt]) + responses_batch, _, _ = self.aiml_skill([each_utt], [None]) log.info(f" Bot says: {responses_batch[0]}") history_of_responses.append(responses_batch) diff --git a/tests/test_dsl_skill.py b/tests/test_dsl_skill.py index b4d4bff049..6a332b44ce 100644 --- a/tests/test_dsl_skill.py +++ b/tests/test_dsl_skill.py @@ -2,7 +2,7 @@ from deeppavlov import configs, build_model from deeppavlov.core.common.file import read_json -from deeppavlov.skills.dsl_skill.dsl_skill import DSLMeta +from deeppavlov.skills.dsl_skill import DSLMeta from deeppavlov.utils.pip_wrapper.pip_wrapper import install_from_config log = getLogger(__name__) @@ -43,7 +43,7 @@ def greeting(context): class TestDSLSkill: def setup(self): - self.skill_config = read_json(configs.dsl_skill.dsl_skill) + self.skill_config = read_json(configs.skills.dsl_skill) install_from_config(self.skill_config) def test_simple_skill(self): diff --git a/tests/test_quick_start.py b/tests/test_quick_start.py index 396798c4a0..59cc944b39 100644 --- a/tests/test_quick_start.py +++ b/tests/test_quick_start.py @@ -23,8 +23,8 @@ from deeppavlov.core.common.paths import get_settings_path from deeppavlov.core.data.utils import get_all_elems_from_json from deeppavlov.download import deep_download -from deeppavlov.utils.server.server import get_server_params, SERVER_CONFIG_FILENAME -from deeppavlov.utils.socket.socket import SOCKET_CONFIG_FILENAME +from deeppavlov.utils.server import get_server_params +from deeppavlov.utils.socket import SOCKET_CONFIG_FILENAME tests_dir = Path(__file__).parent test_configs_path = tests_dir / "deeppavlov" / "configs" @@ -386,9 +386,7 @@ def interact(config_path, model_directory, qr_list=None): @staticmethod def interact_api(config_path): - server_conf_file = get_settings_path() / SERVER_CONFIG_FILENAME - - server_params = get_server_params(server_conf_file, config_path) + server_params = get_server_params(config_path) url_base = 'http://{}:{}'.format(server_params['host'], api_port or server_params['port']) url = urljoin(url_base.replace('http://0.0.0.0:', 'http://127.0.0.1:'), server_params['model_endpoint']) @@ -430,9 +428,9 @@ def interact_api(config_path): @staticmethod def interact_socket(config_path, socket_type): - socket_conf_file = get_settings_path() / SOCKET_CONFIG_FILENAME + socket_config_path = get_settings_path() / SOCKET_CONFIG_FILENAME - socket_params = get_server_params(socket_conf_file, config_path) + socket_params = get_server_params(config_path, socket_config_path) model_args_names = socket_params['model_args_names'] host = socket_params['host'] diff --git a/tests/test_rasa_skill.py b/tests/test_rasa_skill.py index 00e6ed99b0..7235ade7f3 100644 --- a/tests/test_rasa_skill.py +++ b/tests/test_rasa_skill.py @@ -1,7 +1,5 @@ from logging import getLogger -from deeppavlov.agents.default_agent.default_agent import DefaultAgent -from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector from deeppavlov import configs, build_model from deeppavlov.utils.pip_wrapper.pip_wrapper import install_from_config @@ -10,32 +8,26 @@ class TestRASASkill: def setup(self): - # print(configs.aiml_skill) config_ref = configs.skills.rasa_skill install_from_config(config_ref) - # rasa_skill = build_model( - # "/home/alx/Workspace/DeepPavlov/deeppavlov/configs/aiml_skill/rasa_skill.json", - # download=True) - rasa_skill = build_model( - config_ref, - download=True) - self.agent = DefaultAgent([rasa_skill], skills_selector=HighestConfidenceSelector()) + self.rasa_skill = build_model(config_ref, download=True) def test_simple_reaction(self): - user_messages_sequence = ["Hello", - "What can you do?", - "Tell me a joke", - "Learn my pants are Red", - "LET DISCUSS MOVIES", - "Comedy movies are nice to watch", - "I LIKE WATCHING COMEDY!", - "Ok, goodbye" - ] + user_messages_sequence = [ + "Hello", + "What can you do?", + "Tell me a joke", + "Learn my pants are Red", + "LET DISCUSS MOVIES", + "Comedy movies are nice to watch", + "I LIKE WATCHING COMEDY!", + "Ok, goodbye" + ] history_of_responses = [] for each_utt in user_messages_sequence: log.info(f"User says: {each_utt}") - responses_batch = self.agent([each_utt]) + responses_batch, _ = self.rasa_skill([each_utt]) log.info(f" Bot says: {responses_batch[0]}") history_of_responses.append(responses_batch) From 2e6db6fba3cf1fb13aa31b1f7a77f9ccac9c4e74 Mon Sep 17 00:00:00 2001 From: Dilyara Baymurzina Date: Thu, 31 Oct 2019 17:07:24 +0300 Subject: [PATCH 17/28] style: code style fixes (#1046) * fix: automatic code style correction * fix: squad_iterator * fix: delete document_bert_ner_iterator * fix: revert json files to dev version * fix: removed from registry * refactor: fix merge mistakes --- deeppavlov/__init__.py | 2 + deeppavlov/__main__.py | 1 + deeppavlov/core/common/chainer.py | 9 +- deeppavlov/core/common/check_gpu.py | 1 - deeppavlov/core/common/errors.py | 1 + deeppavlov/core/common/metrics_registry.py | 2 + deeppavlov/core/common/params.py | 4 +- deeppavlov/core/common/prints.py | 1 + deeppavlov/core/common/registry.json | 1 - deeppavlov/core/common/registry.py | 1 + .../core/data/data_learning_iterator.py | 1 + deeppavlov/core/data/simple_vocab.py | 7 +- deeppavlov/core/data/utils.py | 2 +- deeppavlov/core/layers/keras_layers.py | 26 +-- .../core/layers/tf_attention_mechanisms.py | 16 +- .../core/layers/tf_csoftmax_attention.py | 56 ++--- deeppavlov/core/layers/tf_layers.py | 18 +- deeppavlov/core/models/component.py | 2 +- deeppavlov/core/models/estimator.py | 1 + deeppavlov/core/models/keras_model.py | 8 +- deeppavlov/core/models/lr_scheduled_model.py | 20 +- deeppavlov/core/models/nn_model.py | 1 + deeppavlov/core/models/serializable.py | 4 +- deeppavlov/core/models/tf_backend.py | 4 + deeppavlov/core/models/tf_model.py | 8 +- deeppavlov/core/trainers/fit_trainer.py | 1 + deeppavlov/core/trainers/nn_trainer.py | 9 +- .../basic_classification_iterator.py | 3 +- .../dataset_iterators/dialog_iterator.py | 1 + .../document_bert_ner_iterator.py | 211 ------------------ .../dstc2_intents_iterator.py | 1 + .../dataset_iterators/dstc2_ner_iterator.py | 6 +- .../elmo_file_paths_iterator.py | 10 +- .../dataset_iterators/file_paths_iterator.py | 4 +- .../kvret_dialog_iterator.py | 13 +- .../morphotagger_iterator.py | 3 +- .../ner_few_shot_iterator.py | 1 + .../dataset_iterators/siamese_iterator.py | 2 +- .../snips_intents_iterator.py | 2 - .../dataset_iterators/squad_iterator.py | 11 +- .../dataset_iterators/typos_iterator.py | 3 +- .../amazon_ecommerce_reader.py | 5 +- .../basic_classification_reader.py | 4 +- .../dataset_readers/conll2003_reader.py | 7 +- deeppavlov/dataset_readers/dstc2_reader.py | 4 +- .../dataset_readers/file_paths_reader.py | 2 +- deeppavlov/dataset_readers/kvret_reader.py | 17 +- deeppavlov/dataset_readers/line_reader.py | 2 +- .../morphotagging_dataset_reader.py | 7 +- .../paraphraser_pretrain_reader.py | 3 - .../dataset_readers/paraphraser_reader.py | 2 +- deeppavlov/dataset_readers/snips_reader.py | 2 +- deeppavlov/dataset_readers/sq_reader.py | 4 +- .../dataset_readers/squad_dataset_reader.py | 2 +- deeppavlov/dataset_readers/typos_reader.py | 2 + .../dataset_readers/ubuntu_dstc7_mt_reader.py | 18 +- .../dataset_readers/ubuntu_v1_mt_reader.py | 3 +- .../dataset_readers/ubuntu_v2_mt_reader.py | 6 +- .../dataset_readers/ubuntu_v2_reader.py | 4 +- deeppavlov/deep.py | 2 +- deeppavlov/deprecated/agent/agent.py | 2 + deeppavlov/deprecated/agent/filter.py | 1 + deeppavlov/deprecated/agent/processor.py | 1 + deeppavlov/deprecated/agent/rich_content.py | 2 + .../agents/default_agent/default_agent.py | 1 + .../agents/ecommerce_agent/ecommerce_agent.py | 9 +- .../agents/filters/transparent_filter.py | 1 + .../default_rich_content_processor.py | 1 + .../processors/highest_confidence_selector.py | 1 + .../agents/processors/random_selector.py | 1 + .../rich_content/default_rich_content.py | 7 +- deeppavlov/deprecated/skill/skill.py | 3 +- .../skills/default_skill/default_skill.py | 5 +- .../skills/ecommerce_skill/bleu_retrieve.py | 21 +- .../skills/ecommerce_skill/tfidf_retrieve.py | 18 +- .../pattern_matching_skill.py | 14 +- deeppavlov/download.py | 9 +- deeppavlov/evolve.py | 2 +- deeppavlov/metrics/bleu.py | 6 +- deeppavlov/metrics/elmo_metrics.py | 1 - deeppavlov/metrics/fmeasure.py | 34 +-- deeppavlov/metrics/google_bleu.py | 146 ++++++------ deeppavlov/metrics/recall_at_k.py | 2 + .../models/api_requester/api_requester.py | 3 +- deeppavlov/models/api_requester/api_router.py | 2 +- deeppavlov/models/bert/bert_classifier.py | 3 +- deeppavlov/models/bert/bert_ranker.py | 13 +- deeppavlov/models/bert/bert_squad.py | 6 +- .../classifiers/keras_classification_model.py | 14 +- .../classifiers/ru_obscenity_classifier.py | 22 +- deeppavlov/models/classifiers/utils.py | 6 +- deeppavlov/models/elmo/bilm_model.py | 5 +- deeppavlov/models/elmo/elmo.py | 24 +- deeppavlov/models/elmo/elmo2tfhub.py | 6 +- deeppavlov/models/elmo/elmo_model.py | 32 +-- .../models/embedders/abstract_embedder.py | 1 + deeppavlov/models/embedders/elmo_embedder.py | 3 +- .../embedders/tfidf_weighted_embedder.py | 4 +- deeppavlov/models/go_bot/network.py | 16 +- deeppavlov/models/go_bot/templates.py | 8 +- deeppavlov/models/go_bot/tracker.py | 6 +- deeppavlov/models/kbqa/entity_linking.py | 21 +- .../models/kbqa/kb_answer_parser_wikidata.py | 9 +- deeppavlov/models/morpho_tagger/cells.py | 5 +- deeppavlov/models/morpho_tagger/common.py | 2 +- .../models/morpho_tagger/common_tagger.py | 1 + deeppavlov/models/morpho_tagger/lemmatizer.py | 3 +- .../models/morpho_tagger/morpho_tagger.py | 29 +-- deeppavlov/models/ner/bio.py | 1 + deeppavlov/models/ner/svm.py | 1 + .../models/preprocessors/bert_preprocessor.py | 1 - .../models/preprocessors/capitalization.py | 1 + .../models/preprocessors/char_splitter.py | 1 + .../dirty_comments_preprocessor.py | 1 + deeppavlov/models/preprocessors/mask.py | 1 + deeppavlov/models/preprocessors/one_hotter.py | 1 + .../preprocessors/random_embeddings_matrix.py | 1 + .../preprocessors/russian_lemmatizer.py | 1 + deeppavlov/models/preprocessors/sanitizer.py | 1 + .../preprocessors/siamese_preprocessor.py | 4 +- .../preprocessors/squad_preprocessor.py | 16 +- .../preprocessors/str_token_reverser.py | 5 +- .../models/preprocessors/str_utf8_encoder.py | 15 +- .../ranking/bilstm_gru_siamese_network.py | 8 +- .../models/ranking/bilstm_siamese_network.py | 19 +- .../deep_attention_matching_network.py | 2 +- ...ention_matching_network_use_transformer.py | 21 +- .../models/ranking/keras_siamese_model.py | 7 +- .../matching_models/dam_utils/layers.py | 145 ++++++------ .../matching_models/dam_utils/operations.py | 71 +++--- deeppavlov/models/ranking/metrics.py | 5 +- .../models/ranking/mpm_siamese_network.py | 9 +- .../ranking/sequential_matching_network.py | 1 - deeppavlov/models/ranking/siamese_model.py | 3 +- .../models/ranking/siamese_predictor.py | 13 +- .../models/ranking/tf_base_matching_model.py | 11 +- deeppavlov/models/seq2seq_go_bot/bot.py | 13 +- .../models/seq2seq_go_bot/dialog_state.py | 1 - deeppavlov/models/seq2seq_go_bot/kb.py | 10 +- .../models/seq2seq_go_bot/kb_attn_layer.py | 12 +- deeppavlov/models/seq2seq_go_bot/network.py | 42 ++-- .../models/sklearn/sklearn_component.py | 1 + deeppavlov/models/slotfill/slotfill.py | 1 + deeppavlov/models/slotfill/slotfill_raw.py | 1 + .../brillmoore/error_model.py | 2 +- .../electors/kenlm_elector.py | 6 +- .../levenshtein/levenshtein_searcher.py | 58 ++--- .../levenshtein/tabled_trie.py | 25 +-- deeppavlov/models/squad/squad.py | 5 +- deeppavlov/models/squad/utils.py | 4 +- .../models/tokenizers/lazy_tokenizer.py | 1 + .../models/tokenizers/nltk_moses_tokenizer.py | 2 +- .../models/tokenizers/nltk_tokenizer.py | 1 + .../models/tokenizers/ru_sent_tokenizer.py | 2 +- deeppavlov/models/tokenizers/ru_tokenizer.py | 8 +- .../models/tokenizers/spacy_tokenizer.py | 2 +- .../models/tokenizers/split_tokenizer.py | 1 + .../models/vectorizers/word_vectorizer.py | 4 +- deeppavlov/skills/aiml_skill/aiml_skill.py | 2 + deeppavlov/skills/dsl_skill/context.py | 3 +- deeppavlov/utils/pip_wrapper/__init__.py | 2 +- deeppavlov/utils/pip_wrapper/pip_wrapper.py | 4 +- deeppavlov/utils/server/server.py | 5 +- deeppavlov/utils/settings/log_config.json | 2 +- deeppavlov/utils/settings/socket_config.json | 2 +- deeppavlov/vocabs/typos.py | 9 +- setup.py | 20 +- tests/test_quick_start.py | 36 +-- tests/test_tf_layers.py | 1 - utils/prepare/hashes.py | 6 +- utils/prepare/registry.py | 2 +- 171 files changed, 815 insertions(+), 953 deletions(-) delete mode 100644 deeppavlov/dataset_iterators/document_bert_ner_iterator.py diff --git a/deeppavlov/__init__.py b/deeppavlov/__init__.py index 6765198e9f..277241fef9 100644 --- a/deeppavlov/__init__.py +++ b/deeppavlov/__init__.py @@ -26,11 +26,13 @@ from .download import deep_download from .core.common.chainer import Chainer + # TODO: make better def train_model(config: [str, Path, dict], download: bool = False, recursive: bool = False) -> Chainer: train_evaluate_model_from_config(config, download=download, recursive=recursive) return build_model(config, load_trained=True) + def evaluate_model(config: [str, Path, dict], download: bool = False, recursive: bool = False) -> dict: return train_evaluate_model_from_config(config, to_train=False, download=download, recursive=recursive) diff --git a/deeppavlov/__main__.py b/deeppavlov/__main__.py index 9a34bb3f2f..d5d8c0580d 100644 --- a/deeppavlov/__main__.py +++ b/deeppavlov/__main__.py @@ -1,3 +1,4 @@ if __name__ == '__main__': from .deep import main + main() diff --git a/deeppavlov/core/common/chainer.py b/deeppavlov/core/common/chainer.py index 26f157d96f..8aef21dc6c 100644 --- a/deeppavlov/core/common/chainer.py +++ b/deeppavlov/core/common/chainer.py @@ -46,6 +46,7 @@ class Chainer(Component): out_params: names of pipeline inference outputs in_y: names of additional inputs for pipeline training and evaluation modes """ + def __init__(self, in_x: Union[str, list] = None, out_params: Union[str, list] = None, in_y: Union[str, list] = None, *args, **kwargs) -> None: self.pipe: List[Tuple[Tuple[List[str], List[str]], List[str], Component]] = [] @@ -150,9 +151,9 @@ def append(self, component: Union[Component, FunctionType], in_x: [str, list, di component: NNModel main = True - assert self.train_map.issuperset(in_x+in_y), ('Arguments {} are expected but only {} are set' - .format(in_x+in_y, self.train_map)) - preprocessor = Chainer(self.in_x, in_x+in_y, self.in_y) + assert self.train_map.issuperset(in_x + in_y), ('Arguments {} are expected but only {} are set' + .format(in_x + in_y, self.train_map)) + preprocessor = Chainer(self.in_x, in_x + in_y, self.in_y) for (t_in_x_keys, t_in_x), t_out, t_component in self.train_pipe: if t_in_x_keys: t_in_x = dict(zip(t_in_x_keys, t_in_x)) @@ -160,7 +161,7 @@ def append(self, component: Union[Component, FunctionType], in_x: [str, list, di def train_on_batch(*args, **kwargs): preprocessed = preprocessor.compute(*args, **kwargs) - if len(in_x+in_y) == 1: + if len(in_x + in_y) == 1: preprocessed = [preprocessed] if keys: return component.train_on_batch(**dict(zip(keys, preprocessed))) diff --git a/deeppavlov/core/common/check_gpu.py b/deeppavlov/core/common/check_gpu.py index 56817b3ab7..d768417785 100644 --- a/deeppavlov/core/common/check_gpu.py +++ b/deeppavlov/core/common/check_gpu.py @@ -19,7 +19,6 @@ log = getLogger(__name__) - _gpu_available = None diff --git a/deeppavlov/core/common/errors.py b/deeppavlov/core/common/errors.py index d5d4ce23e2..1cef661aa9 100644 --- a/deeppavlov/core/common/errors.py +++ b/deeppavlov/core/common/errors.py @@ -19,6 +19,7 @@ class ConfigError(Exception): """Any configuration error.""" + def __init__(self, message): super(ConfigError, self).__init__() self.message = message diff --git a/deeppavlov/core/common/metrics_registry.py b/deeppavlov/core/common/metrics_registry.py index 78fa53f005..28bf3a6e1b 100644 --- a/deeppavlov/core/common/metrics_registry.py +++ b/deeppavlov/core/common/metrics_registry.py @@ -29,6 +29,7 @@ def fn_from_str(name: str) -> Callable[..., Any]: def register_metric(metric_name: str) -> Callable[..., Any]: """Decorator for metric registration.""" + def decorate(fn): fn_name = fn.__module__ + ':' + fn.__name__ if metric_name in _REGISTRY and _REGISTRY[metric_name] != fn_name: @@ -36,6 +37,7 @@ def decorate(fn): .format(metric_name)) _REGISTRY[metric_name] = fn_name return fn + return decorate diff --git a/deeppavlov/core/common/params.py b/deeppavlov/core/common/params.py index 8e9afbd1a2..e74bb594a7 100644 --- a/deeppavlov/core/common/params.py +++ b/deeppavlov/core/common/params.py @@ -82,7 +82,7 @@ def from_params(params: Dict, mode: str = 'infer', serialized: Any = None, **kwa _refs.clear() _refs.update(refs) try: - _refs[config_params['id']] = model + _refs[config_params['id']] = model except KeyError: pass return model @@ -100,7 +100,7 @@ def from_params(params: Dict, mode: str = 'infer', serialized: Any = None, **kwa try: spec = inspect.getfullargspec(obj) - if 'mode' in spec.args+spec.kwonlyargs or spec.varkw is not None: + if 'mode' in spec.args + spec.kwonlyargs or spec.varkw is not None: kwargs['mode'] = mode component = obj(**dict(config_params, **kwargs)) diff --git a/deeppavlov/core/common/prints.py b/deeppavlov/core/common/prints.py index 7f824d2b19..28360f0e51 100644 --- a/deeppavlov/core/common/prints.py +++ b/deeppavlov/core/common/prints.py @@ -18,5 +18,6 @@ class RedirectedPrints(redirect_stdout): """Context manager for temporarily redirecting stdout to another stream """ + def __init__(self, new_target=sys.stderr): super().__init__(new_target=new_target) diff --git a/deeppavlov/core/common/registry.json b/deeppavlov/core/common/registry.json index 49e94c6ed4..a45f35af92 100644 --- a/deeppavlov/core/common/registry.json +++ b/deeppavlov/core/common/registry.json @@ -33,7 +33,6 @@ "dialog_state": "deeppavlov.models.seq2seq_go_bot.dialog_state:DialogState", "dictionary_vectorizer": "deeppavlov.models.vectorizers.word_vectorizer:DictionaryVectorizer", "dirty_comments_preprocessor": "deeppavlov.models.preprocessors.dirty_comments_preprocessor:DirtyCommentsPreprocessor", - "document_bert_ner_iterator": "deeppavlov.dataset_iterators.document_bert_ner_iterator:DocumentBertNerIterator", "document_chunker": "deeppavlov.models.preprocessors.odqa_preprocessors:DocumentChunker", "dstc2_intents_iterator": "deeppavlov.dataset_iterators.dstc2_intents_iterator:Dstc2IntentsDatasetIterator", "dstc2_ner_iterator": "deeppavlov.dataset_iterators.dstc2_ner_iterator:Dstc2NerDatasetIterator", diff --git a/deeppavlov/core/common/registry.py b/deeppavlov/core/common/registry.py index 048bf0b1bd..932c4da714 100644 --- a/deeppavlov/core/common/registry.py +++ b/deeppavlov/core/common/registry.py @@ -45,6 +45,7 @@ def register(name: str = None) -> type: Register classes that could be initialized from JSON configuration file. If name is not passed, the class name is converted to snake-case. """ + def decorate(model_cls: type, reg_name: str = None) -> type: model_name = reg_name or short_name(model_cls) global _REGISTRY diff --git a/deeppavlov/core/data/data_learning_iterator.py b/deeppavlov/core/data/data_learning_iterator.py index d2ee6af42c..26fb83ba50 100644 --- a/deeppavlov/core/data/data_learning_iterator.py +++ b/deeppavlov/core/data/data_learning_iterator.py @@ -31,6 +31,7 @@ class DataLearningIterator: shuffle: whether to shuffle data during batching random: instance of ``Random`` initialized with a seed """ + def split(self, *args, **kwargs): """ Manipulate self.train, self.valid, and self.test into their final form. """ pass diff --git a/deeppavlov/core/data/simple_vocab.py b/deeppavlov/core/data/simple_vocab.py index fde9433079..8332d4d680 100644 --- a/deeppavlov/core/data/simple_vocab.py +++ b/deeppavlov/core/data/simple_vocab.py @@ -40,9 +40,10 @@ class SimpleVocabulary(Estimator): unk_token: label assigned to unknown tokens. freq_drop_load: if True, then frequencies of tokens are set to min_freq on the model load. """ + def __init__(self, special_tokens: Tuple[str, ...] = tuple(), - max_tokens: int = 2**30, + max_tokens: int = 2 ** 30, min_freq: int = 0, pad_with_zeros: bool = False, unk_token: Optional[str] = None, @@ -118,7 +119,7 @@ def load(self): self._add_tokens_with_freqs(tokens, counts) elif not self.load_path.parent.is_dir(): raise ConfigError("Provided `load_path` for {} doesn't exist!".format( - self.__class__.__name__)) + self.__class__.__name__)) else: raise ConfigError("`load_path` for {} is not provided!".format(self)) @@ -135,7 +136,7 @@ def load_line(self, ln): else: token, cnt = ln.split('\t', 1) return token, cnt - + @property def len(self): return len(self) diff --git a/deeppavlov/core/data/utils.py b/deeppavlov/core/data/utils.py index e1b3cf29e7..5289779165 100644 --- a/deeppavlov/core/data/utils.py +++ b/deeppavlov/core/data/utils.py @@ -279,7 +279,7 @@ def _copytree(src: Path, dest: Path) -> None: shutil.copy(str(f), str(f_dest)) -def file_md5(fpath: Union[str, Path], chunk_size: int = 2**16) -> Optional[str]: +def file_md5(fpath: Union[str, Path], chunk_size: int = 2 ** 16) -> Optional[str]: """Return md5 hash value for file contents. Args: diff --git a/deeppavlov/core/layers/keras_layers.py b/deeppavlov/core/layers/keras_layers.py index eb0dd60b85..7c1d379fee 100644 --- a/deeppavlov/core/layers/keras_layers.py +++ b/deeppavlov/core/layers/keras_layers.py @@ -33,7 +33,7 @@ def expand_tile(units, axis): repetitions = [1, 1, 1, 1] repetitions[axis] = n_time_steps if axis == 1: - expanded = Reshape(target_shape=( (1,) + K.int_shape(units)[1:] ))(units) + expanded = Reshape(target_shape=((1,) + K.int_shape(units)[1:]))(units) else: expanded = Reshape(target_shape=(K.int_shape(units)[1:2] + (1,) + K.int_shape(units)[2:]))(units) return K.tile(expanded, repetitions) @@ -113,9 +113,9 @@ def build(self, input_shape): self.W = [] for i in range(self.output_dim): self.W.append(self.add_weight(name='kernel', - shape=(1, input_shape[0][-1]), - initializer='uniform', - trainable=True)) + shape=(1, input_shape[0][-1]), + initializer='uniform', + trainable=True)) super(FullMatchingLayer, self).build(input_shape) # Be sure to call this at the end def call(self, x): @@ -153,9 +153,9 @@ def build(self, input_shape): self.W = [] for i in range(self.output_dim): self.W.append(self.add_weight(name='kernel', - shape=(1, input_shape[0][-1]), - initializer='uniform', - trainable=True)) + shape=(1, input_shape[0][-1]), + initializer='uniform', + trainable=True)) super(MaxpoolingMatchingLayer, self).build(input_shape) # Be sure to call this at the end def call(self, x): @@ -193,9 +193,9 @@ def build(self, input_shape): self.W = [] for i in range(self.output_dim): self.W.append(self.add_weight(name='kernel', - shape=(1, input_shape[0][-1]), - initializer='uniform', - trainable=True)) + shape=(1, input_shape[0][-1]), + initializer='uniform', + trainable=True)) super(AttentiveMatchingLayer, self).build(input_shape) # Be sure to call this at the end def call(self, x): @@ -241,9 +241,9 @@ def build(self, input_shape): self.W = [] for i in range(self.output_dim): self.W.append(self.add_weight(name='kernel', - shape=(1, input_shape[0][-1]), - initializer='uniform', - trainable=True)) + shape=(1, input_shape[0][-1]), + initializer='uniform', + trainable=True)) super(MaxattentiveMatchingLayer, self).build(input_shape) # Be sure to call this at the end def call(self, x): diff --git a/deeppavlov/core/layers/tf_attention_mechanisms.py b/deeppavlov/core/layers/tf_attention_mechanisms.py index cf7460290d..c75f6f3282 100644 --- a/deeppavlov/core/layers/tf_attention_mechanisms.py +++ b/deeppavlov/core/layers/tf_attention_mechanisms.py @@ -47,8 +47,8 @@ def general_attention(key, context, hidden_size, projected_align=False): tf.layers.dense(key, hidden_size, kernel_initializer=xav()) r_projected_key = tf.reshape(projected_key, shape=[-1, hidden_size, 1]) - lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2) - lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2) + lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size // 2) + lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size // 2) (output_fw, output_bw), states = \ tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell, cell_bw=lstm_bw_cell, @@ -139,8 +139,8 @@ def cs_general_attention(key, context, hidden_size, depth, projected_align=False kernel_initializer=xav(), name='projected_context') - lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2) - lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2) + lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size // 2) + lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size // 2) (output_fw, output_bw), states = \ tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell, cell_bw=lstm_bw_cell, @@ -192,8 +192,8 @@ def bahdanau_attention(key, context, hidden_size, projected_align=False): tf.tile(tf.reshape(projected_key, shape=[-1, 1, hidden_size]), [1, max_num_tokens, 1]) - lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2) - lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2) + lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size // 2) + lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size // 2) (output_fw, output_bw), states = \ tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell, cell_bw=lstm_bw_cell, @@ -308,8 +308,8 @@ def cs_bahdanau_attention(key, context, hidden_size, depth, projected_align=Fals tf.tile(tf.reshape(projected_key, shape=[-1, 1, hidden_size]), [1, max_num_tokens, 1]) - lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2) - lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2) + lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size // 2) + lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size // 2) (output_fw, output_bw), states = \ tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell, cell_bw=lstm_bw_cell, diff --git a/deeppavlov/core/layers/tf_csoftmax_attention.py b/deeppavlov/core/layers/tf_csoftmax_attention.py index f73bcc3079..764a727dc2 100644 --- a/deeppavlov/core/layers/tf_csoftmax_attention.py +++ b/deeppavlov/core/layers/tf_csoftmax_attention.py @@ -114,28 +114,27 @@ def attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketch, key attn_alignment_dims = hidden_for_attn_alignment.get_shape().as_list() attn_alignment_hidden_size = attn_alignment_dims[2] - repeated_sketch = tf.tile(tf.reshape(sketch, [-1, 1, hidden_size]), (1,num_tokens, 1)) - concat_mem = tf.concat([hidden_for_sketch, repeated_sketch],-1) + repeated_sketch = tf.tile(tf.reshape(sketch, [-1, 1, hidden_size]), (1, num_tokens, 1)) + concat_mem = tf.concat([hidden_for_sketch, repeated_sketch], -1) - - concat_mem = tf.reshape(concat_mem, [-1, num_tokens, 2*hidden_size]) # dirty trick + concat_mem = tf.reshape(concat_mem, [-1, num_tokens, 2 * hidden_size]) # dirty trick reduce_mem = tf.layers.dense(concat_mem, hidden_size) projected_key = tf.layers.dense(key, hidden_size) - t_key = tf.reshape(projected_key,[-1, hidden_size, 1]) + t_key = tf.reshape(projected_key, [-1, hidden_size, 1]) score = tf.reshape(tf.matmul(reduce_mem, t_key), [-1, num_tokens]) inv_cum_att = tf.reshape(tf.ones_like(cum_att) - cum_att, [-1, num_tokens]) att = csoftmax(score, inv_cum_att) - t_reduce_mem = tf.transpose(reduce_mem, [0,2,1]) - t_hidden_for_attn_alignment = tf.transpose(hidden_for_attn_alignment, [0,2,1]) + t_reduce_mem = tf.transpose(reduce_mem, [0, 2, 1]) + t_hidden_for_attn_alignment = tf.transpose(hidden_for_attn_alignment, [0, 2, 1]) r_att = tf.reshape(att, [-1, num_tokens, 1]) - next_sketch = tf.squeeze(tf.matmul(t_reduce_mem,r_att),-1) - aligned_hidden_sketch = tf.squeeze(tf.matmul(t_hidden_for_attn_alignment,r_att),-1) + next_sketch = tf.squeeze(tf.matmul(t_reduce_mem, r_att), -1) + aligned_hidden_sketch = tf.squeeze(tf.matmul(t_hidden_for_attn_alignment, r_att), -1) return next_sketch, att, aligned_hidden_sketch @@ -165,11 +164,13 @@ def attention_gen_block(hidden_for_sketch, hidden_for_attn_alignment, key, atten aligned_hiddens = [] cum_att = tf.zeros(shape=[batch_size, num_tokens]) # cumulative attention for i in range(attention_depth): - sketch, cum_att_, aligned_hidden = attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketches[-1], key, cum_att) - sketches.append(sketch) #sketch - aligned_hiddens.append(aligned_hidden) #sketch + sketch, cum_att_, aligned_hidden = attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, + sketches[-1], key, cum_att) + sketches.append(sketch) # sketch + aligned_hiddens.append(aligned_hidden) # sketch cum_att += cum_att_ - final_aligned_hiddens = tf.reshape(tf.transpose(tf.stack(aligned_hiddens), [1, 0, 2]),[1, attention_depth, attn_alignment_hidden_size]) + final_aligned_hiddens = tf.reshape(tf.transpose(tf.stack(aligned_hiddens), [1, 0, 2]), + [1, attention_depth, attn_alignment_hidden_size]) return final_aligned_hiddens @@ -197,25 +198,24 @@ def attention_bah_step(hidden_for_sketch, hidden_for_attn_alignment, sketch, cum attn_alignment_dims = hidden_for_attn_alignment.get_shape().as_list() attn_alignment_hidden_size = attn_alignment_dims[2] - repeated_sketch = tf.tile(tf.reshape(sketch, [-1, 1, hidden_size]), (1,num_tokens, 1)) - concat_mem = tf.concat([hidden_for_sketch, repeated_sketch],-1) - + repeated_sketch = tf.tile(tf.reshape(sketch, [-1, 1, hidden_size]), (1, num_tokens, 1)) + concat_mem = tf.concat([hidden_for_sketch, repeated_sketch], -1) - concat_mem = tf.reshape(concat_mem, [-1, num_tokens, 2*hidden_size]) # dirty trick + concat_mem = tf.reshape(concat_mem, [-1, num_tokens, 2 * hidden_size]) # dirty trick reduce_mem = tf.layers.dense(concat_mem, hidden_size) - score = tf.squeeze(tf.layers.dense(reduce_mem, units = 1, - use_bias=False),-1) + score = tf.squeeze(tf.layers.dense(reduce_mem, units=1, + use_bias=False), -1) inv_cum_att = tf.reshape(tf.ones_like(cum_att) - cum_att, [-1, num_tokens]) att = csoftmax(score, inv_cum_att) - t_reduce_mem = tf.transpose(reduce_mem, [0,2,1]) - t_hidden_for_attn_alignment = tf.transpose(hidden_for_attn_alignment, [0,2,1]) + t_reduce_mem = tf.transpose(reduce_mem, [0, 2, 1]) + t_hidden_for_attn_alignment = tf.transpose(hidden_for_attn_alignment, [0, 2, 1]) r_att = tf.reshape(att, [-1, num_tokens, 1]) - next_sketch = tf.squeeze(tf.matmul(t_reduce_mem,r_att),-1) - aligned_hidden_sketch = tf.squeeze(tf.matmul(t_hidden_for_attn_alignment,r_att),-1) + next_sketch = tf.squeeze(tf.matmul(t_reduce_mem, r_att), -1) + aligned_hidden_sketch = tf.squeeze(tf.matmul(t_hidden_for_attn_alignment, r_att), -1) return next_sketch, att, aligned_hidden_sketch @@ -245,9 +245,11 @@ def attention_bah_block(hidden_for_sketch, hidden_for_attn_alignment, attention_ aligned_hiddens = [] cum_att = tf.zeros(shape=[batch_size, num_tokens]) # cumulative attention for i in range(attention_depth): - sketch, cum_att_, aligned_hidden = attention_bah_step(hidden_for_sketch, hidden_for_attn_alignment, sketches[-1], cum_att) - sketches.append(sketch) #sketch - aligned_hiddens.append(aligned_hidden) #sketch + sketch, cum_att_, aligned_hidden = attention_bah_step(hidden_for_sketch, hidden_for_attn_alignment, + sketches[-1], cum_att) + sketches.append(sketch) # sketch + aligned_hiddens.append(aligned_hidden) # sketch cum_att += cum_att_ - final_aligned_hiddens = tf.reshape(tf.transpose(tf.stack(aligned_hiddens), [1, 0, 2]),[1, attention_depth, attn_alignment_hidden_size]) + final_aligned_hiddens = tf.reshape(tf.transpose(tf.stack(aligned_hiddens), [1, 0, 2]), + [1, attention_depth, attn_alignment_hidden_size]) return final_aligned_hiddens diff --git a/deeppavlov/core/layers/tf_layers.py b/deeppavlov/core/layers/tf_layers.py index 7cb8298fb8..ee41a50725 100644 --- a/deeppavlov/core/layers/tf_layers.py +++ b/deeppavlov/core/layers/tf_layers.py @@ -22,8 +22,9 @@ log = getLogger(__name__) - INITIALIZER = tf.orthogonal_initializer + + # INITIALIZER = xavier_initializer @@ -537,13 +538,13 @@ def cudnn_gru(units, n_hidden, n_layers=1, trainable_initial_states=False, initial_h = input_initial_h or init_h - h, h_last = gru(tf.transpose(units, (1, 0, 2)), (initial_h, )) + h, h_last = gru(tf.transpose(units, (1, 0, 2)), (initial_h,)) h = tf.transpose(h, (1, 0, 2)) h_last = tf.squeeze(h_last, axis=0)[-1] # extract last layer state # Extract last states if they are provided if seq_lengths is not None: - indices = tf.stack([tf.range(tf.shape(h)[0]), seq_lengths-1], axis=1) + indices = tf.stack([tf.range(tf.shape(h)[0]), seq_lengths - 1], axis=1) h_last = tf.gather_nd(h, indices) return h, h_last @@ -586,6 +587,7 @@ def cudnn_compatible_gru(units, n_hidden, n_layers=1, trainable_initial_states=F with tf.variable_scope('cudnn_gru', reuse=reuse): def single_cell(): return tf.contrib.cudnn_rnn.CudnnCompatibleGRUCell(n_hidden) + cell = tf.nn.rnn_cell.MultiRNNCell([single_cell() for _ in range(n_layers)]) units = tf.transpose(units, (1, 0, 2)) @@ -598,7 +600,7 @@ def single_cell(): return tf.contrib.cudnn_rnn.CudnnCompatibleGRUCell(n_hidden) # Extract last states if they are provided if seq_lengths is not None: - indices = tf.stack([tf.range(tf.shape(h)[0]), seq_lengths-1], axis=1) + indices = tf.stack([tf.range(tf.shape(h)[0]), seq_lengths - 1], axis=1) h_last = tf.gather_nd(h, indices) return h, h_last @@ -606,7 +608,6 @@ def single_cell(): return tf.contrib.cudnn_rnn.CudnnCompatibleGRUCell(n_hidden) def cudnn_gru_wrapper(units, n_hidden, n_layers=1, trainable_initial_states=False, seq_lengths=None, input_initial_h=None, name='cudnn_gru', reuse=False): - if check_gpu_existence(): return cudnn_gru(units, n_hidden, n_layers, trainable_initial_states, seq_lengths, input_initial_h, name, reuse) @@ -672,7 +673,7 @@ def cudnn_lstm(units, n_hidden, n_layers=1, trainable_initial_states=None, seq_l # Extract last states if they are provided if seq_lengths is not None: - indices = tf.stack([tf.range(tf.shape(h)[0]), seq_lengths-1], axis=1) + indices = tf.stack([tf.range(tf.shape(h)[0]), seq_lengths - 1], axis=1) h_last = tf.gather_nd(h, indices) return h, (h_last, c_last) @@ -740,7 +741,7 @@ def single_cell(): return tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell(n_hidden) # Extract last states if they are provided if seq_lengths is not None: - indices = tf.stack([tf.range(tf.shape(h)[0]), seq_lengths-1], axis=1) + indices = tf.stack([tf.range(tf.shape(h)[0]), seq_lengths - 1], axis=1) h_last = tf.gather_nd(h, indices) return h, (h_last, c_last) @@ -748,7 +749,6 @@ def single_cell(): return tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell(n_hidden) def cudnn_lstm_wrapper(units, n_hidden, n_layers=1, trainable_initial_states=None, seq_lengths=None, initial_h=None, initial_c=None, name='cudnn_lstm', reuse=False): - if check_gpu_existence(): return cudnn_lstm(units, n_hidden, n_layers, trainable_initial_states, seq_lengths, initial_h, initial_c, name, reuse) @@ -945,4 +945,4 @@ def variational_dropout(units, keep_prob, fixed_mask_dims=(1,)): noise_shape = [units_shape[n] for n in range(len(units.shape))] for dim in fixed_mask_dims: noise_shape[dim] = 1 - return tf.nn.dropout(units, rate=1-keep_prob, noise_shape=noise_shape) + return tf.nn.dropout(units, rate=1 - keep_prob, noise_shape=noise_shape) diff --git a/deeppavlov/core/models/component.py b/deeppavlov/core/models/component.py index 8e6f599293..c8b8f1886f 100644 --- a/deeppavlov/core/models/component.py +++ b/deeppavlov/core/models/component.py @@ -16,12 +16,12 @@ from logging import getLogger - log = getLogger(__name__) class Component(metaclass=ABCMeta): """Abstract class for all callables that could be used in Chainer's pipe.""" + @abstractmethod def __call__(self, *args, **kwargs): pass diff --git a/deeppavlov/core/models/estimator.py b/deeppavlov/core/models/estimator.py index 9cccd305d5..ddefc63abb 100644 --- a/deeppavlov/core/models/estimator.py +++ b/deeppavlov/core/models/estimator.py @@ -20,6 +20,7 @@ class Estimator(Component, Serializable): """Abstract class for components that could be fitted on the data as a whole.""" + @abstractmethod def fit(self, *args, **kwargs): pass diff --git a/deeppavlov/core/models/keras_model.py b/deeppavlov/core/models/keras_model.py index 5cf941159d..7c4bedf276 100644 --- a/deeppavlov/core/models/keras_model.py +++ b/deeppavlov/core/models/keras_model.py @@ -12,21 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import inspect from abc import abstractmethod -from copy import deepcopy from logging import getLogger -from typing import Optional, List, Union -import numpy as np import tensorflow as tf from keras import backend as K from overrides import overrides +from deeppavlov.core.models.lr_scheduled_model import LRScheduledModel from deeppavlov.core.models.nn_model import NNModel from deeppavlov.core.models.tf_backend import TfModelMeta -from deeppavlov.core.models.lr_scheduled_model import LRScheduledModel - log = getLogger(__name__) @@ -101,6 +96,7 @@ class LRScheduledKerasModel(LRScheduledModel, KerasModel): KerasModel enhanced with optimizer, learning rate and momentum management and search. """ + def __init__(self, **kwargs): """ Initialize model with given parameters diff --git a/deeppavlov/core/models/lr_scheduled_model.py b/deeppavlov/core/models/lr_scheduled_model.py index e3c68c6a40..6d2e5a9637 100644 --- a/deeppavlov/core/models/lr_scheduled_model.py +++ b/deeppavlov/core/models/lr_scheduled_model.py @@ -12,17 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Union, Tuple, List, Optional -from logging import getLogger -from abc import abstractmethod import math +from abc import abstractmethod from enum import IntEnum +from logging import getLogger +from typing import Union, Tuple, List, Optional import numpy as np from deeppavlov.core.common.errors import ConfigError - log = getLogger(__name__) @@ -80,8 +79,8 @@ def __init__(self, dec_type: Union[str, DecayType], start_val: float, self.div = 1.0 if not self.start_val else self.end_val / self.start_val def __str__(self): - return f"DecayScheduler(start_val={self.start_val}, end_val={self.end_val}"\ - f", dec_type={self.dec_type.name}, num_it={self.nb}, extra={self.extra})" + return f"DecayScheduler(start_val={self.start_val}, end_val={self.end_val}" \ + f", dec_type={self.dec_type.name}, num_it={self.nb}, extra={self.extra})" def next_val(self) -> float: self.iters = min(self.iters + 1, self.nb) @@ -334,8 +333,8 @@ def fit(self, *args): if not isinstance(report, dict): report = {'loss': report} # Calculating smoothed loss - avg_loss = self._fit_beta*avg_loss + (1 - self._fit_beta)*report['loss'] - smoothed_loss = avg_loss / (1 - self._fit_beta**(i + 1)) + avg_loss = self._fit_beta * avg_loss + (1 - self._fit_beta) * report['loss'] + smoothed_loss = avg_loss / (1 - self._fit_beta ** (i + 1)) lrs.append(self._lr) losses.append(smoothed_loss) log.info(f"Batch {i}/{num_batches}: smooth_loss = {smoothed_loss}" @@ -392,7 +391,7 @@ def _get_best(values: List[float], losses: List[float], assert len(values) == len(losses), "lengths of values and losses should be equal" min_ind = np.argmin(losses) for i in range(min_ind - 1, 0, -1): - if (losses[i] * max_loss_div > losses[min_ind]) or\ + if (losses[i] * max_loss_div > losses[min_ind]) or \ (values[i] * min_val_div < values[min_ind]): return values[i + 1] return values[min_ind] / min_val_div @@ -417,7 +416,7 @@ def process_event(self, event_name: str, data: dict) -> None: self._learning_rate_last_impatience = data['impatience'] - if (self._learning_rate_drop_patience is not None) and\ + if (self._learning_rate_drop_patience is not None) and \ (self._learning_rate_cur_impatience >= self._learning_rate_drop_patience): self._learning_rate_cur_impatience = 0 @@ -447,4 +446,3 @@ def process_event(self, event_name: str, data: dict) -> None: data['learning_rate'] = self._lr if (self._mom is not None) and ('momentum' not in data): data['momentum'] = self._mom - diff --git a/deeppavlov/core/models/nn_model.py b/deeppavlov/core/models/nn_model.py index 94bd4cf4ee..2a756b5c32 100644 --- a/deeppavlov/core/models/nn_model.py +++ b/deeppavlov/core/models/nn_model.py @@ -20,6 +20,7 @@ class NNModel(Component, Serializable): """Abstract class for deep learning components.""" + @abstractmethod def train_on_batch(self, x: list, y: list): pass diff --git a/deeppavlov/core/models/serializable.py b/deeppavlov/core/models/serializable.py index bb05ff5c9d..6d097476c5 100644 --- a/deeppavlov/core/models/serializable.py +++ b/deeppavlov/core/models/serializable.py @@ -27,7 +27,9 @@ class Serializable(metaclass=ABCMeta): :class:`deeppavlov.models.model.serializable.Serializable` is an abstract base class that expresses the interface for all models that can serialize data to a path. """ - def __init__(self, save_path: Optional[Union[str, Path]], load_path: Optional[Union[str, Path]] = None, mode: str = 'infer', + + def __init__(self, save_path: Optional[Union[str, Path]], load_path: Optional[Union[str, Path]] = None, + mode: str = 'infer', *args, **kwargs) -> None: if save_path: diff --git a/deeppavlov/core/models/tf_backend.py b/deeppavlov/core/models/tf_backend.py index 3568c7f4ce..e52f59b68d 100644 --- a/deeppavlov/core/models/tf_backend.py +++ b/deeppavlov/core/models/tf_backend.py @@ -21,10 +21,12 @@ def _graph_wrap(func, graph): """Constructs function encapsulated in the graph.""" + @wraps(func) def _wrapped(*args, **kwargs): with graph.as_default(): return func(*args, **kwargs) + return _wrapped @@ -37,11 +39,13 @@ def _wrapped(*args, **kwargs): with graph.as_default(): K.set_session(session) return func(*args, **kwargs) + return _wrapped class TfModelMeta(with_metaclass(type, ABCMeta)): """Metaclass that helps all child classes to have their own graph and session.""" + def __call__(cls, *args, **kwargs): obj = cls.__new__(cls) from .keras_model import KerasModel diff --git a/deeppavlov/core/models/tf_model.py b/deeppavlov/core/models/tf_model.py index 4987821cd7..39d867165b 100644 --- a/deeppavlov/core/models/tf_model.py +++ b/deeppavlov/core/models/tf_model.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Iterable, Union, Tuple, Optional from collections import defaultdict from logging import getLogger from pathlib import Path +from typing import Iterable, Union, Tuple, Optional import numpy as np import tensorflow as tf @@ -24,10 +24,9 @@ from deeppavlov.core.common.errors import ConfigError from deeppavlov.core.common.registry import cls_from_str +from deeppavlov.core.models.lr_scheduled_model import LRScheduledModel from deeppavlov.core.models.nn_model import NNModel from deeppavlov.core.models.tf_backend import TfModelMeta -from deeppavlov.core.models.lr_scheduled_model import LRScheduledModel - log = getLogger(__name__) @@ -244,7 +243,7 @@ def get_train_op(self, def get_optimizer(self): return self._optimizer - def load(self, + def load(self, exclude_scopes: Optional[Iterable] = ('Optimizer', 'learning_rate', 'momentum'), @@ -253,4 +252,3 @@ def load(self, def process_event(self, *args, **kwargs): LRScheduledModel.process_event(self, *args, **kwargs) - diff --git a/deeppavlov/core/trainers/fit_trainer.py b/deeppavlov/core/trainers/fit_trainer.py index 4cdf156541..7a7820f962 100644 --- a/deeppavlov/core/trainers/fit_trainer.py +++ b/deeppavlov/core/trainers/fit_trainer.py @@ -56,6 +56,7 @@ class FitTrainer: (default is ``-1``) **kwargs: additional parameters whose names will be logged but otherwise ignored """ + def __init__(self, chainer_config: dict, *, batch_size: int = -1, metrics: Iterable[Union[str, dict]] = ('accuracy',), evaluation_targets: Iterable[str] = ('valid', 'test'), diff --git a/deeppavlov/core/trainers/nn_trainer.py b/deeppavlov/core/trainers/nn_trainer.py index f2f51d8d19..f70509598c 100644 --- a/deeppavlov/core/trainers/nn_trainer.py +++ b/deeppavlov/core/trainers/nn_trainer.py @@ -87,6 +87,7 @@ class NNTrainer(FitTrainer): * Save the model if it happened before 1st validation (to capture early training results), don't save otherwise. """ + def __init__(self, chainer_config: dict, *, batch_size: int = 1, epochs: int = -1, start_epoch_num: int = 0, @@ -115,7 +116,7 @@ def __init__(self, chainer_config: dict, *, batch_size: int = 1, def _improved(op): return lambda score, baseline: False if baseline is None or score is None \ - else op(score,baseline) + else op(score, baseline) if metric_optimization == 'maximize': self.improved = _improved(lambda a, b: a > b) @@ -156,7 +157,7 @@ def save(self) -> None: def _is_initial_validation(self): return self.validation_number == 0 - + def _is_first_validation(self): return self.validation_number == 1 @@ -240,7 +241,7 @@ def _log(self, iterator: DataLearningIterator, report.update(self.last_result) if self.losses: - report['loss'] = sum(self.losses)/len(self.losses) + report['loss'] = sum(self.losses) / len(self.losses) self.losses.clear() metrics.append(('loss', report['loss'])) @@ -342,5 +343,3 @@ def train(self, iterator: DataLearningIterator) -> None: if self.validation_number < 1: log.info('Save model to capture early training results') self.save() - - diff --git a/deeppavlov/dataset_iterators/basic_classification_iterator.py b/deeppavlov/dataset_iterators/basic_classification_iterator.py index 1168142c4d..390a6ba442 100644 --- a/deeppavlov/dataset_iterators/basic_classification_iterator.py +++ b/deeppavlov/dataset_iterators/basic_classification_iterator.py @@ -46,6 +46,7 @@ class BasicClassificationDatasetIterator(DataLearningIterator): Attributes: data: dictionary of data with fields "train", "valid" and "test" (or some of them) """ + def __init__(self, data: dict, fields_to_merge: List[str] = None, merged_field: str = None, field_to_split: str = None, split_fields: List[str] = None, split_proportions: List[float] = None, @@ -81,7 +82,7 @@ def __init__(self, data: dict, raise IOError("Given field to split BUT not given names of split fields") def _split_data(self, field_to_split: str = None, split_fields: List[str] = None, - split_proportions: List[float] = None, split_seed: int=None, stratify: bool = None) -> bool: + split_proportions: List[float] = None, split_seed: int = None, stratify: bool = None) -> bool: """ Split given field of dataset to the given list of fields with corresponding proportions diff --git a/deeppavlov/dataset_iterators/dialog_iterator.py b/deeppavlov/dataset_iterators/dialog_iterator.py index 447c47c578..59d56cb2af 100644 --- a/deeppavlov/dataset_iterators/dialog_iterator.py +++ b/deeppavlov/dataset_iterators/dialog_iterator.py @@ -63,6 +63,7 @@ class DialogDBResultDatasetIterator(DataLearningIterator): valid: list of tuples ``(db_result dictionary, '')`` from "valid" data test: list of tuples ``(db_result dictionary, '')`` from "test" data """ + @staticmethod def _db_result(data): x, y = data diff --git a/deeppavlov/dataset_iterators/document_bert_ner_iterator.py b/deeppavlov/dataset_iterators/document_bert_ner_iterator.py deleted file mode 100644 index 58fb0cacb3..0000000000 --- a/deeppavlov/dataset_iterators/document_bert_ner_iterator.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright 2017 Neural Networks and Deep Learning lab, MIPT -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from copy import copy -from random import Random -from typing import List, Dict, Tuple, Any, Iterator, Optional -import itertools -from logging import getLogger - -import numpy as np -from bert_dp.preprocessing import convert_examples_to_features, InputExample, InputFeatures -from bert_dp.tokenization import FullTokenizer - -from deeppavlov.core.common.registry import register -from deeppavlov.core.data.data_learning_iterator import DataLearningIterator -from deeppavlov.core.commands.utils import expand_path -from deeppavlov.core.common.registry import register -from deeppavlov.core.data.utils import zero_pad -from deeppavlov.core.models.component import Component - -logger = getLogger(__name__) - - -@register('document_bert_ner_iterator') -class DocumentBertNerIterator(DataLearningIterator): - """Dataset iterator for learning models, e. g. neural networks. - - Args: - data: list of (x, y) pairs for every data type in ``'train'``, ``'valid'`` and ``'test'`` - seed: random seed for data shuffling - shuffle: whether to shuffle data during batching - - Attributes: - shuffle: whether to shuffle data during batching - random: instance of ``Random`` initialized with a seed - """ - - def __init__(self, - data: Dict[str, List[Tuple[Any, Any]]], - bert_tokenizer_vocab_file: str, - do_lower_case: bool = False, - left_context_rate: float = 0.5, - max_seq_length: int = None, - one_sample_per_doc: bool = False, - seed: int = None, - shuffle: bool = True, - *args, **kwargs) -> None: - self.max_seq_length = max_seq_length or float('inf') - self.one_sample_per_doc = one_sample_per_doc - self.left_context_rate = left_context_rate - vocab_file = str(expand_path(bert_tokenizer_vocab_file)) - self.tokenizer = FullTokenizer(vocab_file=vocab_file, - do_lower_case=do_lower_case) - super().__init__(data, seed, shuffle, *args, **kwargs) - - def gen_batches(self, batch_size: int, data_type: str = 'train', - shuffle: bool = None) -> Iterator[Tuple[tuple, tuple]]: - """Generate batches of inputs and expected output to train neural networks - - Args: - batch_size: number of samples in batch - data_type: can be either 'train', 'test', or 'valid' - shuffle: whether to shuffle dataset before batching - - Yields: - a tuple of a batch of inputs and a batch of expected outputs - """ - if shuffle is None: - shuffle = self.shuffle - - data = self.data[data_type] - # doc_data: list of tuples (doc_id, list of doc samples) - doc_data = [(doc_id, [self.rm_doc_id(s) for s in doc]) - for doc_id, doc in itertools.groupby(data, key=self.get_doc_id)] - num_docs = len(doc_data) - - if num_docs == 0: - return - - # get all sentences from document - doc_chunks = [self.chunks_from_doc(doc) for doc_id, doc in doc_data] - if self.one_sample_per_doc: - samples = [next(chunk) for chunk in doc_chunks] - else: - samples = [s for chunk in doc_chunks for s in chunk] - num_samples = len(samples) - - order = list(range(num_samples)) - - if shuffle: - self.random.shuffle(order) - - if batch_size < 0: - batch_size = num_samples - - for i in range((num_samples - 1) // batch_size + 1): - yield tuple(zip(*[samples[o] - for o in order[i * batch_size: (i + 1) * batch_size]])) - - def get_instances(self, data_type: str = 'train') -> Tuple[tuple, tuple]: - data = self.data[data_type] - data_wo_doc_ids = (self.rm_doc_id(s) for s in data) - return tuple(zip(*data_wo_doc_ids)) - - @staticmethod - def get_doc_id(sample: Tuple[Any, Any]) -> int: - return sample[0][-1] - - @staticmethod - def rm_doc_id(sample: Tuple[Any, Any]) -> Tuple[Any, Any]: - x, y = sample - if len(x) > 2: - return (x[:-1], y) - return (x[0], y) - - @staticmethod - def get_text(sample: Tuple[Any, Any]) -> List[str]: - x, y = sample - if not isinstance(x[0], str): - return x[0] - return x - - @staticmethod - def merge_samples(samples: List[Tuple[Any, Any]]) -> Tuple[Any, Any]: - out_x, out_y = [], [] - for x, y in samples: - if not isinstance(x[0], str): - if not out_x: - out_x = [[]] * len(x) - out_x = tuple(out_x_i + x_i for out_x_i, x_i in zip(out_x, x)) - else: - out_x.extend(x) - out_y.extend(y) - return (out_x, out_y) - - def sample_from_doc(self, doc: List[Tuple[Any, Any]]) -> Tuple[Any, Any]: - sample_id = self.random.randint(0, len(doc) - 1) - doc_texts = [self.get_text(s) for s in doc] - rich_sample_ids = self.get_context_indices(doc_texts, - sample_id=sample_id, - subtokenizer=self.tokenizer, - max_subtokens_length=self.max_seq_length, - left_context_rate=self.left_context_rate, - random=self.random) - return self.merge_samples((doc[i] for i in rich_sample_ids)) - - def chunks_from_doc(self, doc: List[Tuple[Any, Any]]) -> List[Tuple[Any, Any]]: - pull_of_samples = copy(doc) - pull_of_texts = [self.get_text(s) for s in doc] - while pull_of_samples: - rich_sample_ids = self.get_context_indices(pull_of_texts, - sample_id=0, - subtokenizer=self.tokenizer, - max_subtokens_length=self.max_seq_length, - left_context_rate=0.0, - random=self.random) - # TODO: split differently & replace tags with 'X's for contexts - yield self.merge_samples((pull_of_samples[i] for i in rich_sample_ids)) - pull_of_samples = pull_of_samples[len(rich_sample_ids):] - pull_of_texts = pull_of_texts[len(rich_sample_ids):] - if len(rich_sample_ids) != max(rich_sample_ids) + 1: - raise RuntimeError("can't split doc {doc} into chunks") - - @staticmethod - def get_context_indices(samples: List[List[str]], - sample_id: int, - subtokenizer: FullTokenizer, - max_subtokens_length: int, - left_context_rate: float = 0.5, - random: Random = Random(31)) -> List[int]: - rich_sample_indices = [sample_id] - - toks = samples[sample_id] - l_ctx = samples[:sample_id] - r_ctx = samples[sample_id + 1:] - - subtoks_len = len([st for t in toks - for st in subtokenizer.tokenize(t)]) - l_i, r_i = 0, 0 - while (l_i < len(l_ctx)) or (r_i < len(r_ctx)): - l_rate = left_context_rate if r_i < len(r_ctx) else 1.0 - if (l_i < len(l_ctx)) and (random.random() < l_rate): - # add one sentence from left_context - subtoks = [st for t in l_ctx[-l_i-1] - for st in subtokenizer.tokenize(t)] - if subtoks_len + len(subtoks) > max_subtokens_length: - break - subtoks_len += len(subtoks) - rich_sample_indices = [sample_id - l_i - 1] + rich_sample_indices - l_i += 1 - else: - # add one sentence from right_context - subtoks = [st for t in r_ctx[r_i] for st in subtokenizer.tokenize(t)] - if subtoks_len + len(subtoks) > max_subtokens_length: - break - subtoks_len += len(subtoks) - rich_sample_indices.append(sample_id + r_i + 1) - r_i += 1 - return rich_sample_indices - diff --git a/deeppavlov/dataset_iterators/dstc2_intents_iterator.py b/deeppavlov/dataset_iterators/dstc2_intents_iterator.py index b893b5710d..3ad34bee4c 100644 --- a/deeppavlov/dataset_iterators/dstc2_intents_iterator.py +++ b/deeppavlov/dataset_iterators/dstc2_intents_iterator.py @@ -43,6 +43,7 @@ class Dstc2IntentsDatasetIterator(BasicClassificationDatasetIterator): Attributes: data: dictionary of data with fields "train", "valid" and "test" (or some of them) """ + def __init__(self, data: dict, fields_to_merge: List[str] = None, merged_field: str = None, field_to_split: str = None, split_fields: List[str] = None, split_proportions: List[float] = None, diff --git a/deeppavlov/dataset_iterators/dstc2_ner_iterator.py b/deeppavlov/dataset_iterators/dstc2_ner_iterator.py index 420e3f3a95..7b12721497 100644 --- a/deeppavlov/dataset_iterators/dstc2_ner_iterator.py +++ b/deeppavlov/dataset_iterators/dstc2_ner_iterator.py @@ -14,7 +14,6 @@ import json import logging -from overrides import overrides from typing import List, Tuple, Dict, Any from deeppavlov.core.commands.utils import expand_path @@ -36,6 +35,7 @@ class Dstc2NerDatasetIterator(DataLearningIterator): seed: value for random seed shuffle: whether to shuffle the data """ + def __init__(self, data: Dict[str, List[Tuple]], slot_values_path: str, @@ -88,8 +88,8 @@ def _add_bio_markup(self, slot_tokens = entity.split() slot_len = len(slot_tokens) if n + slot_len <= n_toks and \ - self._is_equal_sequences(tokens[n: n + slot_len], - slot_tokens): + self._is_equal_sequences(tokens[n: n + slot_len], + slot_tokens): tags[n] = 'B-' + slot_type for k in range(1, slot_len): tags[n + k] = 'I-' + slot_type diff --git a/deeppavlov/dataset_iterators/elmo_file_paths_iterator.py b/deeppavlov/dataset_iterators/elmo_file_paths_iterator.py index 84349b574c..a887fe8b4c 100644 --- a/deeppavlov/dataset_iterators/elmo_file_paths_iterator.py +++ b/deeppavlov/dataset_iterators/elmo_file_paths_iterator.py @@ -91,15 +91,15 @@ def _line2ids(self, line): reversed_token_ids = list(reversed(token_ids)) token_ids = token_ids[1:] reversed_token_ids = reversed_token_ids[1:] - + return char_ids, reversed_char_ids, token_ids, reversed_token_ids - + def _line_generator(self, shard_generator): for shard in shard_generator: line_generator = chunk_generator(shard, 1) for line in line_generator: line = line[0] - char_ids, reversed_char_ids, token_ids, reversed_token_ids =\ + char_ids, reversed_char_ids, token_ids, reversed_token_ids = \ self._line2ids(line) yield char_ids, reversed_char_ids, token_ids, reversed_token_ids @@ -124,13 +124,13 @@ def _batch_generator(line_generator, batch_size, unroll_steps): sti.clear() sti.extend(_s) - char_ids, reversed_char_ids, token_ids, reversed_token_ids =\ + char_ids, reversed_char_ids, token_ids, reversed_token_ids = \ zip(*batch) yield char_ids, reversed_char_ids, token_ids, reversed_token_ids except StopIteration: pass - def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: Optional[bool] = None)\ + def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: Optional[bool] = None) \ -> Iterator[Tuple[str, str]]: if shuffle is None: shuffle = self.shuffle diff --git a/deeppavlov/dataset_iterators/file_paths_iterator.py b/deeppavlov/dataset_iterators/file_paths_iterator.py index 1230d115ba..9d8769f8b2 100644 --- a/deeppavlov/dataset_iterators/file_paths_iterator.py +++ b/deeppavlov/dataset_iterators/file_paths_iterator.py @@ -39,7 +39,7 @@ class FilePathsIterator(DataLearningIterator): def __init__(self, data: Dict[str, List[Union[str, Path]]], - seed: Optional[int] = None, + seed: Optional[int] = None, shuffle: bool = True, *args, **kwargs) -> None: self.seed = seed @@ -58,7 +58,7 @@ def _shard_generator(self, shards: List[Union[str, Path]], shuffle: bool = False self.np_random.shuffle(lines) yield lines - def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: Optional[bool] = None)\ + def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: Optional[bool] = None) \ -> Iterator[Tuple[str, str]]: if shuffle is None: shuffle = self.shuffle diff --git a/deeppavlov/dataset_iterators/kvret_dialog_iterator.py b/deeppavlov/dataset_iterators/kvret_dialog_iterator.py index dcc3f8e772..c2147c63a2 100644 --- a/deeppavlov/dataset_iterators/kvret_dialog_iterator.py +++ b/deeppavlov/dataset_iterators/kvret_dialog_iterator.py @@ -30,7 +30,8 @@ class KvretDialogDatasetIterator(DataLearningIterator): valid: list of "valid" ``(context, response)`` tuples test: list of "test" ``(context, response)`` tuples """ -# TODO: write custom batch_generator: order of utterances from one dialogue is presumed + + # TODO: write custom batch_generator: order of utterances from one dialogue is presumed @staticmethod def _dialogs(data): dialogs = [] @@ -38,14 +39,14 @@ def _dialogs(data): task = None for x, y in data: if x.get('episode_done'): - #history = [] + # history = [] history = "" dialogs.append((([], [], [], [], []), ([], []))) task = y['task'] - #history.append((x, y)) + # history.append((x, y)) history = history + ' ' + x['text'] + ' ' + y['text'] - #x['history'] = history[:-1] - x['history'] = history[:-len(x['text'])-len(y['text'])-2] + # x['history'] = history[:-1] + x['history'] = history[:-len(x['text']) - len(y['text']) - 2] dialogs[-1][0][0].append(x['text']) dialogs[-1][0][1].append(x['dialog_id']) dialogs[-1][0][2].append(x['history']) @@ -68,7 +69,7 @@ def preprocess(self, data, *args, **kwargs): history = history + ' ' + x['text'] + ' ' + y['text'] # x['x_hist'] = x_hist[:-1] # x['y_hist'] = y_hist[:-1] - x['history'] = history[:-len(x['text'])-len(y['text'])-2] + x['history'] = history[:-len(x['text']) - len(y['text']) - 2] x_tuple = (x['text'], x['dialog_id'], x['history'], x['kb_columns'], x['kb_items']) y_tuple = (y['text'], y['task']['intent']) diff --git a/deeppavlov/dataset_iterators/morphotagger_iterator.py b/deeppavlov/dataset_iterators/morphotagger_iterator.py index dca0daa990..b94d5b0285 100644 --- a/deeppavlov/dataset_iterators/morphotagger_iterator.py +++ b/deeppavlov/dataset_iterators/morphotagger_iterator.py @@ -60,8 +60,9 @@ class MorphoTaggerDatasetIterator(DataLearningIterator): For fair comparison with UD Pipe it is set to 0.9 for UD experiments. It is actually used only for Turkish data. """ + def __init__(self, data: Dict[str, List[Tuple[Any, Any]]], seed: int = None, - shuffle: bool = True, min_train_fraction: float = 0.0, + shuffle: bool = True, min_train_fraction: float = 0.0, validation_split: float = 0.2) -> None: self.validation_split = validation_split self.min_train_fraction = min_train_fraction diff --git a/deeppavlov/dataset_iterators/ner_few_shot_iterator.py b/deeppavlov/dataset_iterators/ner_few_shot_iterator.py index a635ec874e..52e1fa38c1 100644 --- a/deeppavlov/dataset_iterators/ner_few_shot_iterator.py +++ b/deeppavlov/dataset_iterators/ner_few_shot_iterator.py @@ -35,6 +35,7 @@ class NERFewShotIterator(DataLearningIterator): the same remove_not_targets: whether to replace all non target tags with `O` tag or not. """ + def __init__(self, data: Dict[str, List[Tuple[Any, Any]]], seed: int = None, diff --git a/deeppavlov/dataset_iterators/siamese_iterator.py b/deeppavlov/dataset_iterators/siamese_iterator.py index 222a09efed..dd418d6532 100644 --- a/deeppavlov/dataset_iterators/siamese_iterator.py +++ b/deeppavlov/dataset_iterators/siamese_iterator.py @@ -65,7 +65,7 @@ def split(self, *args, **kwargs) -> None: self.test = self.train[-self.len_test:] self.train = self.train[:-self.len_test] - def gen_batches(self, batch_size: int, data_type: str = "train", shuffle: bool = True)->\ + def gen_batches(self, batch_size: int, data_type: str = "train", shuffle: bool = True) -> \ Tuple[List[List[Tuple[int, int]]], List[int]]: """Generate batches of inputs and expected outputs to train neural networks. diff --git a/deeppavlov/dataset_iterators/snips_intents_iterator.py b/deeppavlov/dataset_iterators/snips_intents_iterator.py index 306329e762..2a881634ac 100644 --- a/deeppavlov/dataset_iterators/snips_intents_iterator.py +++ b/deeppavlov/dataset_iterators/snips_intents_iterator.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Any - from overrides import overrides from deeppavlov.core.common.registry import register diff --git a/deeppavlov/dataset_iterators/squad_iterator.py b/deeppavlov/dataset_iterators/squad_iterator.py index 518c6b6aa7..c7300799f8 100644 --- a/deeppavlov/dataset_iterators/squad_iterator.py +++ b/deeppavlov/dataset_iterators/squad_iterator.py @@ -95,7 +95,7 @@ def __init__(self, data, seed: Optional[int] = None, shuffle: bool = True, with_ self.np_random = np.random.RandomState(seed) super().__init__(data, seed, shuffle, *args, **kwargs) - def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: bool = None)\ + def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: bool = None) \ -> Generator[Tuple[Tuple[Tuple[str, str]], Tuple[List[str], List[int]]], None, None]: if shuffle is None: @@ -114,7 +114,7 @@ def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: bool = for i in range((data_len - 1) // batch_size + 1): batch = [] - for j in range(i * batch_size, min((i+1) * batch_size, data_len)): + for j in range(i * batch_size, min((i + 1) * batch_size, data_len)): q = data[j]['question'] contexts = data[j]['contexts'] ans_contexts = [c for c in contexts if len(c['answer']) > 0] @@ -131,7 +131,8 @@ def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: bool = context = noans_contexts[np.argmax(random.multinomial(1, noans_scores))] answer_text = [ans['text'] for ans in context['answer']] if len(context['answer']) > 0 else [''] - answer_start = [ans['answer_start'] for ans in context['answer']] if len(context['answer']) > 0 else [-1] + answer_start = [ans['answer_start'] + for ans in context['answer']] if len(context['answer']) > 0 else [-1] batch.append(((context['context'], q), (answer_text, answer_start))) yield tuple(zip(*batch)) @@ -194,7 +195,7 @@ def __init__(self, data, seed: Optional[int] = None, shuffle: bool = False, if self.shuffle: raise RuntimeError('MultiSquadIterator doesn\'t support shuffling.') - def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: bool = None)\ + def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: bool = None) \ -> Generator[Tuple[Tuple[Tuple[str, str]], Tuple[List[str], List[int]]], None, None]: if shuffle is None: @@ -266,4 +267,4 @@ def get_instances(self, data_type: str = 'train') -> Tuple[Tuple[Tuple[str, str] answer_text = [x['text'] for x in context['answer']] answer_start = [x['answer_start'] for x in context['answer']] data_examples.append(((context['context'], question), (answer_text, answer_start))) - return tuple(zip(*data_examples)) \ No newline at end of file + return tuple(zip(*data_examples)) diff --git a/deeppavlov/dataset_iterators/typos_iterator.py b/deeppavlov/dataset_iterators/typos_iterator.py index a5e7eb483a..72561ff542 100644 --- a/deeppavlov/dataset_iterators/typos_iterator.py +++ b/deeppavlov/dataset_iterators/typos_iterator.py @@ -22,7 +22,8 @@ class TyposDatasetIterator(DataLearningIterator): :class:`~deeppavlov.models.spelling_correction.brillmoore.ErrorModel` """ - def split(self, test_ratio: float=0., *args, **kwargs): + + def split(self, test_ratio: float = 0., *args, **kwargs): """Split all data into train and test Args: diff --git a/deeppavlov/dataset_readers/amazon_ecommerce_reader.py b/deeppavlov/dataset_readers/amazon_ecommerce_reader.py index 50ab12ae3e..b8a0dc0acc 100644 --- a/deeppavlov/dataset_readers/amazon_ecommerce_reader.py +++ b/deeppavlov/dataset_readers/amazon_ecommerce_reader.py @@ -21,6 +21,7 @@ logger = getLogger(__name__) + @register('amazon_ecommerce_reader') class AmazonEcommerceReader(DatasetReader): """Class to download and load ecommerce data catalog""" @@ -56,8 +57,8 @@ def read(self, data_path: str, catalog: list, **kwargs) -> Dict[str, List[Tuple[ dataset = { 'train': [((item['Title'], [], {}), item) for item in ec_data_global], 'valid': [], - 'test': [] - } + 'test': [] + } logger.info(f"In total {len(ec_data_global)} items are loaded") return dataset diff --git a/deeppavlov/dataset_readers/basic_classification_reader.py b/deeppavlov/dataset_readers/basic_classification_reader.py index 8b33963dd5..af2a73a9f3 100644 --- a/deeppavlov/dataset_readers/basic_classification_reader.py +++ b/deeppavlov/dataset_readers/basic_classification_reader.py @@ -63,7 +63,9 @@ def read(self, data_path: str, url: str = None, if not Path(data_path, train_file).exists(): if url is None: - raise Exception("data path {} does not exist or is empty, and download url parameter not specified!".format(data_path)) + raise Exception( + "data path {} does not exist or is empty, and download url parameter not specified!".format( + data_path)) log.info("Loading train data from {} to {}".format(url, data_path)) download(source_url=url, dest_file_path=Path(data_path, train_file)) diff --git a/deeppavlov/dataset_readers/conll2003_reader.py b/deeppavlov/dataset_readers/conll2003_reader.py index 19d2bc1d87..0e0958e34a 100644 --- a/deeppavlov/dataset_readers/conll2003_reader.py +++ b/deeppavlov/dataset_readers/conll2003_reader.py @@ -1,5 +1,5 @@ -from pathlib import Path from logging import getLogger +from pathlib import Path from deeppavlov.core.common.registry import register from deeppavlov.core.data.dataset_reader import DatasetReader @@ -7,6 +7,7 @@ log = getLogger(__name__) + @register('conll2003_reader') class Conll2003DatasetReader(DatasetReader): """Class to read training datasets in CoNLL-2003 format""" @@ -94,7 +95,7 @@ def parse_ner_file(self, file_name: Path): except: log.warning('Skip {}, splitted as {}'.format(repr(line), repr(line.split()))) continue - + tags.append(tag) tokens.append(token) @@ -106,7 +107,7 @@ def parse_ner_file(self, file_name: Path): x = x + (self.num_docs,) samples.append((x, tags)) self.num_docs += 1 - + if self.iob: return [(x, self._iob2_to_iob(tags)) for x, tags in samples] diff --git a/deeppavlov/dataset_readers/dstc2_reader.py b/deeppavlov/dataset_readers/dstc2_reader.py index 187d047e52..4d2aa7513d 100644 --- a/deeppavlov/dataset_readers/dstc2_reader.py +++ b/deeppavlov/dataset_readers/dstc2_reader.py @@ -111,7 +111,7 @@ def _read_from_file(cls, file_path, dialogs=False): """Returns data from single file""" log.info(f"[loading dialogs from {file_path}]") - utterances, responses, dialog_indices =\ + utterances, responses, dialog_indices = \ cls._get_turns(cls._iter_file(file_path), with_indices=True) data = list(map(cls._format_turn, zip(utterances, responses))) @@ -289,7 +289,7 @@ def _read_from_file(cls, file_path: str, dialogs: bool = False): """Returns data from single file""" log.info(f"[loading dialogs from {file_path}]") - utterances, responses, dialog_indices =\ + utterances, responses, dialog_indices = \ cls._get_turns(json.load(open(file_path, 'rt')), with_indices=True) data = list(map(cls._format_turn, zip(utterances, responses))) diff --git a/deeppavlov/dataset_readers/file_paths_reader.py b/deeppavlov/dataset_readers/file_paths_reader.py index cbe632f58b..adddc08470 100644 --- a/deeppavlov/dataset_readers/file_paths_reader.py +++ b/deeppavlov/dataset_readers/file_paths_reader.py @@ -57,7 +57,7 @@ def _get_files(self, data_path, tgt): paths = Path(data_path).resolve().glob(tgt) files = [file for file in paths if Path(file).is_file()] paths_info = Path(data_path, tgt).absolute().as_posix() - if not(files): + if not (files): raise Exception(f"Not find files. Data path '{paths_info}' does not exist or does not hold files!") else: log.info(f"Found {len(files)} files located '{paths_info}'.") diff --git a/deeppavlov/dataset_readers/kvret_reader.py b/deeppavlov/dataset_readers/kvret_reader.py index 996eb3f025..275a5f0f5d 100644 --- a/deeppavlov/dataset_readers/kvret_reader.py +++ b/deeppavlov/dataset_readers/kvret_reader.py @@ -78,8 +78,8 @@ def _read_from_file(cls, file_path, dialogs=False): """Returns data from single file""" log.info("[loading dialogs from {}]".format(file_path)) - utterances, responses, dialog_indices =\ - cls._get_turns(cls._iter_file(file_path), with_indices=True) + utterances, responses, dialog_indices = \ + cls._get_turns(cls._iter_file(file_path), with_indices=True) data = list(map(cls._format_turn, zip(utterances, responses))) @@ -105,22 +105,22 @@ def _format_turn(turn): @staticmethod def _check_dialog(dialog): - #TODO: manually fix bad dialogs + # TODO: manually fix bad dialogs driver = True for turn in dialog: if turn['turn'] not in ('driver', 'assistant'): raise RuntimeError("Dataset wrong format: `turn` key value is" " either `driver` or `assistant`.") if driver and turn['turn'] != 'driver': - log.debug("Turn is expected to by driver's, but it's {}'s"\ + log.debug("Turn is expected to by driver's, but it's {}'s" \ .format(turn['turn'])) return False if not driver and turn['turn'] != 'assistant': - log.debug("Turn is expected to be assistant's but it's {}'s"\ + log.debug("Turn is expected to be assistant's but it's {}'s" \ .format(turn['turn'])) return False driver = not driver - #if not driver: + # if not driver: # log.debug("Last turn is expected to be by assistant") # return False return True @@ -143,7 +143,7 @@ def _iter_file(cls, file_path): if cls._check_dialog(dialog): yield dialog, sample['scenario'] else: - log.warn("Skipping {}th dialogue with uuid={}: wrong format."\ + log.warn("Skipping {}th dialogue with uuid={}: wrong format." \ .format(i, sample['scenario']['uuid'])) @staticmethod @@ -172,7 +172,7 @@ def _get_turns(data, with_indices=False): if last_utter and not last_utter[-1].isspace(): last_utter += ' ' responses[-1]['utterance'] = last_utter + 'END_OF_DIALOGUE' - + dialog_indices.append({ 'start': len(utterances), 'end': len(utterances) + len(dialog), @@ -181,4 +181,3 @@ def _get_turns(data, with_indices=False): if with_indices: return utterances, responses, dialog_indices return utterances, responses - diff --git a/deeppavlov/dataset_readers/line_reader.py b/deeppavlov/dataset_readers/line_reader.py index 0699f9b141..651ff56041 100644 --- a/deeppavlov/dataset_readers/line_reader.py +++ b/deeppavlov/dataset_readers/line_reader.py @@ -36,7 +36,7 @@ def read(self, data_path: str = None, *args, **kwargs) -> Dict: content = f.readlines() dataset = dict() - dataset["train"] = [(line, ) for line in content] + dataset["train"] = [(line,) for line in content] dataset["valid"] = [] dataset["test"] = [] diff --git a/deeppavlov/dataset_readers/morphotagging_dataset_reader.py b/deeppavlov/dataset_readers/morphotagging_dataset_reader.py index c23363d170..af0af41196 100644 --- a/deeppavlov/dataset_readers/morphotagging_dataset_reader.py +++ b/deeppavlov/dataset_readers/morphotagging_dataset_reader.py @@ -23,14 +23,15 @@ WORD_COLUMN, POS_COLUMN, TAG_COLUMN = 1, 3, 5 - log = getLogger(__name__) + def get_language(filepath: str) -> str: """Extracts language from typical UD filename """ return filepath.split("-")[0] + def read_infile(infile: Union[Path, str], from_words=False, word_column: int = WORD_COLUMN, pos_column: int = POS_COLUMN, tag_column: int = TAG_COLUMN, max_sents: int = -1, @@ -163,7 +164,7 @@ def read(self, data_path: Union[List, str], for mode, filepath in zip(data_types, data_path): if mode == "dev": mode = "valid" -# if mode == "test": -# kwargs["read_only_words"] = True + # if mode == "test": + # kwargs["read_only_words"] = True data[mode] = read_infile(filepath, **kwargs) return data diff --git a/deeppavlov/dataset_readers/paraphraser_pretrain_reader.py b/deeppavlov/dataset_readers/paraphraser_pretrain_reader.py index 50f041fb89..1ad5d68adf 100644 --- a/deeppavlov/dataset_readers/paraphraser_pretrain_reader.py +++ b/deeppavlov/dataset_readers/paraphraser_pretrain_reader.py @@ -42,7 +42,6 @@ def read(self, dataset = {"train": train_data, "valid": test_data, "test": test_data} return dataset - def int_class(self, str_y): if str_y == '-1': return 0 @@ -53,5 +52,3 @@ def build_data(self, name): with open(name) as f: data = json.load(f) return [([doc['text_1'], doc['text_2']], self.int_class(doc['class'])) for doc in data] - - diff --git a/deeppavlov/dataset_readers/paraphraser_reader.py b/deeppavlov/dataset_readers/paraphraser_reader.py index 638d4e6ce3..e73e12985d 100644 --- a/deeppavlov/dataset_readers/paraphraser_reader.py +++ b/deeppavlov/dataset_readers/paraphraser_reader.py @@ -41,7 +41,7 @@ def read(self, data_path = expand_path(data_path) train_fname = data_path / 'paraphrases.xml' - test_fname = data_path / 'paraphrases_gold.xml' + test_fname = data_path / 'paraphrases_gold.xml' train_data = self.build_data(train_fname, do_lower_case) test_data = self.build_data(test_fname, do_lower_case) dataset = {"train": train_data, "valid": [], "test": test_data} diff --git a/deeppavlov/dataset_readers/snips_reader.py b/deeppavlov/dataset_readers/snips_reader.py index 28ec70b56c..7041df6aa7 100644 --- a/deeppavlov/dataset_readers/snips_reader.py +++ b/deeppavlov/dataset_readers/snips_reader.py @@ -14,8 +14,8 @@ import json from logging import getLogger -from typing import List, Dict, Any, Optional from pathlib import Path +from typing import List, Dict, Any, Optional from overrides import overrides diff --git a/deeppavlov/dataset_readers/sq_reader.py b/deeppavlov/dataset_readers/sq_reader.py index 4ee2503dd7..00949a6cb5 100644 --- a/deeppavlov/dataset_readers/sq_reader.py +++ b/deeppavlov/dataset_readers/sq_reader.py @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path import pickle -from deeppavlov.core.data.dataset_reader import DatasetReader -from deeppavlov.core.data.utils import download from deeppavlov.core.common.registry import register +from deeppavlov.core.data.dataset_reader import DatasetReader @register('sq_reader') diff --git a/deeppavlov/dataset_readers/squad_dataset_reader.py b/deeppavlov/dataset_readers/squad_dataset_reader.py index f41f7f6723..24c0b6d159 100644 --- a/deeppavlov/dataset_readers/squad_dataset_reader.py +++ b/deeppavlov/dataset_readers/squad_dataset_reader.py @@ -47,7 +47,7 @@ class SquadDatasetReader(DatasetReader): url_sber_squad = 'http://files.deeppavlov.ai/datasets/sber_squad-v1.1.tar.gz' url_multi_squad = 'http://files.deeppavlov.ai/datasets/multiparagraph_squad.tar.gz' - def read(self, dir_path: str, dataset: Optional[str] = 'SQuAD', url: Optional[str] = None, *args, **kwargs)\ + def read(self, dir_path: str, dataset: Optional[str] = 'SQuAD', url: Optional[str] = None, *args, **kwargs) \ -> Dict[str, Dict[str, Any]]: """ diff --git a/deeppavlov/dataset_readers/typos_reader.py b/deeppavlov/dataset_readers/typos_reader.py index 8a876beb6a..4bc0f8515a 100644 --- a/deeppavlov/dataset_readers/typos_reader.py +++ b/deeppavlov/dataset_readers/typos_reader.py @@ -72,6 +72,7 @@ class TyposWikipedia(TyposCustom): English Wikipedia's list of common misspellings """ + @staticmethod def build(data_path: str) -> Path: """Download and parse common misspellings list from `Wikipedia `_ @@ -116,6 +117,7 @@ class TyposKartaslov(DatasetReader): a Russian misspellings dataset from `kartaslov `_ """ + def __init__(self): pass diff --git a/deeppavlov/dataset_readers/ubuntu_dstc7_mt_reader.py b/deeppavlov/dataset_readers/ubuntu_dstc7_mt_reader.py index 398a12a369..d7539ae171 100644 --- a/deeppavlov/dataset_readers/ubuntu_dstc7_mt_reader.py +++ b/deeppavlov/dataset_readers/ubuntu_dstc7_mt_reader.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Tuple, Dict -from pathlib import Path import json +from pathlib import Path +from typing import List, Tuple, Dict import numpy as np @@ -36,7 +36,7 @@ class UbuntuDSTC7MTReader(DatasetReader): it can be reduced to 10 (1 true response + 9 random wrong responses) to adapt with succeeding pipeline padding (str): "post" or "pre" context sentences padding """ - + def read(self, data_path: str, num_context_turns: int = 10, @@ -53,7 +53,7 @@ def read(self, dataset = {} dataset["train"] = self._create_dialog_iter(Path(data_path) / 'ubuntu_train_subtask_1.json', "train") dataset["valid"] = self._create_dialog_iter(Path(data_path) / 'ubuntu_dev_subtask_1.json', "valid") - dataset["test"] = self._create_dialog_iter(Path(data_path) / 'ubuntu_test_subtask_1.json', "test") + dataset["test"] = self._create_dialog_iter(Path(data_path) / 'ubuntu_test_subtask_1.json', "test") return dataset def _create_dialog_iter(self, filename, mode="train"): @@ -84,7 +84,7 @@ def _create_dialog_iter(self, filename, mode="train"): for entry in json_data: dialog = entry - utterances = [] # all the context sentences + utterances = [] # all the context sentences for msg in dialog['messages-so-far']: utterances.append(msg['utterance']) @@ -92,7 +92,7 @@ def _create_dialog_iter(self, filename, mode="train"): if mode != "test": true_response = dialog['options-for-correct-answers'][0]['utterance'] - fake_responses = [] # rest (wrong) responses + fake_responses = [] # rest (wrong) responses target_id = "" if mode != "test": correct_answer = dialog['options-for-correct-answers'][0] @@ -106,11 +106,13 @@ def _create_dialog_iter(self, filename, mode="train"): if mode == 'train': data.append((expanded_context + [true_response], 1)) - data.append((expanded_context + list(self.np_random.choice(fake_responses, size=1)), 0)) # random 1 from 99 + data.append( + (expanded_context + list(self.np_random.choice(fake_responses, size=1)), 0)) # random 1 from 99 elif mode == 'valid': # NOTE: labels are useless here... - data.append((expanded_context + [true_response] + list(self.np_random.choice(fake_responses, self.num_responses-1)), 0)) + data.append((expanded_context + [true_response] + list( + self.np_random.choice(fake_responses, self.num_responses - 1)), 0)) elif mode == 'test': data.append((expanded_context + fake_responses, 0)) diff --git a/deeppavlov/dataset_readers/ubuntu_v1_mt_reader.py b/deeppavlov/dataset_readers/ubuntu_v1_mt_reader.py index b4e4eb08f3..6761cee749 100644 --- a/deeppavlov/dataset_readers/ubuntu_v1_mt_reader.py +++ b/deeppavlov/dataset_readers/ubuntu_v1_mt_reader.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Tuple, Union, Dict from pathlib import Path +from typing import List, Tuple, Union, Dict from deeppavlov.core.common.registry import register from deeppavlov.core.data.dataset_reader import DatasetReader @@ -30,6 +30,7 @@ class UbuntuV1MTReader(DatasetReader): num_context_turns: A maximum number of dialogue ``context`` turns. padding: "post" or "pre" context sentences padding """ + def read(self, data_path: str, num_context_turns: int = 1, padding: str = "post", diff --git a/deeppavlov/dataset_readers/ubuntu_v2_mt_reader.py b/deeppavlov/dataset_readers/ubuntu_v2_mt_reader.py index 178c421bfc..57b779bd11 100644 --- a/deeppavlov/dataset_readers/ubuntu_v2_mt_reader.py +++ b/deeppavlov/dataset_readers/ubuntu_v2_mt_reader.py @@ -31,7 +31,7 @@ class UbuntuV2MTReader(DatasetReader): num_context_turns: A maximum number of dialogue ``context`` turns. padding: "post" or "pre" context sentences padding """ - + def read(self, data_path: str, num_context_turns: int = 1, padding: str = "post", @@ -61,7 +61,7 @@ def read(self, data_path: str, dataset["valid"] = self.preprocess_data_validation(valid_fname) dataset["test"] = self.preprocess_data_validation(test_fname) return dataset - + def preprocess_data_train(self, train_fname: Union[Path, str]) -> List[Tuple[List[str], int]]: contexts = [] responses = [] @@ -87,7 +87,7 @@ def preprocess_data_validation(self, fname: Union[Path, str]) -> List[Tuple[List contexts.append(self._expand_context(el[0].split('__eot__'), padding=self.padding)) responses.append(el[1:]) data = [el[0] + el[1] for el in zip(contexts, responses)] - data = [(el, 1) for el in data] # NOTE: labels are useless here actually... + data = [(el, 1) for el in data] # NOTE: labels are useless here actually... return data def _expand_context(self, context: List[str], padding: str) -> List[str]: diff --git a/deeppavlov/dataset_readers/ubuntu_v2_reader.py b/deeppavlov/dataset_readers/ubuntu_v2_reader.py index 2c870ea4ba..00e45bc9c2 100644 --- a/deeppavlov/dataset_readers/ubuntu_v2_reader.py +++ b/deeppavlov/dataset_readers/ubuntu_v2_reader.py @@ -52,7 +52,7 @@ def read(self, data_path: str, dataset["valid"] = self.preprocess_data_validation(valid_fname) dataset["test"] = self.preprocess_data_validation(test_fname) return dataset - + def preprocess_data_train(self, train_fname: Union[Path, str]) -> List[Tuple[List[str], int]]: contexts = [] responses = [] @@ -82,4 +82,4 @@ def preprocess_data_validation(self, fname: Union[Path, str]) -> List[Tuple[List responses.append(el[1:]) data = [[el[0]] + el[1] for el in zip(contexts, responses)] data = [(el, 1) for el in data] - return data \ No newline at end of file + return data diff --git a/deeppavlov/deep.py b/deeppavlov/deep.py index b0533f5095..27f5912987 100644 --- a/deeppavlov/deep.py +++ b/deeppavlov/deep.py @@ -47,7 +47,7 @@ parser.add_argument("--folds", help="number of folds", type=int, default=5) -parser.add_argument("-t", "--token", default=None, help="telegram bot token", type=str) +parser.add_argument("-t", "--token", default=None, help="telegram bot token", type=str) parser.add_argument("-i", "--ms-id", default=None, help="microsoft bot framework app id", type=str) parser.add_argument("-s", "--ms-secret", default=None, help="microsoft bot framework app secret", type=str) diff --git a/deeppavlov/deprecated/agent/agent.py b/deeppavlov/deprecated/agent/agent.py index 9bec306f6c..2154f9d4a5 100644 --- a/deeppavlov/deprecated/agent/agent.py +++ b/deeppavlov/deprecated/agent/agent.py @@ -50,6 +50,7 @@ class Agent(Component, metaclass=ABCMeta): We highly recommend to use wrapped skills for skills inference. dialog_logger: DeepPavlov dialog logging facility. """ + def __init__(self, skills: List[Component]) -> None: self.skills = skills self.history: Dict = defaultdict(list) @@ -118,6 +119,7 @@ class SkillWrapper: skill_id: Skill index in Agent.skills list. agent: Agent instance. """ + def __init__(self, skill: Component, skill_id: int, agent: Agent) -> None: self.skill = skill self.skill_id = skill_id diff --git a/deeppavlov/deprecated/agent/filter.py b/deeppavlov/deprecated/agent/filter.py index d263ccc777..65d60f46bd 100644 --- a/deeppavlov/deprecated/agent/filter.py +++ b/deeppavlov/deprecated/agent/filter.py @@ -22,6 +22,7 @@ class Filter(Component, metaclass=ABCMeta): which is used in Agent to select utterances from incoming batch to be processed for each Agent skill. """ + @abstractmethod def __call__(self, utterances_batch: list, history_batch: list) -> list: """Returns skills-utterances application matrix. diff --git a/deeppavlov/deprecated/agent/processor.py b/deeppavlov/deprecated/agent/processor.py index 2a87f2dee0..b2e99f9bcc 100644 --- a/deeppavlov/deprecated/agent/processor.py +++ b/deeppavlov/deprecated/agent/processor.py @@ -22,6 +22,7 @@ class Processor(Component, metaclass=ABCMeta): which is used in Agent to process skills responses and give one final response for each utterance. """ + # TODO: change *responses to [[], [], ...] argument @abstractmethod def __call__(self, utterances_batch: list, history_batch: list, *responses: list) -> list: diff --git a/deeppavlov/deprecated/agent/rich_content.py b/deeppavlov/deprecated/agent/rich_content.py index 436fa9bdc2..1b4070b951 100644 --- a/deeppavlov/deprecated/agent/rich_content.py +++ b/deeppavlov/deprecated/agent/rich_content.py @@ -24,6 +24,7 @@ class RichItem(metaclass=ABCMeta): at least in json format (mandatory) as well as in the formats compatible with other channels. """ + @abstractmethod def json(self) -> Union[list, dict]: """Returns json compatible state of the control instance including @@ -77,6 +78,7 @@ class RichControl(RichItem, metaclass=ABCMeta): control_json: Control json representation template, which contains control type and content fields. """ + def __init__(self, control_type: str) -> None: self.control_type: str = control_type self.content = None diff --git a/deeppavlov/deprecated/agents/default_agent/default_agent.py b/deeppavlov/deprecated/agents/default_agent/default_agent.py index 4737bb80c4..fbcfff1f93 100644 --- a/deeppavlov/deprecated/agents/default_agent/default_agent.py +++ b/deeppavlov/deprecated/agents/default_agent/default_agent.py @@ -46,6 +46,7 @@ class DefaultAgent(Agent): skills_processor: Initiated agent processor. skills_filter: Initiated agent filter. """ + def __init__(self, skills: List[Component], skills_processor: Optional[Processor] = None, skills_filter: Optional[Filter] = None, *args, **kwargs) -> None: super(DefaultAgent, self).__init__(skills=skills) diff --git a/deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py b/deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py index 5313235d5c..d4b57290c2 100644 --- a/deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py +++ b/deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py @@ -129,15 +129,16 @@ def _call(self, utterances_batch: List[str], utterances_ids: List[int] = None) - return [rich_message] + def _draw_tail(entropy, history): buttons_frame = ButtonsFrame(text="") - buttons_frame.add_button(Button('More', "@next:"+str(len(history)-1))) + buttons_frame.add_button(Button('More', "@next:" + str(len(history) - 1))) caption = "Press More " if entropy: caption += "specify a " + entropy[0][1] for ent_value in entropy[0][2][:4]: - button_a = Button(ent_value[0], f'@entropy:{len(history)-1}:{entropy[0][1]}:{ent_value[0]}') + button_a = Button(ent_value[0], f'@entropy:{len(history) - 1}:{entropy[0][1]}:{ent_value[0]}') buttons_frame.add_button(button_a) buttons_frame.text = caption @@ -147,10 +148,10 @@ def _draw_tail(entropy, history): def _draw_item(item, idx, history): title = item['Title'] if 'ListPrice' in item: - title += " - **$" + item['ListPrice'].split('$')[1]+"**" + title += " - **$" + item['ListPrice'].split('$')[1] + "**" buttons_frame = ButtonsFrame(text=title) - buttons_frame.add_button(Button('Show details', "@details:"+str(len(history)-2)+":"+str(idx))) + buttons_frame.add_button(Button('Show details', "@details:" + str(len(history) - 2) + ":" + str(idx))) return buttons_frame diff --git a/deeppavlov/deprecated/agents/filters/transparent_filter.py b/deeppavlov/deprecated/agents/filters/transparent_filter.py index 494ed4b8c8..1cfc8d2ae5 100644 --- a/deeppavlov/deprecated/agents/filters/transparent_filter.py +++ b/deeppavlov/deprecated/agents/filters/transparent_filter.py @@ -24,6 +24,7 @@ class TransparentFilter(Filter): Attributes: size: Number of agent skills. """ + def __init__(self, skills_count: int, *args, **kwargs) -> None: self.size: int = skills_count diff --git a/deeppavlov/deprecated/agents/processors/default_rich_content_processor.py b/deeppavlov/deprecated/agents/processors/default_rich_content_processor.py index 354fe16403..1baeb4a22f 100644 --- a/deeppavlov/deprecated/agents/processors/default_rich_content_processor.py +++ b/deeppavlov/deprecated/agents/processors/default_rich_content_processor.py @@ -18,6 +18,7 @@ class DefaultRichContentWrapper(Processor): """Returns RichControl wrapped responses with highest confidence.""" + def __init__(self, *args, **kwargs) -> None: pass diff --git a/deeppavlov/deprecated/agents/processors/highest_confidence_selector.py b/deeppavlov/deprecated/agents/processors/highest_confidence_selector.py index 8493cc5da9..21a41b85a7 100644 --- a/deeppavlov/deprecated/agents/processors/highest_confidence_selector.py +++ b/deeppavlov/deprecated/agents/processors/highest_confidence_selector.py @@ -17,6 +17,7 @@ class HighestConfidenceSelector(Processor): """Returns for each utterance response with highest confidence.""" + def __init__(self, *args, **kwargs) -> None: pass diff --git a/deeppavlov/deprecated/agents/processors/random_selector.py b/deeppavlov/deprecated/agents/processors/random_selector.py index 1b70b58218..58b5c64177 100644 --- a/deeppavlov/deprecated/agents/processors/random_selector.py +++ b/deeppavlov/deprecated/agents/processors/random_selector.py @@ -19,6 +19,7 @@ class RandomSelector(Processor): """Returns response of a random skill for each utterance.""" + def __init__(self, *args, **kwargs) -> None: pass diff --git a/deeppavlov/deprecated/agents/rich_content/default_rich_content.py b/deeppavlov/deprecated/agents/rich_content/default_rich_content.py index 5c55eee6ab..fa31c1c12d 100644 --- a/deeppavlov/deprecated/agents/rich_content/default_rich_content.py +++ b/deeppavlov/deprecated/agents/rich_content/default_rich_content.py @@ -26,6 +26,7 @@ class PlainText(RichControl): Attributes: content: Text of the message. """ + def __init__(self, text: str) -> None: super(PlainText, self).__init__('plain_text') self.content: str = text @@ -42,7 +43,7 @@ def json(self) -> dict: self.control_json['content'] = self.content return self.control_json - def ms_bot_framework(self)-> dict: + def ms_bot_framework(self) -> dict: """Returns MS Bot Framework compatible state of the PlainText instance. Creating MS Bot Framework activity blank with "text" field populated. @@ -91,6 +92,7 @@ class Button(RichControl): name: Displayed name of the button. callback: Plain text returned as callback when button pressed. """ + def __init__(self, name: str, callback: str) -> None: super(Button, self).__init__('button') self.name: str = name @@ -136,7 +138,8 @@ class ButtonsFrame(RichControl): text: Text displayed with embedded buttons. content: Container with Button objects. """ - def __init__(self, text: Optional[str]=None) -> None: + + def __init__(self, text: Optional[str] = None) -> None: super(ButtonsFrame, self).__init__('buttons_frame') self.text: [str, None] = text self.content: list = [] diff --git a/deeppavlov/deprecated/skill/skill.py b/deeppavlov/deprecated/skill/skill.py index 7e0c6aa7ac..15b46f4dbb 100644 --- a/deeppavlov/deprecated/skill/skill.py +++ b/deeppavlov/deprecated/skill/skill.py @@ -24,9 +24,10 @@ class Skill(Component, metaclass=ABCMeta): Skill is a DeepPavlov component, which provides handling dialog state, dialog history and rich content. """ + @abstractmethod def __call__(self, utterances_batch: list, history_batch: list, - states_batch: Optional[list]=None) -> Union[Tuple[list, list], Tuple[list, list, Optional[list]]]: + states_batch: Optional[list] = None) -> Union[Tuple[list, list], Tuple[list, list, Optional[list]]]: """Returns skill inference result. Returns batches of skill inference results, estimated confidence diff --git a/deeppavlov/deprecated/skills/default_skill/default_skill.py b/deeppavlov/deprecated/skills/default_skill/default_skill.py index 6894031bc7..730518d1e5 100644 --- a/deeppavlov/deprecated/skills/default_skill/default_skill.py +++ b/deeppavlov/deprecated/skills/default_skill/default_skill.py @@ -31,12 +31,13 @@ class DefaultStatelessSkill(Skill): Attributes: model: DeepPavlov model to be wrapped into default skill instance. """ - def __init__(self, model: Chainer, lang: str='en', *args, **kwargs) -> None: + + def __init__(self, model: Chainer, lang: str = 'en', *args, **kwargs) -> None: self.model = model self.proposal: str = proposals[lang] def __call__(self, utterances_batch: list, history_batch: list, - states_batch: Optional[list]=None) -> Tuple[list, list, list]: + states_batch: Optional[list] = None) -> Tuple[list, list, list]: """Returns skill inference result. Returns batches of skill inference results, estimated confidence diff --git a/deeppavlov/deprecated/skills/ecommerce_skill/bleu_retrieve.py b/deeppavlov/deprecated/skills/ecommerce_skill/bleu_retrieve.py index 00a05aaaef..4303746b47 100644 --- a/deeppavlov/deprecated/skills/ecommerce_skill/bleu_retrieve.py +++ b/deeppavlov/deprecated/skills/ecommerce_skill/bleu_retrieve.py @@ -49,7 +49,6 @@ class EcommerceSkillBleu(Skill): min_entropy: min entropy threshold for specifying """ - def __init__(self, preprocess: Component, save_path: str, @@ -74,7 +73,6 @@ def __init__(self, if kwargs.get('mode') != 'train': self.load() - def fit(self, data: List[Dict[Any, Any]]) -> None: """Preprocess items `title` and `description` from the `data` @@ -88,17 +86,15 @@ def fit(self, data: List[Dict[Any, Any]]) -> None: log.info(f"Items to nlp: {len(data)}") self.ec_data = [dict(item, **{ 'title_nlped': self.preprocess.spacy2dict(self.preprocess.analyze(item['Title'])), - 'feat_nlped': self.preprocess.spacy2dict(self.preprocess.analyze(item['Title']+'. '+item['Feature'])) + 'feat_nlped': self.preprocess.spacy2dict(self.preprocess.analyze(item['Title'] + '. ' + item['Feature'])) }) for item in data] log.info('Data are nlped') - def save(self, **kwargs) -> None: """Save classifier parameters""" log.info(f"Saving model to {self.save_path}") save_pickle(self.ec_data, self.save_path) - def load(self, **kwargs) -> None: """Load classifier parameters""" log.info(f"Loading model from {self.load_path}") @@ -110,7 +106,6 @@ def load(self, **kwargs) -> None: log.info(f"Loaded items {len(self.ec_data)}") - def __call__(self, queries: List[str], history: List[Any], states: List[Dict[Any, Any]]) -> \ Tuple[Tuple[List[Any], List[Any]], List[float], List[Any]]: """Retrieve catalog items according to the BLEU measure @@ -165,12 +160,12 @@ def __call__(self, queries: List[str], history: List[Any], states: List[Dict[Any state['Price'] = money_range score_title = [bleu_advanced(self.preprocess.lemmas(item['title_nlped']), - self.preprocess.lemmas(self.preprocess.filter_nlp_title(query)), - weights = (1,), penalty = False) for item in self.ec_data] + self.preprocess.lemmas(self.preprocess.filter_nlp_title(query)), + weights=(1,), penalty=False) for item in self.ec_data] score_feat = [bleu_advanced(self.preprocess.lemmas(item['feat_nlped']), self.preprocess.lemmas(self.preprocess.filter_nlp(query)), - weights = (0.3, 0.7), penalty = False) for idx, item in enumerate(self.ec_data)] + weights=(0.3, 0.7), penalty=False) for idx, item in enumerate(self.ec_data)] scores = np.mean([score_feat, score_title], axis=0).tolist() @@ -182,8 +177,9 @@ def __call__(self, queries: List[str], history: List[Any], states: List[Dict[Any results_args_sim = [idx for idx in results_args if scores[idx] >= self.min_similarity] - log.debug(f"Items before similarity filtering {len(results_args)} and after {len(results_args_sim)} with th={self.min_similarity} " + - f"the best one has score {scores[results_args[0]]} with title {self.ec_data[results_args[0]]['Title']}") + log.debug( + f"Items before similarity filtering {len(results_args)} and after {len(results_args_sim)} with th={self.min_similarity} " + + f"the best one has score {scores[results_args[0]]} with title {self.ec_data[results_args[0]]['Title']}") results_args_sim = self._filter_state(state, results_args_sim) @@ -202,7 +198,6 @@ def __call__(self, queries: List[str], history: List[Any], states: List[Dict[Any return (response, entropies), confidence, back_states - def _clean_items(self, results: List[int]) -> List[Any]: local_response: List = [] for idx in results: @@ -212,7 +207,6 @@ def _clean_items(self, results: List[int]) -> List[Any]: local_response.append(temp) return local_response - def _filter_state(self, state: Dict[Any, Any], results_args_sim: List[int]) -> List[Any]: for key, value in state.items(): log.debug(f"Filtering for {key}:{value}") @@ -235,7 +229,6 @@ def _filter_state(self, state: Dict[Any, Any], results_args_sim: List[int]) -> L return results_args_sim - def _entropy_subquery(self, results_args: List[int]) -> List[Tuple[float, str, List[Tuple[str, int]]]]: """Calculate entropy of selected attributes for items from the catalog. diff --git a/deeppavlov/deprecated/skills/ecommerce_skill/tfidf_retrieve.py b/deeppavlov/deprecated/skills/ecommerce_skill/tfidf_retrieve.py index a801f7e7ec..5ba516c441 100644 --- a/deeppavlov/deprecated/skills/ecommerce_skill/tfidf_retrieve.py +++ b/deeppavlov/deprecated/skills/ecommerce_skill/tfidf_retrieve.py @@ -45,7 +45,6 @@ class EcommerceSkillTfidf(Component): min_entropy: min entropy threshold for specifying """ - def __init__(self, save_path: str, load_path: str, @@ -64,7 +63,6 @@ def __init__(self, if kwargs.get('mode') != 'train': self.load() - def fit(self, data, query) -> None: """Preprocess items `title` and `description` from the `data` @@ -78,22 +76,20 @@ def fit(self, data, query) -> None: self.x_train_features = vstack(list(query)) self.ec_data = data - def save(self) -> None: """Save classifier parameters""" log.info("Saving to {}".format(self.save_path)) path = expand_path(self.save_path) save_pickle((self.ec_data, self.x_train_features), path) - def load(self) -> None: """Load classifier parameters""" log.info("Loading from {}".format(self.load_path)) self.ec_data, self.x_train_features = load_pickle( expand_path(self.load_path)) - - def __call__(self, q_vects: List[csr_matrix], histories: List[Any], states: List[Dict[Any, Any]]) -> Tuple[Tuple[List[Dict[Any, Any]], List[Any]], List[float], Dict[Any, Any]]: + def __call__(self, q_vects: List[csr_matrix], histories: List[Any], states: List[Dict[Any, Any]]) -> Tuple[ + Tuple[List[Dict[Any, Any]], List[Any]], List[float], Dict[Any, Any]]: """Retrieve catalog items according to the TFIDF measure Parameters: @@ -129,7 +125,7 @@ def __call__(self, q_vects: List[csr_matrix], histories: List[Any], states: List log.info(f"Search query {q_vect}") - if len(states) >= idx+1: + if len(states) >= idx + 1: state = states[idx] else: state = {'start': 0, 'stop': 5} @@ -184,17 +180,14 @@ def __call__(self, q_vects: List[csr_matrix], histories: List[Any], states: List entropies.append(self._entropy_subquery(answer_ids)) return (items, entropies), confidences, back_states - def _csr_to_list(self, csr: csr_matrix) -> List[Any]: return [csr.data.tolist(), csr.indices.tolist()] - def _list_to_csr(self, _list: List) -> csr_matrix: row_ind = [0] * len(_list[0]) col_ind = _list[1] return csr_matrix((_list[0], (row_ind, col_ind))) - def _take_complex_query(self, q_prev: csr_matrix, q_cur: csr_matrix) -> bool: """Decides whether to use the long compound query or the current short query @@ -217,7 +210,6 @@ def _take_complex_query(self, q_prev: csr_matrix, q_cur: csr_matrix) -> bool: return False - def _similarity(self, q_vect: Union[csr_matrix, List]) -> List[float]: """Calculates cosine similarity between the user's query and product items. @@ -229,13 +221,12 @@ def _similarity(self, q_vect: Union[csr_matrix, List]) -> List[float]: """ norm = sparse_norm(q_vect) * sparse_norm(self.x_train_features, axis=1) - cos_similarities = np.array(q_vect.dot(self.x_train_features.T).todense())/norm + cos_similarities = np.array(q_vect.dot(self.x_train_features.T).todense()) / norm cos_similarities = cos_similarities[0] cos_similarities = np.nan_to_num(cos_similarities) return cos_similarities - def _state_based_filter(self, ids: List[int], state: Dict[Any, Any]): """Filters the candidates based on the key-values from the state @@ -259,7 +250,6 @@ def _state_based_filter(self, ids: List[int], state: Dict[Any, Any]): if self.ec_data[idx][key].lower() == value.lower()] return ids - def _entropy_subquery(self, results_args: List[int]) -> List[Tuple[float, str, List[Tuple[str, int]]]]: """Calculate entropy of selected attributes for items from the catalog. diff --git a/deeppavlov/deprecated/skills/pattern_matching_skill/pattern_matching_skill.py b/deeppavlov/deprecated/skills/pattern_matching_skill/pattern_matching_skill.py index 04c0eabbf2..5fb8357d4f 100644 --- a/deeppavlov/deprecated/skills/pattern_matching_skill/pattern_matching_skill.py +++ b/deeppavlov/deprecated/skills/pattern_matching_skill/pattern_matching_skill.py @@ -44,8 +44,9 @@ class PatternMatchingSkill(Skill): ignore_case: Turns on utterances case ignoring. default_confidence: The default confidence. """ - def __init__(self, responses: List[str], patterns: Optional[List[str]]=None, - regex: bool=False, ignore_case: bool=True, default_confidence: float = 1) -> None: + + def __init__(self, responses: List[str], patterns: Optional[List[str]] = None, + regex: bool = False, ignore_case: bool = True, default_confidence: float = 1) -> None: if isinstance(responses, str): responses = [responses] self.responses = responses @@ -64,7 +65,7 @@ def __init__(self, responses: List[str], patterns: Optional[List[str]]=None, self.patterns = patterns def __call__(self, utterances_batch: list, history_batch: list, - states_batch: Optional[list]=None) -> Tuple[list, list]: + states_batch: Optional[list] = None) -> Tuple[list, list]: """Returns skill inference result. Returns batches of skill inference results, estimated confidence @@ -89,10 +90,11 @@ def __call__(self, utterances_batch: list, history_batch: list, if self.ignore_case: utterances_batch = [utterance.lower() for utterance in utterances_batch] if self.regex: - confidence = [self.default_confidence*float(any([pattern.search(utterance) for pattern in self.patterns])) - for utterance in utterances_batch] + confidence = [ + self.default_confidence * float(any([pattern.search(utterance) for pattern in self.patterns])) + for utterance in utterances_batch] else: - confidence = [self.default_confidence*float(any([pattern in utterance for pattern in self.patterns])) + confidence = [self.default_confidence * float(any([pattern in utterance for pattern in self.patterns])) for utterance in utterances_batch] return response, confidence diff --git a/deeppavlov/download.py b/deeppavlov/download.py index 6b4f48723a..e3d0d6e490 100644 --- a/deeppavlov/download.py +++ b/deeppavlov/download.py @@ -24,7 +24,8 @@ import deeppavlov from deeppavlov.core.commands.utils import expand_path, parse_config -from deeppavlov.core.data.utils import download, download_decompress, get_all_elems_from_json, file_md5, set_query_parameter, path_set_md5 +from deeppavlov.core.data.utils import download, download_decompress, get_all_elems_from_json, file_md5, \ + set_query_parameter, path_set_md5 log = getLogger(__name__) @@ -60,7 +61,7 @@ def get_config_downloads(config: Union[str, Path, dict]) -> Set[Tuple[str, Path] return downloads -def get_configs_downloads(config: Optional[Union[str, Path, dict]]=None) -> Dict[str, Set[Path]]: +def get_configs_downloads(config: Optional[Union[str, Path, dict]] = None) -> Dict[str, Set[Path]]: all_downloads = defaultdict(set) if config: @@ -105,7 +106,7 @@ def check_md5(url: str, dest_paths: List[Path]) -> bool: for base_path in not_done: log.info(f'Copying data from {done} to {base_path}') for p in expected.keys(): - shutil.copy(done/p, base_path/p) + shutil.copy(done / p, base_path / p) return True @@ -147,7 +148,7 @@ def deep_download(config: Union[str, Path, dict]) -> None: download_resource(url, dest_paths) -def main(args: Optional[List[str]]=None) -> None: +def main(args: Optional[List[str]] = None) -> None: args = parser.parse_args(args) log.info("Downloading...") download_resources(args) diff --git a/deeppavlov/evolve.py b/deeppavlov/evolve.py index ed796d33a5..206f908c97 100644 --- a/deeppavlov/evolve.py +++ b/deeppavlov/evolve.py @@ -195,7 +195,7 @@ def run_population(population, evolution, gpus): f_name = save_path / "config.json" save_json(population[i], f_name) - with save_path.joinpath('out.txt').open('w', encoding='utf8') as outlog,\ + with save_path.joinpath('out.txt').open('w', encoding='utf8') as outlog, \ save_path.joinpath('err.txt').open('w', encoding='utf8') as errlog: env = dict(os.environ) if len(gpus) > 1 or gpus[0] != -1: diff --git a/deeppavlov/metrics/bleu.py b/deeppavlov/metrics/bleu.py index e20fcc3c5b..df94c049e1 100644 --- a/deeppavlov/metrics/bleu.py +++ b/deeppavlov/metrics/bleu.py @@ -25,7 +25,7 @@ @register_metric('bleu_advanced') def bleu_advanced(y_true: List[Any], y_predicted: List[Any], - weights: Tuple=(1,), smoothing_function=SMOOTH.method1, + weights: Tuple = (1,), smoothing_function=SMOOTH.method1, auto_reweigh=False, penalty=True) -> float: """Calculate BLEU score @@ -52,7 +52,7 @@ def bleu_advanced(y_true: List[Any], y_predicted: List[Any], if penalty is True or bpenalty == 0: return bleu_measure - return bleu_measure/bpenalty + return bleu_measure / bpenalty @register_metric('bleu') @@ -78,4 +78,4 @@ def per_item_bleu(y_true, y_predicted): def per_item_dialog_bleu(y_true, y_predicted): y_true = (y['text'] for dialog in y_true for y in dialog) return corpus_bleu([[y_t.lower().split()] for y_t in y_true], - [y_p.lower().split() for y_p in y_predicted]) \ No newline at end of file + [y_p.lower().split() for y_p in y_predicted]) diff --git a/deeppavlov/metrics/elmo_metrics.py b/deeppavlov/metrics/elmo_metrics.py index 160cab88a6..e34a78e5bd 100644 --- a/deeppavlov/metrics/elmo_metrics.py +++ b/deeppavlov/metrics/elmo_metrics.py @@ -31,4 +31,3 @@ def elmo_loss2ppl(losses: List[np.ndarray]) -> float: """ avg_loss = np.mean(losses) return float(np.exp(avg_loss)) - diff --git a/deeppavlov/metrics/fmeasure.py b/deeppavlov/metrics/fmeasure.py index c915b61e47..7c2037b8fb 100644 --- a/deeppavlov/metrics/fmeasure.py +++ b/deeppavlov/metrics/fmeasure.py @@ -39,7 +39,7 @@ def ner_f1(y_true, y_predicted): @register_metric('ner_token_f1') def ner_token_f1(y_true, y_pred, print_results=False): y_true = list(chain(*y_true)) - y_pred= list(chain(*y_pred)) + y_pred = list(chain(*y_pred)) # Drop BIO or BIOES markup assert all(len(tag.split('-')) <= 2 for tag in y_true) @@ -77,7 +77,8 @@ def ner_token_f1(y_true, y_pred, print_results=False): 'f1': f1, 'n_true': n_true, 'n_pred': n_pred, 'tp': tp, 'fp': fp, 'fn': fn} - results['__total__'], accuracy, total_true_entities, total_predicted_entities, total_correct = _global_stats_f1(results) + results['__total__'], accuracy, total_true_entities, total_predicted_entities, total_correct = _global_stats_f1( + results) n_tokens = len(y_true) if print_results: log.debug('TOKEN LEVEL F1') @@ -85,7 +86,8 @@ def ner_token_f1(y_true, y_pred, print_results=False): return results['__total__']['f1'] -def _print_conll_report(results, accuracy, total_true_entities, total_predicted_entities, n_tokens, total_correct, short_report=False, entity_of_interest=None): +def _print_conll_report(results, accuracy, total_true_entities, total_predicted_entities, n_tokens, total_correct, + short_report=False, entity_of_interest=None): tags = list(results.keys()) s = 'processed {len} tokens ' \ @@ -124,12 +126,13 @@ def _print_conll_report(results, accuracy, total_true_entities, total_predicted_ tot_predicted=results[tag]['n_pred']) elif entity_of_interest is not None: s += '\t' + entity_of_interest + ': precision: {tot_prec:.2f}%; ' \ - 'recall: {tot_recall:.2f}%; ' \ - 'F1: {tot_f1:.2f} ' \ - '{tot_predicted}\n\n'.format(tot_prec=results[entity_of_interest]['precision'], - tot_recall=results[entity_of_interest]['recall'], - tot_f1=results[entity_of_interest]['f1'], - tot_predicted=results[entity_of_interest]['n_pred']) + 'recall: {tot_recall:.2f}%; ' \ + 'F1: {tot_f1:.2f} ' \ + '{tot_predicted}\n\n'.format(tot_prec=results[entity_of_interest]['precision'], + tot_recall=results[entity_of_interest]['recall'], + tot_f1=results[entity_of_interest]['f1'], + tot_predicted=results[entity_of_interest][ + 'n_pred']) log.debug(s) @@ -381,11 +384,12 @@ def precision_recall_f1(y_true, y_pred, print_results=True, short_report=False, tot_predicted=results[tag]['n_pred']) elif entity_of_interest is not None: s += '\t' + entity_of_interest + ': precision: {tot_prec:.2f}%; ' \ - 'recall: {tot_recall:.2f}%; ' \ - 'F1: {tot_f1:.2f} ' \ - '{tot_predicted}\n\n'.format(tot_prec=results[entity_of_interest]['precision'], - tot_recall=results[entity_of_interest]['recall'], - tot_f1=results[entity_of_interest]['f1'], - tot_predicted=results[entity_of_interest]['n_pred']) + 'recall: {tot_recall:.2f}%; ' \ + 'F1: {tot_f1:.2f} ' \ + '{tot_predicted}\n\n'.format( + tot_prec=results[entity_of_interest]['precision'], + tot_recall=results[entity_of_interest]['recall'], + tot_f1=results[entity_of_interest]['f1'], + tot_predicted=results[entity_of_interest]['n_pred']) log.debug(s) return results diff --git a/deeppavlov/metrics/google_bleu.py b/deeppavlov/metrics/google_bleu.py index 9fe6466ad4..f94a09cc99 100644 --- a/deeppavlov/metrics/google_bleu.py +++ b/deeppavlov/metrics/google_bleu.py @@ -26,87 +26,87 @@ def _get_ngrams(segment, max_order): - """Extracts all n-grams upto a given maximum order from an input segment. + """Extracts all n-grams upto a given maximum order from an input segment. - Args: - segment: text segment from which n-grams will be extracted. - max_order: maximum length in tokens of the n-grams returned by this - methods. + Args: + segment: text segment from which n-grams will be extracted. + max_order: maximum length in tokens of the n-grams returned by this + methods. - Returns: - The Counter containing all n-grams upto max_order in segment - with a count of how many times each n-gram occurred. - """ - ngram_counts = collections.Counter() - for order in range(1, max_order + 1): - for i in range(0, len(segment) - order + 1): - ngram = tuple(segment[i:i+order]) - ngram_counts[ngram] += 1 - return ngram_counts + Returns: + The Counter containing all n-grams upto max_order in segment + with a count of how many times each n-gram occurred. + """ + ngram_counts = collections.Counter() + for order in range(1, max_order + 1): + for i in range(0, len(segment) - order + 1): + ngram = tuple(segment[i:i + order]) + ngram_counts[ngram] += 1 + return ngram_counts def compute_bleu(reference_corpus, translation_corpus, max_order=4, smooth=False): - """Computes BLEU score of translated segments against one or more references. - - Args: - reference_corpus: list of lists of references for each translation. Each - reference should be tokenized into a list of tokens. - translation_corpus: list of translations to score. Each translation - should be tokenized into a list of tokens. - max_order: Maximum n-gram order to use when computing BLEU score. - smooth: Whether or not to apply Lin et al. 2004 smoothing. - - Returns: - 3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram - precisions and brevity penalty. - """ - matches_by_order = [0] * max_order - possible_matches_by_order = [0] * max_order - reference_length = 0 - translation_length = 0 - for (references, translation) in zip(reference_corpus, - translation_corpus): - reference_length += min(len(r) for r in references) - translation_length += len(translation) - - merged_ref_ngram_counts = collections.Counter() - for reference in references: - merged_ref_ngram_counts |= _get_ngrams(reference, max_order) - translation_ngram_counts = _get_ngrams(translation, max_order) - overlap = translation_ngram_counts & merged_ref_ngram_counts - for ngram in overlap: - matches_by_order[len(ngram)-1] += overlap[ngram] - for order in range(1, max_order+1): - possible_matches = len(translation) - order + 1 - if possible_matches > 0: - possible_matches_by_order[order-1] += possible_matches - - precisions = [0] * max_order - for i in range(0, max_order): - if smooth: - precisions[i] = ((matches_by_order[i] + 1.) / - (possible_matches_by_order[i] + 1.)) + """Computes BLEU score of translated segments against one or more references. + + Args: + reference_corpus: list of lists of references for each translation. Each + reference should be tokenized into a list of tokens. + translation_corpus: list of translations to score. Each translation + should be tokenized into a list of tokens. + max_order: Maximum n-gram order to use when computing BLEU score. + smooth: Whether or not to apply Lin et al. 2004 smoothing. + + Returns: + 3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram + precisions and brevity penalty. + """ + matches_by_order = [0] * max_order + possible_matches_by_order = [0] * max_order + reference_length = 0 + translation_length = 0 + for (references, translation) in zip(reference_corpus, + translation_corpus): + reference_length += min(len(r) for r in references) + translation_length += len(translation) + + merged_ref_ngram_counts = collections.Counter() + for reference in references: + merged_ref_ngram_counts |= _get_ngrams(reference, max_order) + translation_ngram_counts = _get_ngrams(translation, max_order) + overlap = translation_ngram_counts & merged_ref_ngram_counts + for ngram in overlap: + matches_by_order[len(ngram) - 1] += overlap[ngram] + for order in range(1, max_order + 1): + possible_matches = len(translation) - order + 1 + if possible_matches > 0: + possible_matches_by_order[order - 1] += possible_matches + + precisions = [0] * max_order + for i in range(0, max_order): + if smooth: + precisions[i] = ((matches_by_order[i] + 1.) / + (possible_matches_by_order[i] + 1.)) + else: + if possible_matches_by_order[i] > 0: + precisions[i] = (float(matches_by_order[i]) / + possible_matches_by_order[i]) + else: + precisions[i] = 0.0 + + if min(precisions) > 0: + p_log_sum = sum((1. / max_order) * math.log(p) for p in precisions) + geo_mean = math.exp(p_log_sum) else: - if possible_matches_by_order[i] > 0: - precisions[i] = (float(matches_by_order[i]) / - possible_matches_by_order[i]) - else: - precisions[i] = 0.0 + geo_mean = 0 - if min(precisions) > 0: - p_log_sum = sum((1. / max_order) * math.log(p) for p in precisions) - geo_mean = math.exp(p_log_sum) - else: - geo_mean = 0 + ratio = float(translation_length) / reference_length - ratio = float(translation_length) / reference_length - - if ratio > 1.0: - bp = 1. - else: - bp = math.exp(1 - 1. / ratio) + if ratio > 1.0: + bp = 1. + else: + bp = math.exp(1 - 1. / ratio) - bleu = geo_mean * bp + bleu = geo_mean * bp - return (bleu, precisions, bp, ratio, translation_length, reference_length) + return (bleu, precisions, bp, ratio, translation_length, reference_length) diff --git a/deeppavlov/metrics/recall_at_k.py b/deeppavlov/metrics/recall_at_k.py index bafc2f7332..a9525b3eb4 100644 --- a/deeppavlov/metrics/recall_at_k.py +++ b/deeppavlov/metrics/recall_at_k.py @@ -42,6 +42,7 @@ def recall_at_k(y_true: List[int], y_pred: List[List[np.ndarray]], k: int): num_correct += 1 return float(num_correct) / num_examples + @register_metric('r@1') def r_at_1(y_true, y_pred): return recall_at_k(y_true, y_pred, k=1) @@ -56,6 +57,7 @@ def r_at_2(y_true, y_pred): def r_at_5(labels, predictions): return recall_at_k(labels, predictions, k=5) + @register_metric('r@10') def r_at_10(labels, predictions): return recall_at_k(labels, predictions, k=10) diff --git a/deeppavlov/models/api_requester/api_requester.py b/deeppavlov/models/api_requester/api_requester.py index 65bf689131..df87db4523 100644 --- a/deeppavlov/models/api_requester/api_requester.py +++ b/deeppavlov/models/api_requester/api_requester.py @@ -23,7 +23,8 @@ class ApiRequester(Component): param_names: list of parameter names for API requests. debatchify: if True, single instances will be sent to the API endpoint instead of batches. """ - def __init__(self, url: str, out: [int, list], param_names: [list, tuple]=(), debatchify: bool=False, + + def __init__(self, url: str, out: [int, list], param_names: [list, tuple] = (), debatchify: bool = False, *args, **kwargs): self.url = url self.param_names = param_names diff --git a/deeppavlov/models/api_requester/api_router.py b/deeppavlov/models/api_requester/api_router.py index a59c255bc5..71f636cd2b 100644 --- a/deeppavlov/models/api_requester/api_router.py +++ b/deeppavlov/models/api_requester/api_router.py @@ -23,7 +23,7 @@ class ApiRouter(Component): n_workers: The maximum number of subprocesses to run """ - def __init__(self, api_requesters: List[ApiRequester], n_workers: int=1, *args, **kwargs): + def __init__(self, api_requesters: List[ApiRequester], n_workers: int = 1, *args, **kwargs): self.api_requesters = api_requesters self.n_workers = n_workers diff --git a/deeppavlov/models/bert/bert_classifier.py b/deeppavlov/models/bert/bert_classifier.py index d5c7b616e9..a22307046a 100644 --- a/deeppavlov/models/bert/bert_classifier.py +++ b/deeppavlov/models/bert/bert_classifier.py @@ -48,6 +48,7 @@ class BertClassifierModel(LRScheduledTFModel): pretrained_bert: pretrained Bert checkpoint min_learning_rate: min value of learning rate if learning rate decay is used """ + # TODO: add warmup # TODO: add head-only pre-training def __init__(self, bert_config_file, n_classes, keep_prob, @@ -153,7 +154,7 @@ def _init_placeholders(self): self.token_types_ph = tf.placeholder(shape=(None, None), dtype=tf.int32, name='token_types_ph') if not self.one_hot_labels: - self.y_ph = tf.placeholder(shape=(None, ), dtype=tf.int32, name='y_ph') + self.y_ph = tf.placeholder(shape=(None,), dtype=tf.int32, name='y_ph') else: self.y_ph = tf.placeholder(shape=(None, self.n_classes), dtype=tf.float32, name='y_ph') diff --git a/deeppavlov/models/bert/bert_ranker.py b/deeppavlov/models/bert/bert_ranker.py index 4bed99afca..d01923e417 100644 --- a/deeppavlov/models/bert/bert_ranker.py +++ b/deeppavlov/models/bert/bert_ranker.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from logging import getLogger -from typing import List, Dict, Union -from collections import OrderedDict import re +from collections import OrderedDict +from logging import getLogger from operator import itemgetter +from typing import List, Dict, Union import numpy as np import tensorflow as tf @@ -26,8 +26,8 @@ from deeppavlov.core.commands.utils import expand_path from deeppavlov.core.common.registry import register -from deeppavlov.models.bert.bert_classifier import BertClassifierModel from deeppavlov.core.models.tf_model import LRScheduledTFModel +from deeppavlov.models.bert.bert_classifier import BertClassifierModel logger = getLogger(__name__) @@ -301,7 +301,6 @@ def train_on_batch(self, features_li: List[List[InputFeatures]], y: Union[List[i _, loss = self.sess.run([self.train_op, self.loss], feed_dict=feed_dict) return {'loss': loss, 'learning_rate': feed_dict[self.learning_rate_ph]} - def __call__(self, features_li: List[List[InputFeatures]]) -> Union[List[int], List[List[float]]]: """Calculate scores for the given context over candidate responses. @@ -362,8 +361,8 @@ class BertSepRankerPredictor(BertSepRankerModel): """ def __init__(self, bert_config_file, interact_mode=0, batch_size=32, - resps=None, resp_features=None, resp_vecs=None, - conts=None, cont_features=None, cont_vecs=None, **kwargs) -> None: + resps=None, resp_features=None, resp_vecs=None, + conts=None, cont_features=None, cont_vecs=None, **kwargs) -> None: super().__init__(bert_config_file=bert_config_file, **kwargs) diff --git a/deeppavlov/models/bert/bert_squad.py b/deeppavlov/models/bert/bert_squad.py index 7f5cfd6491..c0fa1550aa 100644 --- a/deeppavlov/models/bert/bert_squad.py +++ b/deeppavlov/models/bert/bert_squad.py @@ -137,7 +137,7 @@ def _init_graph(self): logit_mask = self.token_types_ph # [CLS] token is used as no answer - mask = tf.concat([tf.ones((bs, 1), dtype=tf.int32), tf.zeros((bs, seq_len-1), dtype=tf.int32)], axis=-1) + mask = tf.concat([tf.ones((bs, 1), dtype=tf.int32), tf.zeros((bs, seq_len - 1), dtype=tf.int32)], axis=-1) logit_mask = logit_mask + mask logits_st = softmax_mask(logits_st, logit_mask) @@ -258,7 +258,8 @@ def __call__(self, features: List[InputFeatures]) -> Tuple[List[int], List[int], input_type_ids = [f.input_type_ids for f in features] feed_dict = self._build_feed_dict(input_ids, input_masks, input_type_ids) - st, end, logits, scores = self.sess.run([self.start_pred, self.end_pred, self.yp_logits, self.yp_score], feed_dict=feed_dict) + st, end, logits, scores = self.sess.run([self.start_pred, self.end_pred, self.yp_logits, self.yp_score], + feed_dict=feed_dict) return st, end, logits.tolist(), scores.tolist() @@ -288,6 +289,7 @@ class BertSQuADInferModel(Component): lang: either `en` or `ru`, it is used to select sentence tokenizer """ + def __init__(self, squad_model_config: str, vocab_file: str, do_lower_case: bool, diff --git a/deeppavlov/models/classifiers/keras_classification_model.py b/deeppavlov/models/classifiers/keras_classification_model.py index bcde5df3d3..865ef811a9 100644 --- a/deeppavlov/models/classifiers/keras_classification_model.py +++ b/deeppavlov/models/classifiers/keras_classification_model.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from copy import deepcopy +from logging import getLogger from pathlib import Path from typing import List, Tuple, Optional, Generator, Union -from logging import getLogger -from copy import deepcopy -import numpy as np import keras.metrics import keras.optimizers +import numpy as np from keras import backend as K from keras.layers import Dense, Input from keras.layers import concatenate, Activation, Concatenate, Reshape @@ -36,8 +36,8 @@ from deeppavlov.core.common.errors import ConfigError from deeppavlov.core.common.file import save_json, read_json from deeppavlov.core.common.registry import register -from deeppavlov.core.models.keras_model import LRScheduledKerasModel from deeppavlov.core.layers.keras_layers import additive_self_attention, multiplicative_self_attention +from deeppavlov.core.models.keras_model import LRScheduledKerasModel log = getLogger(__name__) @@ -284,7 +284,8 @@ def _load(self, model_name: str) -> None: try: model.load_weights(str(weights_path)) except ValueError: - raise ConfigError("Some non-changable parameters of neural network differ from given pre-trained model") + raise ConfigError( + "Some non-changable parameters of neural network differ from given pre-trained model") self.model = model @@ -378,7 +379,8 @@ def save(self, fname: str = None) -> None: self.opt["epochs_done"] = self.epochs_done if isinstance(self.opt.get("learning_rate", None), float): self.opt["final_learning_rate"] = K.eval(self.optimizer.lr) / (1. + - K.eval(self.optimizer.decay) * self.batches_seen) + K.eval( + self.optimizer.decay) * self.batches_seen) if self.opt.get("load_path") and self.opt.get("save_path"): if self.opt.get("save_path") != self.opt.get("load_path"): diff --git a/deeppavlov/models/classifiers/ru_obscenity_classifier.py b/deeppavlov/models/classifiers/ru_obscenity_classifier.py index d22ead8fa1..2c3ef26d36 100644 --- a/deeppavlov/models/classifiers/ru_obscenity_classifier.py +++ b/deeppavlov/models/classifiers/ru_obscenity_classifier.py @@ -1,16 +1,14 @@ -from typing import List, Union -from pathlib import Path -from logging import getLogger -import errno import json import re -import os +from logging import getLogger +from pathlib import Path +from typing import List, Union import pymorphy2 +from deeppavlov.core.commands.utils import expand_path from deeppavlov.core.common.registry import register from deeppavlov.core.models.estimator import Component -from deeppavlov.core.commands.utils import expand_path log = getLogger(__name__) @@ -107,14 +105,14 @@ def _check_obscenity(self, text: str) -> bool: word = word.lower() word.replace('ё', 'е') normal_word = self.morph.parse(word)[0].normal_form - if normal_word in self.obscenity_words_exception\ + if normal_word in self.obscenity_words_exception \ or word in self.obscenity_words_exception: continue - if normal_word in self.obscenity_words\ - or word in self.obscenity_words\ - or bool(self.regexp.findall(normal_word))\ - or bool(self.regexp.findall(word))\ - or bool(self.regexp2.findall(normal_word))\ + if normal_word in self.obscenity_words \ + or word in self.obscenity_words \ + or bool(self.regexp.findall(normal_word)) \ + or bool(self.regexp.findall(word)) \ + or bool(self.regexp2.findall(normal_word)) \ or bool(self.regexp2.findall(word)): return True return False diff --git a/deeppavlov/models/classifiers/utils.py b/deeppavlov/models/classifiers/utils.py index 03e6feabc0..14b01e960c 100644 --- a/deeppavlov/models/classifiers/utils.py +++ b/deeppavlov/models/classifiers/utils.py @@ -21,7 +21,7 @@ log = getLogger(__name__) -def labels2onehot(labels: [List[str], List[List[str]], np.ndarray], classes: [list, np.ndarray]) -> np.ndarray: +def labels2onehot(labels: [List[str], List[List[str]], np.ndarray], classes: [list, np.ndarray]) -> np.ndarray: """ Convert labels to one-hot vectors for multi-class multi-label classification @@ -49,7 +49,7 @@ def labels2onehot(labels: [List[str], List[List[str]], np.ndarray], classes: [l return y -def proba2labels(proba: [list, np.ndarray], confident_threshold: float, classes: [list, np.ndarray]) -> List[List]: +def proba2labels(proba: [list, np.ndarray], confident_threshold: float, classes: [list, np.ndarray]) -> List[List]: """ Convert vectors of probabilities to labels using confident threshold (if probability to belong with the class is bigger than confident_threshold, sample belongs with the class; @@ -74,7 +74,7 @@ def proba2labels(proba: [list, np.ndarray], confident_threshold: float, classes: return y -def proba2onehot(proba: [list, np.ndarray], confident_threshold: float, classes: [list, np.ndarray]) -> np.ndarray: +def proba2onehot(proba: [list, np.ndarray], confident_threshold: float, classes: [list, np.ndarray]) -> np.ndarray: """ Convert vectors of probabilities to one-hot representations using confident threshold diff --git a/deeppavlov/models/elmo/bilm_model.py b/deeppavlov/models/elmo/bilm_model.py index 665bd2ca86..cc7eacb8b0 100644 --- a/deeppavlov/models/elmo/bilm_model.py +++ b/deeppavlov/models/elmo/bilm_model.py @@ -48,6 +48,7 @@ class LanguageModel(object): 'dim' is the hidden state size. Set 'dim' == 'projection_dim' to skip a projection layer. """ + def __init__(self, options, is_training): self.options = options self.is_training = is_training @@ -285,14 +286,14 @@ def high(x, ww_carry, bb_carry, ww_tr, bb_tr): W_carry, b_carry, W_transform, b_transform) self.token_embedding_layers.append(tf.reshape(embedding, - [batch_size, unroll_steps, highway_dim])) + [batch_size, unroll_steps, highway_dim])) # finally project down to projection dim if needed if use_proj: embedding = tf.matmul(embedding, W_proj_cnn) + b_proj_cnn if self.bidirectional: embedding_reverse = tf.matmul(embedding_reverse, W_proj_cnn) \ - + b_proj_cnn + + b_proj_cnn self.token_embedding_layers.append( tf.reshape(embedding, [batch_size, unroll_steps, projection_dim]) ) diff --git a/deeppavlov/models/elmo/elmo.py b/deeppavlov/models/elmo/elmo.py index cb9b28c647..f197ae7c15 100644 --- a/deeppavlov/models/elmo/elmo.py +++ b/deeppavlov/models/elmo/elmo.py @@ -214,7 +214,7 @@ def __init__(self, unroll_steps: Optional[int] = None, n_tokens_vocab: Optional[int] = None, lstm: Optional[dict] = None, - dropout: Optional[float] = None, # Regularization + dropout: Optional[float] = None, # Regularization n_negative_samples_batch: Optional[int] = None, # Train options all_clip_norm_val: Optional[float] = None, initial_accumulator_value: float = 1.0, @@ -230,10 +230,10 @@ def __init__(self, **kwargs) -> None: # ================ Checking input args ================= - if not(options_json_path or (char_cnn and bidirectional and unroll_steps - and n_tokens_vocab and lstm and dropout and - n_negative_samples_batch and all_clip_norm_val - )): + if not (options_json_path or (char_cnn and bidirectional and unroll_steps + and n_tokens_vocab and lstm and dropout and + n_negative_samples_batch and all_clip_norm_val + )): raise Warning('Use options_json_path or/and direct params to set net architecture.') self.options = self._load_options(options_json_path) self._update_arch_options(char_cnn, bidirectional, unroll_steps, n_tokens_vocab, lstm) @@ -250,7 +250,7 @@ def __init__(self, self.train_options = {} self.valid_options = {'batch_size': 256, 'unroll_steps': 1, 'n_gpus': 1} - self.model_mode='' + self.model_mode = '' tf.set_random_seed(seed) np.random.seed(seed) @@ -306,7 +306,7 @@ def _update_other_options(self, dropout, n_negative_samples_batch, all_clip_norm if all_clip_norm_val is not None: self.options['all_clip_norm_val'] = all_clip_norm_val - def _get_epoch_from(self, epoch_load_path, default = 0): + def _get_epoch_from(self, epoch_load_path, default=0): path = self.load_path path = path.parent / epoch_load_path candidates = path.resolve().glob('[0-9]*') @@ -315,7 +315,7 @@ def _get_epoch_from(self, epoch_load_path, default = 0): epoch_num = max(candidates, default=default) return epoch_num - def _build_graph(self, graph, train = True): + def _build_graph(self, graph, train=True): with graph.as_default(): with tf.device('/cpu:0'): init_step = 0 @@ -417,8 +417,8 @@ def _init_session(self): def _fill_feed_dict(self, char_ids_batches, reversed_char_ids_batches, - token_ids_batches = None, - reversed_token_ids_batches = None): + token_ids_batches=None, + reversed_token_ids_batches=None): # init state tensors feed_dict = {t: v for t, v in zip(self.init_state_tensors, self.init_state_values)} @@ -475,7 +475,7 @@ def load(self, epoch: Optional[int] = None) -> None: saver = tf.train.Saver() saver.restore(self.sess, path) else: - log.info(f'[A checkpoint not found in {path}]') + log.info(f'[A checkpoint not found in {path}]') @overrides def save(self, epoch: Optional[int] = None) -> None: @@ -540,7 +540,7 @@ def _build_model(self, train: bool, epoch: Optional[int] = None, **kwargs): train=False) with self.graph.as_default(): - self.init_state_values, self.init_state_tensors, self.final_state_tensors =\ + self.init_state_values, self.init_state_tensors, self.final_state_tensors = \ self._init_session() self.load(epoch) diff --git a/deeppavlov/models/elmo/elmo2tfhub.py b/deeppavlov/models/elmo/elmo2tfhub.py index 72276707d5..a304bf6837 100644 --- a/deeppavlov/models/elmo/elmo2tfhub.py +++ b/deeppavlov/models/elmo/elmo2tfhub.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import shutil + import numpy as np import tensorflow as tf import tensorflow_hub as hub -import shutil from deeppavlov.models.elmo.elmo_model import BidirectionalLanguageModel, weight_layers @@ -30,6 +31,7 @@ def make_module_spec(options, weight_file): Returns: A module spec object used for constructing a TF-Hub module. """ + def module_fn(): """Spec function for a token embedding module.""" # init @@ -89,7 +91,7 @@ def sentence_tagging_and_padding(sen_dim): # Input placeholders to the biLM. tokens = tf.placeholder(shape=(None, None), dtype=tf.string, name='ph2tokens') - sequence_len = tf.placeholder(shape=(None, ), dtype=tf.int32, name='ph2sequence_len') + sequence_len = tf.placeholder(shape=(None,), dtype=tf.int32, name='ph2sequence_len') tok_shape = tf.shape(tokens) line_tokens = tf.reshape(tokens, shape=[-1], name='reshape2line_tokens') diff --git a/deeppavlov/models/elmo/elmo_model.py b/deeppavlov/models/elmo/elmo_model.py index 000af987e6..8e475dcedb 100644 --- a/deeppavlov/models/elmo/elmo_model.py +++ b/deeppavlov/models/elmo/elmo_model.py @@ -142,7 +142,7 @@ def _build_ops(self, lm_graph): for layer in layers: layer_wo_bos_eos = layer[:, 1:, :] layer_wo_bos_eos = tf.reverse_sequence( - layer_wo_bos_eos, + layer_wo_bos_eos, lm_graph.sequence_lengths - 1, seq_axis=1, batch_axis=0, @@ -182,7 +182,7 @@ def _build_ops(self, lm_graph): mask_wo_bos_eos = tf.cast(mask_wo_bos_eos, 'bool') return { - 'lm_embeddings': lm_embeddings, + 'lm_embeddings': lm_embeddings, 'lengths': sequence_length_wo_bos_eos, 'token_embeddings': lm_graph.embedding, 'mask': mask_wo_bos_eos, @@ -253,6 +253,7 @@ class BidirectionalLanguageModelGraph(object): Creates the computational graph and holds the ops necessary for runnint a bidirectional language model """ + def __init__(self, options, weight_file, ids_placeholder, use_character_inputs=True, embedding_weight_file=None, max_batch_size=128): @@ -405,15 +406,15 @@ def make_convolutions(inp): if use_proj: assert n_filters > projection_dim with tf.variable_scope('CNN_proj'): - W_proj_cnn = tf.get_variable( - "W_proj", [n_filters, projection_dim], - initializer=tf.random_normal_initializer( - mean=0.0, stddev=np.sqrt(1.0 / n_filters)), - dtype=DTYPE) - b_proj_cnn = tf.get_variable( - "b_proj", [projection_dim], - initializer=tf.constant_initializer(0.0), - dtype=DTYPE) + W_proj_cnn = tf.get_variable( + "W_proj", [n_filters, projection_dim], + initializer=tf.random_normal_initializer( + mean=0.0, stddev=np.sqrt(1.0 / n_filters)), + dtype=DTYPE) + b_proj_cnn = tf.get_variable( + "b_proj", [projection_dim], + initializer=tf.constant_initializer(0.0), + dtype=DTYPE) # apply highways layers def high(x, ww_carry, bb_carry, ww_tr, bb_tr): @@ -586,7 +587,7 @@ def _build_lstms(self): init_states[i][batch_size:, :]], axis=0) state_update_op = tf.assign(init_states[i], new_state) update_ops.append(state_update_op) - + layer_input = layer_output self.mask = mask @@ -623,6 +624,7 @@ def weight_layers(name, bilm_ops, l2_coef=None, 'regularization_op': op to compute regularization term } """ + def _l2_regularizer(weights): if l2_coef is not None: return l2_coef * tf.reduce_sum(tf.square(weights)) @@ -647,7 +649,7 @@ def _do_ln(x): x_masked = x * broadcast_mask N = tf.reduce_sum(mask_float) * lm_dim mean = tf.reduce_sum(x_masked) / N - variance = tf.reduce_sum(((x_masked - mean) * broadcast_mask)**2) / N + variance = tf.reduce_sum(((x_masked - mean) * broadcast_mask) ** 2) / N return tf.nn.batch_normalization( x, mean, variance, None, None, 1E-12 ) @@ -662,7 +664,7 @@ def _do_ln(x): with tf.variable_scope("aggregation", reuse=reuse): W = tf.get_variable( '{}_ELMo_W'.format(name), - shape=(n_lm_layers, ), + shape=(n_lm_layers,), initializer=tf.zeros_initializer, regularizer=_l2_regularizer, trainable=True, @@ -697,7 +699,7 @@ def _do_ln(x): with tf.variable_scope("aggregation", reuse=reuse): gamma = tf.get_variable( '{}_ELMo_gamma'.format(name), - shape=(1, ), + shape=(1,), initializer=tf.ones_initializer, regularizer=None, trainable=True, diff --git a/deeppavlov/models/embedders/abstract_embedder.py b/deeppavlov/models/embedders/abstract_embedder.py index 7a297212f4..c9a52c2b70 100644 --- a/deeppavlov/models/embedders/abstract_embedder.py +++ b/deeppavlov/models/embedders/abstract_embedder.py @@ -43,6 +43,7 @@ class Embedder(Component, Serializable, metaclass=ABCMeta): mean: whether to return one mean embedding vector per sample load_path: path with pre-trained fastText binary model """ + def __init__(self, load_path: Union[str, Path], pad_zero: bool = False, mean: bool = False, **kwargs) -> None: """ Initialize embedder with given parameters diff --git a/deeppavlov/models/embedders/elmo_embedder.py b/deeppavlov/models/embedders/elmo_embedder.py index a6dd603bb6..09990ce648 100644 --- a/deeppavlov/models/embedders/elmo_embedder.py +++ b/deeppavlov/models/embedders/elmo_embedder.py @@ -131,6 +131,7 @@ class ELMoEmbedder(Component, metaclass=TfModelMeta): """ + def __init__(self, spec: str, elmo_output_names: Optional[List] = None, dim: Optional[int] = None, pad_zero: bool = False, concat_last_axis: bool = True, max_token: Optional[int] = None, @@ -224,7 +225,7 @@ def _fill_batch(self, batch): batch = [batch_line[:self.max_token] for batch_line in batch] tokens_length = [len(batch_line) for batch_line in batch] tokens_length_max = max(tokens_length) - batch = [batch_line + ['']*(tokens_length_max - len(batch_line)) for batch_line in batch] + batch = [batch_line + [''] * (tokens_length_max - len(batch_line)) for batch_line in batch] return batch, tokens_length diff --git a/deeppavlov/models/embedders/tfidf_weighted_embedder.py b/deeppavlov/models/embedders/tfidf_weighted_embedder.py index c80a72601c..880138de33 100644 --- a/deeppavlov/models/embedders/tfidf_weighted_embedder.py +++ b/deeppavlov/models/embedders/tfidf_weighted_embedder.py @@ -197,7 +197,8 @@ def __call__(self, batch: List[List[str]], tags_batch: Optional[List[List[str]]] if self.tags_vocab: if tags_batch is None: raise ConfigError("TfidfWeightedEmbedder got 'tags_vocab_path' but __call__ did not get tags_batch.") - batch = [self._tags_encode(sample, tags_sample, mean=mean) for sample, tags_sample in zip(batch, tags_batch)] + batch = [self._tags_encode(sample, tags_sample, mean=mean) for sample, tags_sample in + zip(batch, tags_batch)] else: if tags_batch: raise ConfigError("TfidfWeightedEmbedder got tags batch, but 'tags_vocab_path' is empty.") @@ -301,4 +302,3 @@ def _tags_encode(self, tokens: List[str], tags: List[str], mean: bool) -> Union[ embedded_tokens = np.array([weights[i] * embedded_tokens[i] for i in range(len(tokens))]) return embedded_tokens - diff --git a/deeppavlov/models/go_bot/network.py b/deeppavlov/models/go_bot/network.py index e7c304f95d..1401aaeae6 100644 --- a/deeppavlov/models/go_bot/network.py +++ b/deeppavlov/models/go_bot/network.py @@ -13,9 +13,9 @@ # limitations under the License. import collections +import copy import json import re -import copy from logging import getLogger from typing import Dict, Any, List, Optional, Union, Tuple @@ -282,7 +282,7 @@ def _encode_context(self, # random embedding instead of zeros if np.all(emb_features < 1e-20): emb_dim = self.embedder.dim - emb_features = np.fabs(np.random.normal(0, 1/emb_dim, emb_dim)) + emb_features = np.fabs(np.random.normal(0, 1 / emb_dim, emb_dim)) # Intent features intent_features = [] @@ -321,11 +321,11 @@ def _encode_context(self, if self.debug: log.debug(f"Context features = {context_features}") - debug_msg = f"num bow features = {bow_features}" +\ - f", num emb features = {emb_features}" +\ - f", num intent features = {intent_features}" +\ - f", num state features = {len(state_features)}" +\ - f", num context features = {len(context_features)}" +\ + debug_msg = f"num bow features = {bow_features}" + \ + f", num emb features = {emb_features}" + \ + f", num intent features = {intent_features}" + \ + f", num state features = {len(state_features)}" + \ + f", num context features = {len(context_features)}" + \ f", prev_action shape = {len(state['prev_action'])}" log.debug(debug_msg) @@ -559,7 +559,7 @@ def network_call(self, feed_dict[self._emb_context] = emb_context feed_dict[self._key] = key - probs, prediction, state =\ + probs, prediction, state = \ self.sess.run([self._probs, self._prediction, self._state], feed_dict=feed_dict) diff --git a/deeppavlov/models/go_bot/templates.py b/deeppavlov/models/go_bot/templates.py index 1b5870ba34..e6cb49d663 100644 --- a/deeppavlov/models/go_bot/templates.py +++ b/deeppavlov/models/go_bot/templates.py @@ -87,13 +87,13 @@ def update(self, default="", dontcare=""): self.dontcare = self.dontcare or dontcare def __contains__(self, t): - return t.default and (t.default == self.default)\ - or t.dontcare and (t.dontcare == self.dontcare) + return t.default and (t.default == self.default) \ + or t.dontcare and (t.dontcare == self.dontcare) def __eq__(self, other): if isinstance(other, self.__class__): - return (self.default == other.default)\ - and (self.dontcare == other.dontcare) + return (self.default == other.default) \ + and (self.dontcare == other.dontcare) return False def __hash__(self): diff --git a/deeppavlov/models/go_bot/tracker.py b/deeppavlov/models/go_bot/tracker.py index 9b16a0220f..bcf7c7f866 100644 --- a/deeppavlov/models/go_bot/tracker.py +++ b/deeppavlov/models/go_bot/tracker.py @@ -64,6 +64,7 @@ class DefaultTracker(Tracker): Parameters: slot_names: list of slots that should be tracked. """ + def __init__(self, slot_names: List[str]) -> None: self.slot_names = list(slot_names) self.reset_state() @@ -83,6 +84,7 @@ def reset_state(self): def update_state(self, slots): def _filter(slots): return filter(lambda s: s[0] in self.slot_names, slots) + if isinstance(slots, list): self.history.extend(_filter(slots)) elif isinstance(slots, dict): @@ -120,6 +122,7 @@ class FeaturizedTracker(Tracker): Parameters: slot_names: list of slots that should be tracked. """ + def __init__(self, slot_names: List[str]) -> None: self.slot_names = list(slot_names) self.reset_state() @@ -139,6 +142,7 @@ def reset_state(self): def update_state(self, slots): def _filter(slots): return filter(lambda s: s[0] in self.slot_names, slots) + prev_state = self.get_state() if isinstance(slots, list): self.history.extend(_filter(slots)) @@ -174,7 +178,7 @@ def _diff_features(self, state): feats = np.zeros(self.state_size, dtype=np.float32) curr_state = self.get_state() for i, slot in enumerate(self.slot_names): - if (slot in curr_state) and (slot in state) and\ + if (slot in curr_state) and (slot in state) and \ (curr_state[slot] != state[slot]): feats[i] = 1. return feats diff --git a/deeppavlov/models/kbqa/entity_linking.py b/deeppavlov/models/kbqa/entity_linking.py index 90090fe206..0fd9408aba 100644 --- a/deeppavlov/models/kbqa/entity_linking.py +++ b/deeppavlov/models/kbqa/entity_linking.py @@ -11,18 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import itertools import pickle from logging import getLogger from typing import List, Dict, Tuple, Optional -import itertools -from fuzzywuzzy import fuzz -import pymorphy2 import nltk +import pymorphy2 +from fuzzywuzzy import fuzz from deeppavlov.core.common.registry import register -from deeppavlov.core.models.serializable import Serializable from deeppavlov.core.models.component import Component +from deeppavlov.core.models.serializable import Serializable from deeppavlov.models.spelling_correction.levenshtein.levenshtein_searcher import LevenshteinSearcher log = getLogger(__name__) @@ -149,7 +149,7 @@ def _log_entities(self, srtd_cand_ent): entities_to_print = [] for name, q, ratio, n_rel in srtd_cand_ent: entities_to_print.append(f'{name}, http://wikidata.org/wiki/{q}, {ratio}, {n_rel}') - log.debug('\n'+'\n'.join(entities_to_print)) + log.debug('\n' + '\n'.join(entities_to_print)) def find_candidate_entities(self, entity: str) -> List[str]: candidate_entities = list(self.name_to_q.get(entity, [])) @@ -226,7 +226,8 @@ def filter_triplets_rus(entity_triplets: List[List[List[str]]], question_tokens: if triplet[0] == property_is_instance_of and triplet[1] == id_for_entity_asteroid: entity_is_asteroid = True break - if found_what_template and (entity_is_human or entity_is_named or entity_is_asteroid or wiki_entity[2]<90): + if found_what_template and ( + entity_is_human or entity_is_named or entity_is_asteroid or wiki_entity[2] < 90): continue filtered_entity_triplets.append(triplets_for_entity) filtered_entities.append(wiki_entity) @@ -270,8 +271,8 @@ def candidate_entities_names(self, candidate_entities: List[Tuple[str]]) -> List return candidate_names def sort_found_entities(self, candidate_entities: List[Tuple[str]], - candidate_names: List[List[str]], - entity: str) -> Tuple[List[str], List[str], List[Tuple[str]]]: + candidate_names: List[List[str]], + entity: str) -> Tuple[List[str], List[str], List[Tuple[str]]]: entities_ratios = [] for candidate, entity_names in zip(candidate_entities, candidate_names): entity_id = candidate[0] @@ -286,6 +287,6 @@ def sort_found_entities(self, candidate_entities: List[Tuple[str]], srtd_with_ratios = sorted(entities_ratios, key=lambda x: (x[2], x[3]), reverse=True) wiki_entities = [ent[1] for ent in srtd_with_ratios if ent[2] > 84] - confidences = [float(ent[2])*0.01 for ent in srtd_with_ratios if ent[2] > 84] - + confidences = [float(ent[2]) * 0.01 for ent in srtd_with_ratios if ent[2] > 84] + return wiki_entities, confidences, srtd_with_ratios diff --git a/deeppavlov/models/kbqa/kb_answer_parser_wikidata.py b/deeppavlov/models/kbqa/kb_answer_parser_wikidata.py index 9e422d304c..e866b8b501 100644 --- a/deeppavlov/models/kbqa/kb_answer_parser_wikidata.py +++ b/deeppavlov/models/kbqa/kb_answer_parser_wikidata.py @@ -13,16 +13,15 @@ # limitations under the License. import pickle -from pathlib import Path -from string import punctuation from logging import getLogger +from string import punctuation from typing import List, Tuple, Optional, Dict import numpy as np -from deeppavlov.core.models.serializable import Serializable from deeppavlov.core.common.registry import register from deeppavlov.core.models.component import Component +from deeppavlov.core.models.serializable import Serializable from deeppavlov.models.kbqa.entity_linking import EntityLinker log = getLogger(__name__) @@ -115,7 +114,7 @@ def __call__(self, tokens_batch: List[List[str]], top_k_relations, top_k_probs = self._parse_relations_probs(relations_probs) top_k_relation_names = [self._relations_mapping[rel] for rel in top_k_relations] if self._debug: - log.debug("top k relations {}" .format(str(top_k_relation_names))) + log.debug("top k relations {}".format(str(top_k_relation_names))) obj, confidence = self._match_triplet(entity_triplets, entity_linking_confidences, top_k_relations, @@ -194,7 +193,7 @@ def entities_and_rels_from_templates(self, tokens: List[List[str]]) -> Tuple[str if template_start in s_sanitized and template_end in s_sanitized: template_start_pos = s_sanitized.find(template_start) template_end_pos = s_sanitized.find(template_end) - ent_cand = s_sanitized[template_start_pos+len(template_start): template_end_pos or len(s_sanitized)] + ent_cand = s_sanitized[template_start_pos + len(template_start): template_end_pos or len(s_sanitized)] if len(ent_cand) < len(ent) or len(ent) == 0: ent = ent_cand relation = self.templates[template] diff --git a/deeppavlov/models/morpho_tagger/cells.py b/deeppavlov/models/morpho_tagger/cells.py index b1fc40e72c..1736859c75 100644 --- a/deeppavlov/models/morpho_tagger/cells.py +++ b/deeppavlov/models/morpho_tagger/cells.py @@ -68,7 +68,6 @@ def weighted_sum(first, second, sigma, first_threshold=-np.inf, second_threshold class WeightedCombinationLayer(kl.Layer): - """ A class for weighted combination of probability distributions """ @@ -132,7 +131,7 @@ def call(self, inputs, **kwargs): embedded_features = kb.bias_add( embedded_features, self.features_bias, data_format="channels_last") if self.use_dimension_bias: - tiling_shape = [1] * (kb.ndim(first)-1) + [kb.shape(first)[-1]] + tiling_shape = [1] * (kb.ndim(first) - 1) + [kb.shape(first)[-1]] embedded_features = kb.tile(embedded_features, tiling_shape) embedded_features = kb.bias_add( embedded_features, self.dimensions_bias, data_format="channels_last") @@ -177,4 +176,4 @@ def positions_func(inputs, pad=0): """ position_inputs = kb.cumsum(kb.ones_like(inputs, dtype="float32"), axis=1) position_inputs *= kb.cast(kb.not_equal(inputs, pad), "float32") - return kb.log(1.0 + position_inputs) \ No newline at end of file + return kb.log(1.0 + position_inputs) diff --git a/deeppavlov/models/morpho_tagger/common.py b/deeppavlov/models/morpho_tagger/common.py index 9abf0bc06e..4f6371536a 100644 --- a/deeppavlov/models/morpho_tagger/common.py +++ b/deeppavlov/models/morpho_tagger/common.py @@ -109,7 +109,7 @@ def set_format_mode(self, format_mode: str = "basic") -> None: def _make_format_string(self) -> None: if self.format_mode == "basic": - self.format_string = "{}\t{}\t{}\t{}" + self.format_string = "{}\t{}\t{}\t{}" elif self.format_mode.lower() in ["conllu", "ud"]: self.format_string = "{}\t{}\t_\t{}\t_\t{}\t_\t_\t_\t_" else: diff --git a/deeppavlov/models/morpho_tagger/common_tagger.py b/deeppavlov/models/morpho_tagger/common_tagger.py index 6b7b905b39..86959d9919 100644 --- a/deeppavlov/models/morpho_tagger/common_tagger.py +++ b/deeppavlov/models/morpho_tagger/common_tagger.py @@ -21,6 +21,7 @@ EPS = 1e-15 + # AUXILIARY = ['PAD', 'BEGIN', 'END', 'UNKNOWN'] # AUXILIARY_CODES = PAD, BEGIN, END, UNKNOWN = 0, 1, 2, 3 diff --git a/deeppavlov/models/morpho_tagger/lemmatizer.py b/deeppavlov/models/morpho_tagger/lemmatizer.py index 2f0450ab51..7cb12dfb59 100644 --- a/deeppavlov/models/morpho_tagger/lemmatizer.py +++ b/deeppavlov/models/morpho_tagger/lemmatizer.py @@ -19,8 +19,8 @@ from pymorphy2 import MorphAnalyzer from russian_tagsets import converters -from deeppavlov.core.models.serializable import Serializable from deeppavlov.core.common.registry import register +from deeppavlov.core.models.serializable import Serializable from deeppavlov.models.morpho_tagger.common_tagger import get_tag_distance @@ -77,6 +77,7 @@ class UDPymorphyLemmatizer(BasicLemmatizer): Lemma is selected from one of PyMorphy parses, the parse whose tag resembles the most a known UD tag is chosen. """ + def __init__(self, save_path: Optional[str] = None, load_path: Optional[str] = None, transform_lemmas=False, **kwargs) -> None: self.transform_lemmas = transform_lemmas diff --git a/deeppavlov/models/morpho_tagger/morpho_tagger.py b/deeppavlov/models/morpho_tagger/morpho_tagger.py index b4a1dd0f7a..ad112cb206 100644 --- a/deeppavlov/models/morpho_tagger/morpho_tagger.py +++ b/deeppavlov/models/morpho_tagger/morpho_tagger.py @@ -16,11 +16,11 @@ from pathlib import Path from typing import List, Optional, Union, Tuple -import numpy as np +import keras.backend as kb import keras.layers as kl import keras.optimizers as ko import keras.regularizers as kreg -import keras.backend as kb +import numpy as np from keras import Model from deeppavlov.core.common.registry import register @@ -75,6 +75,7 @@ class MorphoTagger(KerasModel): A subclass of :class:`~deeppavlov.core.models.keras_model.KerasModel` """ + def __init__(self, symbols: SimpleVocabulary, tags: SimpleVocabulary, @@ -166,7 +167,7 @@ def _initialize(self): def build(self): """Builds the network using Keras. """ - word_inputs = kl.Input(shape=(None, MAX_WORD_LENGTH+2), dtype="int32") + word_inputs = kl.Input(shape=(None, MAX_WORD_LENGTH + 2), dtype="int32") inputs = [word_inputs] word_outputs = self._build_word_cnn(word_inputs) if len(self.word_vectorizers) > 0: @@ -231,17 +232,17 @@ def _build_basic_network(self, word_outputs): lstm_outputs = kl.Dropout(self.word_dropout)(word_outputs) else: lstm_outputs = word_outputs - for j in range(self.word_lstm_layers-1): + for j in range(self.word_lstm_layers - 1): lstm_outputs = kl.Bidirectional( kl.LSTM(self.word_lstm_units[j], return_sequences=True, dropout=self.lstm_dropout))(lstm_outputs) lstm_outputs = kl.Bidirectional( - kl.LSTM(self.word_lstm_units[-1], return_sequences=True, - dropout=self.lstm_dropout))(lstm_outputs) + kl.LSTM(self.word_lstm_units[-1], return_sequences=True, + dropout=self.lstm_dropout))(lstm_outputs) pre_outputs = kl.TimeDistributed( - kl.Dense(len(self.tags), activation="softmax", - activity_regularizer=self.regularizer), - name="p")(lstm_outputs) + kl.Dense(len(self.tags), activation="softmax", + activity_regularizer=self.regularizer), + name="p")(lstm_outputs) return pre_outputs, lstm_outputs def _transform_batch(self, data, labels=None, transform_to_one_hot=True): @@ -309,7 +310,7 @@ def __call__(self, *x_batch, **kwargs) -> Union[List, np.ndarray]: """ return self.predict_on_batch(x_batch, **kwargs) - def _make_sent_vector(self, sent: List, bucket_length: int =None) -> np.ndarray: + def _make_sent_vector(self, sent: List, bucket_length: int = None) -> np.ndarray: """Transforms a sentence to Numpy array, which will be the network input. Args: @@ -321,14 +322,14 @@ def _make_sent_vector(self, sent: List, bucket_length: int =None) -> np.ndarray: in j-th word of i-th input sentence. """ bucket_length = bucket_length or len(sent) - answer = np.zeros(shape=(bucket_length, MAX_WORD_LENGTH+2), dtype=np.int32) + answer = np.zeros(shape=(bucket_length, MAX_WORD_LENGTH + 2), dtype=np.int32) for i, word in enumerate(sent): answer[i, 0] = self.tags["BEGIN"] m = min(len(word), MAX_WORD_LENGTH) for j, x in enumerate(word[-m:]): - answer[i, j+1] = self.symbols[x] - answer[i, m+1] = self.tags["END"] - answer[i, m+2:] = self.tags["PAD"] + answer[i, j + 1] = self.symbols[x] + answer[i, m + 1] = self.tags["END"] + answer[i, m + 2:] = self.tags["PAD"] return answer def _make_tags_vector(self, tags, bucket_length=None) -> np.ndarray: diff --git a/deeppavlov/models/ner/bio.py b/deeppavlov/models/ner/bio.py index 19eca12da1..7eb75015ed 100644 --- a/deeppavlov/models/ner/bio.py +++ b/deeppavlov/models/ner/bio.py @@ -24,6 +24,7 @@ @register('ner_bio_converter') class BIOMarkupRestorer(Component): """Restores BIO markup for tags batch""" + def __init__(self, *args, **kwargs) -> None: pass diff --git a/deeppavlov/models/ner/svm.py b/deeppavlov/models/ner/svm.py index 04b656e4b3..d8eda1538b 100644 --- a/deeppavlov/models/ner/svm.py +++ b/deeppavlov/models/ner/svm.py @@ -36,6 +36,7 @@ class SVMTagger(Estimator): kernel: kernel of SVM (RBF works well in the most of the cases) seed: seed for SVM initialization """ + def __init__(self, return_probabilities: bool = False, kernel: str = 'rbf', seed=42, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.classifier = None diff --git a/deeppavlov/models/preprocessors/bert_preprocessor.py b/deeppavlov/models/preprocessors/bert_preprocessor.py index 86dfdd6df3..5275800e5c 100644 --- a/deeppavlov/models/preprocessors/bert_preprocessor.py +++ b/deeppavlov/models/preprocessors/bert_preprocessor.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import re from logging import getLogger from typing import Tuple, List, Optional, Union diff --git a/deeppavlov/models/preprocessors/capitalization.py b/deeppavlov/models/preprocessors/capitalization.py index 59fe43b74c..3760979471 100644 --- a/deeppavlov/models/preprocessors/capitalization.py +++ b/deeppavlov/models/preprocessors/capitalization.py @@ -38,6 +38,7 @@ class CapitalizationPreprocessor(Component): Attributes: dim: dimensionality of the feature vectors, produced by the featurizer """ + def __init__(self, pad_zeros: bool = True, *args, **kwargs) -> None: self.pad_zeros = pad_zeros self._num_of_features = 4 diff --git a/deeppavlov/models/preprocessors/char_splitter.py b/deeppavlov/models/preprocessors/char_splitter.py index 765a79867a..c242d3612e 100644 --- a/deeppavlov/models/preprocessors/char_splitter.py +++ b/deeppavlov/models/preprocessors/char_splitter.py @@ -25,6 +25,7 @@ @register('char_splitter') class CharSplitter(Component): """This component transforms batch of sequences of tokens into batch of sequences of character sequences.""" + def __init__(self, **kwargs): pass diff --git a/deeppavlov/models/preprocessors/dirty_comments_preprocessor.py b/deeppavlov/models/preprocessors/dirty_comments_preprocessor.py index 90ce6c23fa..656e819478 100644 --- a/deeppavlov/models/preprocessors/dirty_comments_preprocessor.py +++ b/deeppavlov/models/preprocessors/dirty_comments_preprocessor.py @@ -25,6 +25,7 @@ class DirtyCommentsPreprocessor(Component): """ Class implements preprocessing of english texts with low level of literacy such as comments """ + def __init__(self, remove_punctuation: bool = True, *args, **kwargs): self.remove_punctuation = remove_punctuation diff --git a/deeppavlov/models/preprocessors/mask.py b/deeppavlov/models/preprocessors/mask.py index 48da525b01..ca4e3149a2 100644 --- a/deeppavlov/models/preprocessors/mask.py +++ b/deeppavlov/models/preprocessors/mask.py @@ -21,6 +21,7 @@ @register('mask') class Mask(Component): """Takes batch of tokens and returns the masks of corresponding length""" + def __init__(self, *args, **kwargs): pass diff --git a/deeppavlov/models/preprocessors/one_hotter.py b/deeppavlov/models/preprocessors/one_hotter.py index 3accf6451d..7dca070f92 100644 --- a/deeppavlov/models/preprocessors/one_hotter.py +++ b/deeppavlov/models/preprocessors/one_hotter.py @@ -33,6 +33,7 @@ class OneHotter(Component): pad_zeros: whether to pad elements of batch with zeros single_vector: whether to return one vector for the sample (sum of each one-hotted vectors) """ + def __init__(self, depth: int, pad_zeros: bool = False, single_vector=False, *args, **kwargs): self._depth = depth diff --git a/deeppavlov/models/preprocessors/random_embeddings_matrix.py b/deeppavlov/models/preprocessors/random_embeddings_matrix.py index 1fae19b372..b72f75a0fa 100644 --- a/deeppavlov/models/preprocessors/random_embeddings_matrix.py +++ b/deeppavlov/models/preprocessors/random_embeddings_matrix.py @@ -28,6 +28,7 @@ class RandomEmbeddingsMatrix: Attributes: dim: dimensionality of the embeddings """ + def __init__(self, vocab_len: int, emb_dim: int, *args, **kwargs) -> None: self.emb_mat = np.random.randn(vocab_len, emb_dim).astype(np.float32) / np.sqrt(emb_dim) diff --git a/deeppavlov/models/preprocessors/russian_lemmatizer.py b/deeppavlov/models/preprocessors/russian_lemmatizer.py index d05aa1139f..ae68f4fc97 100644 --- a/deeppavlov/models/preprocessors/russian_lemmatizer.py +++ b/deeppavlov/models/preprocessors/russian_lemmatizer.py @@ -21,6 +21,7 @@ @register('pymorphy_russian_lemmatizer') class PymorphyRussianLemmatizer(Component): """Class for lemmatization using PyMorphy.""" + def __init__(self, *args, **kwargs): self.lemmatizer = pymorphy2.MorphAnalyzer() diff --git a/deeppavlov/models/preprocessors/sanitizer.py b/deeppavlov/models/preprocessors/sanitizer.py index 314c0709c6..7d60cd29a9 100644 --- a/deeppavlov/models/preprocessors/sanitizer.py +++ b/deeppavlov/models/preprocessors/sanitizer.py @@ -29,6 +29,7 @@ class Sanitizer(Component): diacritical signs are something like hats and stress marks nums: whether to replace all digits with 1 or not """ + def __init__(self, diacritical: bool = True, nums: bool = False, diff --git a/deeppavlov/models/preprocessors/siamese_preprocessor.py b/deeppavlov/models/preprocessors/siamese_preprocessor.py index c95f01a122..9a7a92332e 100644 --- a/deeppavlov/models/preprocessors/siamese_preprocessor.py +++ b/deeppavlov/models/preprocessors/siamese_preprocessor.py @@ -111,7 +111,7 @@ def __call__(self, x: Union[List[List[str]], List[str]]) -> Iterable[List[List[n else: x_preproc = [[el] for el in x] else: - x_preproc = [el[:self.num_context_turns+self.num_ranking_samples] for el in x] + x_preproc = [el[:self.num_context_turns + self.num_ranking_samples] for el in x] for el in x_preproc: x_tok = self.tokenizer(el) x_ctok = [y if len(y) != 0 else [''] for y in x_tok] @@ -126,7 +126,7 @@ def __call__(self, x: Union[List[List[str]], List[str]]) -> Iterable[List[List[n x_proc = zero_pad_truncate(x_proc, msl, pad=self.padding, trunc=self.truncating) x_proc = list(x_proc) if self.add_raw_text: - x_proc += el # add (self.num_context_turns+self.num_ranking_samples) raw sentences + x_proc += el # add (self.num_context_turns+self.num_ranking_samples) raw sentences yield x_proc def load(self) -> None: diff --git a/deeppavlov/models/preprocessors/squad_preprocessor.py b/deeppavlov/models/preprocessors/squad_preprocessor.py index 62a09cbf6d..f589faf1ce 100644 --- a/deeppavlov/models/preprocessors/squad_preprocessor.py +++ b/deeppavlov/models/preprocessors/squad_preprocessor.py @@ -13,13 +13,13 @@ # limitations under the License. +import bisect import pickle import unicodedata from collections import Counter from logging import getLogger from pathlib import Path from typing import Tuple, List, Union -import bisect import numpy as np from nltk import word_tokenize @@ -53,11 +53,11 @@ def __init__(self, context_limit: int = 450, question_limit: int = 150, char_lim def __call__(self, contexts_raw: Tuple[str, ...], questions_raw: Tuple[str, ...], **kwargs) -> Tuple[ - List[str], List[List[str]], List[List[List[str]]], - List[List[int]], List[List[int]], - List[str], List[List[str]], List[List[List[str]]], - List[List[Tuple[int, int]]] - ]: + List[str], List[List[str]], List[List[List[str]]], + List[List[int]], List[List[int]], + List[str], List[List[str]], List[List[List[str]]], + List[List[Tuple[int, int]]] + ]: """ Performs preprocessing of context and question Args: contexts_raw: batch of contexts to preprocess @@ -100,7 +100,7 @@ def __call__(self, contexts_raw: Tuple[str, ...], questions_raw: Tuple[str, ...] questions_chars.append(q_chars) spans.append(SquadPreprocessor.convert_idx(c, c_tokens)) return contexts, contexts_tokens, contexts_chars, contexts_r2p, contexts_p2r, \ - questions, questions_tokens, questions_chars, spans + questions, questions_tokens, questions_chars, spans @staticmethod def preprocess_str(line: str, return_mapping: bool = False) -> Union[Tuple[str, List[int], List[int]], str]: @@ -480,7 +480,7 @@ def __call__(self, answers_start, answers_end, contexts, bert_features, subtok2c end = self.get_char_position(sub2c, answer_end) subtok = features.tokens[answer_end] subtok = subtok[2:] if subtok.startswith('##') else subtok - answer = context[st:end+len(subtok)] + answer = context[st:end + len(subtok)] answers += [answer] starts += [st] ends += [ends] diff --git a/deeppavlov/models/preprocessors/str_token_reverser.py b/deeppavlov/models/preprocessors/str_token_reverser.py index 7dffe0af14..3c25ff8d68 100644 --- a/deeppavlov/models/preprocessors/str_token_reverser.py +++ b/deeppavlov/models/preprocessors/str_token_reverser.py @@ -1,4 +1,3 @@ - # Copyright 2017 Neural Networks and Deep Learning lab, MIPT # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,6 +27,7 @@ class StrTokenReverser(Component): Args: tokenized: The parameter is only needed to reverse tokenized strings. """ + def __init__(self, tokenized: bool = False, *args, **kwargs) -> None: self.tokenized = tokenized @@ -54,7 +54,7 @@ def __call__(self, batch: Union[str, list, tuple]) -> StrTokenReverserInfo: """ if isinstance(batch, (list, tuple)): batch = batch.copy() - + if self.tokenized: if isinstance(batch, (list, tuple)): if isinstance(batch[-1], str): @@ -69,4 +69,3 @@ def __call__(self, batch: Union[str, list, tuple]) -> StrTokenReverserInfo: return [self(line) for line in batch] else: return self._reverse_str(batch) - diff --git a/deeppavlov/models/preprocessors/str_utf8_encoder.py b/deeppavlov/models/preprocessors/str_utf8_encoder.py index 6f8437f1e4..2aaa0703cb 100644 --- a/deeppavlov/models/preprocessors/str_utf8_encoder.py +++ b/deeppavlov/models/preprocessors/str_utf8_encoder.py @@ -44,7 +44,8 @@ class StrUTF8Encoder(Estimator): bos: Name of a special token of the begin of a sentence. eos: Name of a special token of the end of a sentence. """ - def __init__(self, + + def __init__(self, max_word_length: int = 50, pad_special_char_use: bool = False, word_boundary_special_char_use: bool = False, @@ -56,9 +57,9 @@ def __init__(self, super().__init__(**kwargs) if word_boundary_special_char_use and max_word_length < 3: - raise ConfigError(f"`max_word_length` should be more than 3!") + raise ConfigError(f"`max_word_length` should be more than 3!") if max_word_length < 1: - raise ConfigError(f"`max_word_length` should be more than 1!") + raise ConfigError(f"`max_word_length` should be more than 1!") self._max_word_length = max_word_length self._reverse = reversed_sentense_tokens @@ -85,12 +86,12 @@ def _make_bos_eos(indx): else: code = indx if self._pad_special_char_use: - code = np.pad(code, (0, self._max_word_length - code.shape[0]), 'constant', + code = np.pad(code, (0, self._max_word_length - code.shape[0]), 'constant', constant_values=(self.pad_char)) else: pass return code - + self.bos_chars = _make_bos_eos(self.bos_char) self.eos_chars = _make_bos_eos(self.eos_char) @@ -149,14 +150,14 @@ def save(self) -> None: with self.save_path.open('wt', encoding='utf8') as f: for token in self._word_char_ids.keys(): f.write('{}\n'.format(token)) - + @overrides def fit(self, *args) -> None: words = chain(*args) # filter(None, <>) -- to filter empty words freqs = Counter(filter(None, chain(*words))) for token, _ in freqs.most_common(): - if not(token in self._word_char_ids): + if not (token in self._word_char_ids): self._word_char_ids[token] = self._convert_word_to_char_ids(token) def _convert_word_to_char_ids(self, word): diff --git a/deeppavlov/models/ranking/bilstm_gru_siamese_network.py b/deeppavlov/models/ranking/bilstm_gru_siamese_network.py index e56c2a1187..281633db0a 100644 --- a/deeppavlov/models/ranking/bilstm_gru_siamese_network.py +++ b/deeppavlov/models/ranking/bilstm_gru_siamese_network.py @@ -23,9 +23,9 @@ log = getLogger(__name__) + @register('bilstm_gru_nn') class BiLSTMGRUSiameseNetwork(BiLSTMSiameseNetwork): - """The class implementing a siamese neural network with BiLSTM, GRU and max pooling. GRU is used to take into account multi-turn dialogue ``context``. @@ -48,6 +48,7 @@ class BiLSTMGRUSiameseNetwork(BiLSTMSiameseNetwork): If set to ``False`` random sampling will be used. Only required if ``triplet_loss`` is set to ``True``. """ + def create_model(self) -> Model: input = [] if self.use_matrix: @@ -100,11 +101,10 @@ def create_score_model(self) -> Model: def create_context_model(self) -> Model: m = Model(self.model.inputs[:-1], - self.model.get_layer("gru").output) + self.model.get_layer("gru").output) return m def create_response_model(self) -> Model: m = Model(self.model.inputs[-1], - self.model.get_layer("pooling").get_output_at(-1)) + self.model.get_layer("pooling").get_output_at(-1)) return m - diff --git a/deeppavlov/models/ranking/bilstm_siamese_network.py b/deeppavlov/models/ranking/bilstm_siamese_network.py index 3788b5974d..d192b9e9b1 100644 --- a/deeppavlov/models/ranking/bilstm_siamese_network.py +++ b/deeppavlov/models/ranking/bilstm_siamese_network.py @@ -34,7 +34,6 @@ @register('bilstm_nn') class BiLSTMSiameseNetwork(KerasSiameseModel): - """The class implementing a siamese neural network with BiLSTM and max pooling. There is a possibility to use a binary cross-entropy loss as well as @@ -120,10 +119,10 @@ def lstm_layer(self) -> Layer: rec_in = Orthogonal(seed=self.seed) if self.recurrent == "bilstm" or self.recurrent is None: out = Bidirectional(LSTM(self.hidden_dim, - input_shape=(self.max_sequence_length, self.embedding_dim,), - kernel_initializer=ker_in, - recurrent_initializer=rec_in, - return_sequences=ret_seq), merge_mode='concat') + input_shape=(self.max_sequence_length, self.embedding_dim,), + kernel_initializer=ker_in, + recurrent_initializer=rec_in, + return_sequences=ret_seq), merge_mode='concat') elif self.recurrent == "lstm": out = LSTM(self.hidden_dim, input_shape=(self.max_sequence_length, self.embedding_dim,), @@ -187,7 +186,7 @@ def create_score_model(self) -> Model: def _diff_mult_dist(self, inputs: List[Tensor]) -> Tensor: input1, input2 = inputs - a = K.abs(input1-input2) + a = K.abs(input1 - input2) b = Multiply()(inputs) return K.concatenate([input1, input2, a, b]) @@ -216,7 +215,7 @@ def _pairwise_distances(self, inputs: List[Tensor]) -> Tensor: distances = distances * (1.0 - mask) return distances - def _triplet_loss(self, labels: Tensor, pairwise_dist: Tensor) -> Tensor : + def _triplet_loss(self, labels: Tensor, pairwise_dist: Tensor) -> Tensor: y_true = K.squeeze(labels, axis=1) """Triplet loss function""" if self.hard_triplets: @@ -244,8 +243,8 @@ def _batch_hard_triplet_loss(self, y_true: Tensor, pairwise_dist: Tensor) -> Ten mask_anchor_negative = self._get_anchor_negative_triplet_mask(y_true, pairwise_dist) anchor_negative_dist = mask_anchor_negative * pairwise_dist mask_anchor_negative = self._get_semihard_anchor_negative_triplet_mask(anchor_negative_dist, - hardest_positive_dist, - mask_anchor_negative) + hardest_positive_dist, + mask_anchor_negative) max_anchor_negative_dist = K.max(pairwise_dist, axis=1, keepdims=True) anchor_negative_dist = pairwise_dist + max_anchor_negative_dist * (1.0 - mask_anchor_negative) hardest_negative_dist = K.min(anchor_negative_dist, axis=1, keepdims=True) @@ -290,5 +289,5 @@ def _get_semihard_anchor_negative_triplet_mask(self, negative_dist: Tensor, mask = mask_negative * (1 - mask_semihard) + mask * mask_semihard return mask - def _predict_on_batch(self, batch: List[np.ndarray]) -> np.ndarray: + def _predict_on_batch(self, batch: List[np.ndarray]) -> np.ndarray: return self.score_model.predict_on_batch(x=batch) diff --git a/deeppavlov/models/ranking/deep_attention_matching_network.py b/deeppavlov/models/ranking/deep_attention_matching_network.py index 41b261018b..71b74a64a6 100644 --- a/deeppavlov/models/ranking/deep_attention_matching_network.py +++ b/deeppavlov/models/ranking/deep_attention_matching_network.py @@ -19,9 +19,9 @@ import tensorflow as tf from deeppavlov.core.common.registry import register -from deeppavlov.models.ranking.tf_base_matching_model import TensorflowBaseMatchingModel from deeppavlov.models.ranking.matching_models.dam_utils import layers from deeppavlov.models.ranking.matching_models.dam_utils import operations as op +from deeppavlov.models.ranking.tf_base_matching_model import TensorflowBaseMatchingModel log = getLogger(__name__) diff --git a/deeppavlov/models/ranking/deep_attention_matching_network_use_transformer.py b/deeppavlov/models/ranking/deep_attention_matching_network_use_transformer.py index cd322dd24a..a9dc45ccd0 100644 --- a/deeppavlov/models/ranking/deep_attention_matching_network_use_transformer.py +++ b/deeppavlov/models/ranking/deep_attention_matching_network_use_transformer.py @@ -20,9 +20,9 @@ import tensorflow_hub as hub from deeppavlov.core.common.registry import register -from deeppavlov.models.ranking.tf_base_matching_model import TensorflowBaseMatchingModel from deeppavlov.models.ranking.matching_models.dam_utils import layers from deeppavlov.models.ranking.matching_models.dam_utils import operations as op +from deeppavlov.models.ranking.tf_base_matching_model import TensorflowBaseMatchingModel log = getLogger(__name__) @@ -145,7 +145,6 @@ def _init_sentence_encoder(self): # for resp sentences: shape=(None, 1, 512) self.sent_embedder_response = tf.expand_dims(embed_response, axis=1) - def _init_graph(self): self._init_placeholders() self._init_sentence_encoder() @@ -189,7 +188,7 @@ def _init_graph(self): # context part # a list of length max_turn_num, every element is a tensor with shape [batch, max_turn_len] - list_turn_t = tf.unstack(self.utterance_ph, axis=1) + list_turn_t = tf.unstack(self.utterance_ph, axis=1) list_turn_length = tf.unstack(self.all_utterance_len_ph, axis=1) list_turn_t_sent = tf.unstack(sent_embedder_context, axis=1) @@ -264,7 +263,7 @@ def _init_graph(self): sim = tf.stack(sim_turns, axis=1) log.info('sim shape: %s' % sim.shape) with tf.variable_scope('cnn_aggregation'): - final_info = layers.CNN_3d(sim, 32, 32) # We can improve performance if use 32 filters for each layer + final_info = layers.CNN_3d(sim, 32, 32) # We can improve performance if use 32 filters for each layer # for douban # final_info = layers.CNN_3d(sim, 16, 16) @@ -312,12 +311,12 @@ def _append_sample_to_batch_buffer(self, sample: List[np.ndarray], buf: List[Tup """ sample_len = len(sample) - batch_buffer_context = [] # [batch_size, 10, 50] - batch_buffer_context_len = [] # [batch_size, 10] - batch_buffer_response = [] # [batch_size, 50] + batch_buffer_context = [] # [batch_size, 10, 50] + batch_buffer_context_len = [] # [batch_size, 10] + batch_buffer_response = [] # [batch_size, 50] batch_buffer_response_len = [] # [batch_size] - raw_batch_buffer_context = [] # [batch_size, 10] + raw_batch_buffer_context = [] # [batch_size, 10] raw_batch_buffer_response = [] # [batch_size] context_sentences = sample[:self.num_context_turns] @@ -330,12 +329,12 @@ def _append_sample_to_batch_buffer(self, sample: List[np.ndarray], buf: List[Tup # 4 model inputs # 1. Token indices for context - batch_buffer_context += [context_sentences for sent in response_sentences] # replicate context N times + batch_buffer_context += [context_sentences for sent in response_sentences] # replicate context N times # 2. Token indices for response batch_buffer_response += [response_sentence for response_sentence in response_sentences] # 3. Lengths of all context sentences lens = [] - for context in [context_sentences for sent in response_sentences]: # replicate context N times + for context in [context_sentences for sent in response_sentences]: # replicate context N times context_sentences_lens = [] for sent in context: sent_len = len(sent[sent != 0]) @@ -391,7 +390,7 @@ def _make_batch(self, batch: List[Tuple[np.ndarray]]) -> Dict: input_context_len.append(sample[1]) input_response.append(sample[2]) input_response_len.append(sample[3]) - input_raw_context.append(sample[4]) # raw context is the 4th element of each Tuple in the batch + input_raw_context.append(sample[4]) # raw context is the 4th element of each Tuple in the batch input_raw_response.append(sample[5]) # raw response is the 5th element of each Tuple in the batch return { diff --git a/deeppavlov/models/ranking/keras_siamese_model.py b/deeppavlov/models/ranking/keras_siamese_model.py index 7826d6e671..545f3469ab 100644 --- a/deeppavlov/models/ranking/keras_siamese_model.py +++ b/deeppavlov/models/ranking/keras_siamese_model.py @@ -68,7 +68,7 @@ def __init__(self, self.model = self.create_model() self.compile() if self.load_path.exists(): - self.load() + self.load() else: self.load_initial_emb_matrix() @@ -100,12 +100,12 @@ def create_model(self) -> Model: def create_context_model(self) -> Model: m = Model(self.model.inputs[:-1], - self.model.get_layer("sentence_embedding").get_output_at(0)) + self.model.get_layer("sentence_embedding").get_output_at(0)) return m def create_response_model(self) -> Model: m = Model(self.model.inputs[-1], - self.model.get_layer("sentence_embedding").get_output_at(1)) + self.model.get_layer("sentence_embedding").get_output_at(1)) return m def _train_on_batch(self, batch: List[np.ndarray], y: List[int]) -> float: @@ -121,4 +121,3 @@ def _predict_context_on_batch(self, batch: List[np.ndarray]) -> np.ndarray: def _predict_response_on_batch(self, batch: List[np.ndarray]) -> np.ndarray: return self.response_model.predict_on_batch(batch) - diff --git a/deeppavlov/models/ranking/matching_models/dam_utils/layers.py b/deeppavlov/models/ranking/matching_models/dam_utils/layers.py index b4be0e6db9..037453d77e 100644 --- a/deeppavlov/models/ranking/matching_models/dam_utils/layers.py +++ b/deeppavlov/models/ranking/matching_models/dam_utils/layers.py @@ -90,15 +90,16 @@ def dynamic_L(x): shape=[x.shape[-1]], dtype=tf.float32, initializer=tf.random_uniform_initializer( - -tf.sqrt(6./tf.cast(x.shape[-1], tf.float32)), - tf.sqrt(6./tf.cast(x.shape[-1], tf.float32)))) + -tf.sqrt(6. / tf.cast(x.shape[-1], tf.float32)), + tf.sqrt(6. / tf.cast(x.shape[-1], tf.float32)))) - key = op.dense(x, add_bias=False) #[batch, time, dimension] - weight = tf.reduce_sum(tf.multiply(key, key_0), axis=-1) #[batch, time] - weight = tf.expand_dims(tf.nn.softmax(weight), -1) #[batch, time, 1] + key = op.dense(x, add_bias=False) # [batch, time, dimension] + weight = tf.reduce_sum(tf.multiply(key, key_0), axis=-1) # [batch, time] + weight = tf.expand_dims(tf.nn.softmax(weight), -1) # [batch, time, 1] + + L = tf.reduce_sum(tf.multiply(x, weight), axis=1) # [batch, dimension] + return L - L = tf.reduce_sum(tf.multiply(x, weight), axis=1) #[batch, dimension] - return L def loss(x, y, num_classes=2, is_clip=True, clip_value=10): '''From info x calculate logits as return loss. @@ -152,12 +153,13 @@ def loss(x, y, num_classes=2, is_clip=True, clip_value=10): return loss, logits + def attention( - Q, K, V, - Q_lengths, K_lengths, - attention_type='dot', - is_mask=True, mask_value=-2**32+1, - drop_prob=None): + Q, K, V, + Q_lengths, K_lengths, + attention_type='dot', + is_mask=True, mask_value=-2 ** 32 + 1, + drop_prob=None): '''Add attention layer. Args: Q: a tensor with shape [batch, Q_time, Q_dimension] @@ -182,16 +184,16 @@ def attention( K_time = K.shape[1] if attention_type == 'dot': - logits = op.dot_sim(Q, K) #[batch, Q_time, time] + logits = op.dot_sim(Q, K) # [batch, Q_time, time] if attention_type == 'bilinear': logits = op.bilinear_sim(Q, K) if is_mask: - mask = op.mask(Q_lengths, K_lengths, Q_time, K_time) #[batch, Q_time, K_time] + mask = op.mask(Q_lengths, K_lengths, Q_time, K_time) # [batch, Q_time, K_time] # mask = tf.Print(mask, [logits[0], mask[0]], tf.get_variable_scope().name + " logits, mask: ", summarize=10) logits = mask * logits + (1 - mask) * mask_value # logits = tf.Print(logits, [logits[0]], tf.get_variable_scope().name + " masked logits: ", summarize=10) - + attention = tf.nn.softmax(logits) if drop_prob is not None: @@ -200,6 +202,7 @@ def attention( return op.weighted_sum(attention, V) + def FFN(x, out_dimension_0=None, out_dimension_1=None): '''Add two dense connected layer, max(0, x*W0+b0)*W1+b1. @@ -217,16 +220,17 @@ def FFN(x, out_dimension_0=None, out_dimension_1=None): y = tf.nn.relu(y) with tf.variable_scope('FFN_2'): # z = op.dense(y, out_dimension_1, initializer=tf.keras.initializers.glorot_uniform(seed=42)) # TODO: check - z = op.dense(y, out_dimension_1) #, add_bias=False) #!!!! + z = op.dense(y, out_dimension_1) # , add_bias=False) #!!!! return z + def block( - Q, K, V, - Q_lengths, K_lengths, - attention_type='dot', - is_layer_norm=True, - is_mask=True, mask_value=-2**32+1, - drop_prob=None): + Q, K, V, + Q_lengths, K_lengths, + attention_type='dot', + is_layer_norm=True, + is_mask=True, mask_value=-2 ** 32 + 1, + drop_prob=None): '''Add a block unit from https://arxiv.org/pdf/1706.03762.pdf. Args: Q: a tensor with shape [batch, Q_time, Q_dimension] @@ -241,8 +245,8 @@ def block( Raises: ''' - att = attention(Q, K, V, - Q_lengths, K_lengths, + att = attention(Q, K, V, + Q_lengths, K_lengths, attention_type=attention_type, is_mask=is_mask, mask_value=mask_value, drop_prob=drop_prob) @@ -260,6 +264,7 @@ def block( w = y + z return w + def CNN(x, out_channels, filter_size, pooling_size, add_relu=True): '''Add a convlution layer with relu and max pooling layer. @@ -274,9 +279,9 @@ def CNN(x, out_channels, filter_size, pooling_size, add_relu=True): Raises: ''' - #calculate the last dimension of return - num_features = ((tf.shape(x)[1]-filter_size+1)/pooling_size * - (tf.shape(x)[2]-filter_size+1)/pooling_size) * out_channels + # calculate the last dimension of return + num_features = ((tf.shape(x)[1] - filter_size + 1) / pooling_size * + (tf.shape(x)[2] - filter_size + 1) / pooling_size) * out_channels in_channels = x.shape[-1] weights = tf.get_variable( @@ -297,13 +302,14 @@ def CNN(x, out_channels, filter_size, pooling_size, add_relu=True): conv = tf.nn.relu(conv) pooling = tf.nn.max_pool( - conv, + conv, ksize=[1, pooling_size, pooling_size, 1], - strides=[1, pooling_size, pooling_size, 1], + strides=[1, pooling_size, pooling_size, 1], padding="VALID") return tf.contrib.layers.flatten(pooling) + def CNN_3d(x, out_channels_0, out_channels_1, add_relu=True): '''Add a 3d convlution layer with relu and max pooling layer. @@ -331,20 +337,20 @@ def CNN_3d(x, out_channels_0, out_channels_1, add_relu=True): initializer=tf.zeros_initializer()) conv_0 = tf.nn.conv3d(x, weights_0, strides=[1, 1, 1, 1, 1], padding="SAME") - log.info('conv_0 shape: %s' %conv_0.shape) + log.info('conv_0 shape: %s' % conv_0.shape) conv_0 = conv_0 + bias_0 if add_relu: conv_0 = tf.nn.elu(conv_0) pooling_0 = tf.nn.max_pool3d( - conv_0, + conv_0, ksize=[1, 3, 3, 3, 1], - strides=[1, 3, 3, 3, 1], + strides=[1, 3, 3, 3, 1], padding="SAME") - log.info('pooling_0 shape: %s' %pooling_0.shape) + log.info('pooling_0 shape: %s' % pooling_0.shape) - #layer_1 + # layer_1 weights_1 = tf.get_variable( name='filter_1', shape=[3, 3, 3, out_channels_0, out_channels_1], @@ -357,21 +363,22 @@ def CNN_3d(x, out_channels_0, out_channels_1, add_relu=True): initializer=tf.zeros_initializer()) conv_1 = tf.nn.conv3d(pooling_0, weights_1, strides=[1, 1, 1, 1, 1], padding="SAME") - log.info('conv_1 shape: %s' %conv_1.shape) + log.info('conv_1 shape: %s' % conv_1.shape) conv_1 = conv_1 + bias_1 if add_relu: conv_1 = tf.nn.elu(conv_1) pooling_1 = tf.nn.max_pool3d( - conv_1, + conv_1, ksize=[1, 3, 3, 3, 1], - strides=[1, 3, 3, 3, 1], + strides=[1, 3, 3, 3, 1], padding="SAME") - log.info('pooling_1 shape: %s' %pooling_1.shape) + log.info('pooling_1 shape: %s' % pooling_1.shape) return tf.contrib.layers.flatten(pooling_1) + def CNN_3d_2d(x, out_channels_0, out_channels_1, add_relu=True): '''Add a 3d convlution layer with relu and max pooling layer. @@ -399,20 +406,20 @@ def CNN_3d_2d(x, out_channels_0, out_channels_1, add_relu=True): initializer=tf.zeros_initializer()) conv_0 = tf.nn.conv3d(x, weights_0, strides=[1, 1, 1, 1, 1], padding="SAME") - log.info('conv_0 shape: %s' %conv_0.shape) + log.info('conv_0 shape: %s' % conv_0.shape) conv_0 = conv_0 + bias_0 if add_relu: conv_0 = tf.nn.elu(conv_0) pooling_0 = tf.nn.max_pool3d( - conv_0, + conv_0, ksize=[1, 1, 3, 3, 1], - strides=[1, 1, 3, 3, 1], + strides=[1, 1, 3, 3, 1], padding="SAME") - log.info('pooling_0 shape: %s' %pooling_0.shape) + log.info('pooling_0 shape: %s' % pooling_0.shape) - #layer_1 + # layer_1 weights_1 = tf.get_variable( name='filter_1', shape=[1, 3, 3, out_channels_0, out_channels_1], @@ -425,21 +432,22 @@ def CNN_3d_2d(x, out_channels_0, out_channels_1, add_relu=True): initializer=tf.zeros_initializer()) conv_1 = tf.nn.conv3d(pooling_0, weights_1, strides=[1, 1, 1, 1, 1], padding="SAME") - log.info('conv_1 shape: %s' %conv_1.shape) + log.info('conv_1 shape: %s' % conv_1.shape) conv_1 = conv_1 + bias_1 if add_relu: conv_1 = tf.nn.elu(conv_1) pooling_1 = tf.nn.max_pool3d( - conv_1, + conv_1, ksize=[1, 1, 3, 3, 1], - strides=[1, 1, 3, 3, 1], + strides=[1, 1, 3, 3, 1], padding="SAME") - log.info('pooling_1 shape: %s' %pooling_1.shape) + log.info('pooling_1 shape: %s' % pooling_1.shape) return tf.contrib.layers.flatten(pooling_1) + def CNN_3d_change(x, out_channels_0, out_channels_1, add_relu=True): '''Add a 3d convlution layer with relu and max pooling layer. @@ -459,22 +467,22 @@ def CNN_3d_change(x, out_channels_0, out_channels_1, add_relu=True): name='filter_0', shape=[3, 3, 3, in_channels, out_channels_0], dtype=tf.float32, - #initializer=tf.random_normal_initializer(0, 0.05)) + # initializer=tf.random_normal_initializer(0, 0.05)) initializer=tf.random_uniform_initializer(-0.01, 0.01)) bias_0 = tf.get_variable( name='bias_0', shape=[out_channels_0], dtype=tf.float32, initializer=tf.zeros_initializer()) - #Todo + # Todo g_0 = tf.get_variable(name='scale_0', - shape = [out_channels_0], - dtype=tf.float32, - initializer=tf.ones_initializer()) + shape=[out_channels_0], + dtype=tf.float32, + initializer=tf.ones_initializer()) weights_0 = tf.reshape(g_0, [1, 1, 1, out_channels_0]) * tf.nn.l2_normalize(weights_0, [0, 1, 2]) conv_0 = tf.nn.conv3d(x, weights_0, strides=[1, 1, 1, 1, 1], padding="VALID") - log.info('conv_0 shape: %s' %conv_0.shape) + log.info('conv_0 shape: %s' % conv_0.shape) conv_0 = conv_0 + bias_0 ####### ''' @@ -486,49 +494,50 @@ def CNN_3d_change(x, out_channels_0, out_channels_1, add_relu=True): conv_0 = tf.nn.elu(conv_0) pooling_0 = tf.nn.max_pool3d( - conv_0, + conv_0, ksize=[1, 2, 3, 3, 1], - strides=[1, 2, 3, 3, 1], + strides=[1, 2, 3, 3, 1], padding="VALID") - log.info('pooling_0 shape: %s' %pooling_0.shape) + log.info('pooling_0 shape: %s' % pooling_0.shape) - #layer_1 + # layer_1 weights_1 = tf.get_variable( name='filter_1', shape=[2, 2, 2, out_channels_0, out_channels_1], dtype=tf.float32, initializer=tf.random_uniform_initializer(-0.01, 0.01)) - + bias_1 = tf.get_variable( name='bias_1', shape=[out_channels_1], dtype=tf.float32, initializer=tf.zeros_initializer()) - + g_1 = tf.get_variable(name='scale_1', - shape = [out_channels_1], - dtype=tf.float32, - initializer=tf.ones_initializer()) + shape=[out_channels_1], + dtype=tf.float32, + initializer=tf.ones_initializer()) weights_1 = tf.reshape(g_1, [1, 1, 1, out_channels_1]) * tf.nn.l2_normalize(weights_1, [0, 1, 2]) conv_1 = tf.nn.conv3d(pooling_0, weights_1, strides=[1, 1, 1, 1, 1], padding="VALID") - log.info('conv_1 shape: %s' %conv_1.shape) + log.info('conv_1 shape: %s' % conv_1.shape) conv_1 = conv_1 + bias_1 - #with tf.variable_scope('layer_1'): + # with tf.variable_scope('layer_1'): # conv_1 = op.layer_norm(conv_1, axis=[1, 2, 3, 4]) if add_relu: conv_1 = tf.nn.elu(conv_1) pooling_1 = tf.nn.max_pool3d( - conv_1, + conv_1, ksize=[1, 3, 3, 3, 1], - strides=[1, 3, 3, 3, 1], + strides=[1, 3, 3, 3, 1], padding="VALID") - log.info('pooling_1 shape: %s' %pooling_1.shape) + log.info('pooling_1 shape: %s' % pooling_1.shape) return tf.contrib.layers.flatten(pooling_1) + def RNN_last_state(x, lengths, hidden_size): '''encode x with a gru cell and return the last state. @@ -544,5 +553,3 @@ def RNN_last_state(x, lengths, hidden_size): cell = tf.nn.rnn_cell.GRUCell(hidden_size) outputs, last_states = tf.nn.dynamic_rnn(cell, x, lengths, dtype=tf.float32) return outputs, last_states - - diff --git a/deeppavlov/models/ranking/matching_models/dam_utils/operations.py b/deeppavlov/models/ranking/matching_models/dam_utils/operations.py index 402f427083..a6bd6a5fee 100644 --- a/deeppavlov/models/ranking/matching_models/dam_utils/operations.py +++ b/deeppavlov/models/ranking/matching_models/dam_utils/operations.py @@ -25,27 +25,28 @@ # # Based on authors' Tensorflow code: https://github.com/baidu/Dialogue/tree/master/DAM -from logging import getLogger import math +from logging import getLogger import numpy as np import tensorflow as tf - from scipy.stats import multivariate_normal log = getLogger(__name__) def learning_rate(step_num, d_model=512, warmup_steps=4000): - a = step_num**(-0.5) - b = step_num*warmup_steps**(-1.5) - return a, b, d_model**(-0.5) * min(step_num**(-0.5), step_num*(warmup_steps**(-1.5))) + a = step_num ** (-0.5) + b = step_num * warmup_steps ** (-1.5) + return a, b, d_model ** (-0.5) * min(step_num ** (-0.5), step_num * (warmup_steps ** (-1.5))) + def selu(x): alpha = 1.6732632423543772848170429916717 scale = 1.0507009873554804934193349852946 log.info('use selu') - return scale*tf.where(x>=0.0, x, alpha*tf.nn.elu(x)) + return scale * tf.where(x >= 0.0, x, alpha * tf.nn.elu(x)) + def bilinear_sim_4d(x, y, is_nor=True): '''calulate bilinear similarity with two 4d tensor. @@ -63,7 +64,7 @@ def bilinear_sim_4d(x, y, is_nor=True): bilinear matrix reuse error. ''' M = tf.get_variable( - name="bilinear_matrix", + name="bilinear_matrix", shape=[x.shape[2], y.shape[2], x.shape[3]], dtype=tf.float32, initializer=tf.orthogonal_initializer()) @@ -91,7 +92,7 @@ def bilinear_sim(x, y, is_nor=True): bilinear matrix reuse error. ''' M = tf.get_variable( - name="bilinear_matrix", + name="bilinear_matrix", shape=[x.shape[-1], y.shape[-1]], dtype=tf.float32, # initializer=tf.orthogonal_initializer()) @@ -105,6 +106,7 @@ def bilinear_sim(x, y, is_nor=True): else: return sim + def dot_sim(x, y, is_nor=True): '''calculate dot similarity with two tensor. @@ -129,6 +131,7 @@ def dot_sim(x, y, is_nor=True): else: return sim + def layer_norm(x, axis=None, epsilon=1e-6): '''Add layer normalization. @@ -158,10 +161,11 @@ def layer_norm(x, axis=None, epsilon=1e-6): mean = tf.reduce_mean(x, axis=axis, keepdims=True) variance = tf.reduce_mean(tf.square(x - mean), axis=axis, keepdims=True) - norm = (x-mean) * tf.rsqrt(variance + epsilon) + norm = (x - mean) * tf.rsqrt(variance + epsilon) return scale * norm + bias -def layer_norm_debug(x, axis = None, epsilon=1e-6): + +def layer_norm_debug(x, axis=None, epsilon=1e-6): '''Add layer normalization. Args: @@ -190,9 +194,10 @@ def layer_norm_debug(x, axis = None, epsilon=1e-6): mean = tf.reduce_mean(x, axis=axis, keepdims=True) variance = tf.reduce_mean(tf.square(x - mean), axis=axis, keepdims=True) - norm = (x-mean) * tf.rsqrt(variance + epsilon) + norm = (x - mean) * tf.rsqrt(variance + epsilon) return scale * norm + bias + def dense(x, out_dimension=None, add_bias=True, initializer=tf.orthogonal_initializer()): '''Add dense connected layer, Wx + b. @@ -223,6 +228,7 @@ def dense(x, out_dimension=None, add_bias=True, initializer=tf.orthogonal_initia else: return tf.einsum('bik,kj->bij', x, W) + def matmul_2d(x, out_dimension, drop_prob=None): '''Multiplies 2-d tensor by weights. @@ -246,11 +252,12 @@ def matmul_2d(x, out_dimension, drop_prob=None): return tf.matmul(x, W) + def gauss_positional_encoding_vector(x, role=0, value=0): position = int(x.shape[1]) dimension = int(x.shape[2]) - log.info('position: %s' %position) - log.info('dimension: %s' %dimension) + log.info('position: %s' % position) + log.info('dimension: %s' % dimension) _lambda = tf.get_variable( name='lambda', @@ -259,20 +266,19 @@ def gauss_positional_encoding_vector(x, role=0, value=0): initializer=tf.constant_initializer(value)) _lambda = tf.expand_dims(_lambda, axis=-1) - mean = [position/2.0, dimension/2.0] + mean = [position / 2.0, dimension / 2.0] - #cov = [[position/3.0, 0], [0, dimension/3.0]] - sigma_x = position/math.sqrt(4.0*dimension) - sigma_y = math.sqrt(dimension/4.0) - cov = [[sigma_x*sigma_x, role*sigma_x*sigma_y], - [role*sigma_x*sigma_y, sigma_y*sigma_y]] + # cov = [[position/3.0, 0], [0, dimension/3.0]] + sigma_x = position / math.sqrt(4.0 * dimension) + sigma_y = math.sqrt(dimension / 4.0) + cov = [[sigma_x * sigma_x, role * sigma_x * sigma_y], + [role * sigma_x * sigma_y, sigma_y * sigma_y]] pos = np.dstack(np.mgrid[0:position, 0:dimension]) - rv = multivariate_normal(mean, cov) - signal = rv.pdf(pos) - signal = signal - np.max(signal)/2.0 + signal = rv.pdf(pos) + signal = signal - np.max(signal) / 2.0 signal = tf.multiply(_lambda, signal) signal = tf.expand_dims(signal, axis=0) @@ -281,6 +287,7 @@ def gauss_positional_encoding_vector(x, role=0, value=0): return x + _lambda * signal + def positional_encoding(x, min_timescale=1.0, max_timescale=1.0e4, value=0): '''Adds a bunch of sinusoids of different frequencies to a tensor. @@ -305,14 +312,14 @@ def positional_encoding(x, min_timescale=1.0, max_timescale=1.0e4, value=0): position = tf.to_float(tf.range(length)) num_timescales = channels // 2 log_timescale_increment = ( - math.log(float(max_timescale) / float(min_timescale)) / - (tf.to_float(num_timescales) - 1)) + math.log(float(max_timescale) / float(min_timescale)) / + (tf.to_float(num_timescales) - 1)) inv_timescales = min_timescale * tf.exp( tf.to_float(tf.range(num_timescales)) * -log_timescale_increment) scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0) signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1) signal = tf.pad(signal, [[0, 0], [0, tf.mod(channels, 2)]]) - #signal = tf.reshape(signal, [1, length, channels]) + # signal = tf.reshape(signal, [1, length, channels]) signal = tf.expand_dims(signal, axis=0) return x + _lambda * signal @@ -343,8 +350,8 @@ def positional_encoding_vector(x, min_timescale=1.0, max_timescale=1.0e4, value= position = tf.to_float(tf.range(length)) num_timescales = channels // 2 log_timescale_increment = ( - math.log(float(max_timescale) / float(min_timescale)) / - (tf.to_float(num_timescales) - 1)) + math.log(float(max_timescale) / float(min_timescale)) / + (tf.to_float(num_timescales) - 1)) inv_timescales = min_timescale * tf.exp( tf.to_float(tf.range(num_timescales)) * -log_timescale_increment) scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0) @@ -356,6 +363,7 @@ def positional_encoding_vector(x, min_timescale=1.0, max_timescale=1.0e4, value= return x + signal + def mask(row_lengths, col_lengths, max_row_length, max_col_length): '''Return a mask tensor representing the first N positions of each row and each column. @@ -368,14 +376,15 @@ def mask(row_lengths, col_lengths, max_row_length, max_col_length): Raises: ''' - row_mask = tf.sequence_mask(row_lengths, max_row_length) #bool, [batch, max_row_len] - col_mask = tf.sequence_mask(col_lengths, max_col_length) #bool, [batch, max_col_len] + row_mask = tf.sequence_mask(row_lengths, max_row_length) # bool, [batch, max_row_len] + col_mask = tf.sequence_mask(col_lengths, max_col_length) # bool, [batch, max_col_len] row_mask = tf.cast(tf.expand_dims(row_mask, -1), tf.float32) col_mask = tf.cast(tf.expand_dims(col_mask, -1), tf.float32) return tf.einsum('bik,bjk->bij', row_mask, col_mask) + def weighted_sum(weight, values): '''Calcualte the weighted sum. @@ -389,7 +398,3 @@ def weighted_sum(weight, values): Raises: ''' return tf.einsum('bij,bjk->bik', weight, values) - - - - diff --git a/deeppavlov/models/ranking/metrics.py b/deeppavlov/models/ranking/metrics.py index bb899e9425..98629c61de 100644 --- a/deeppavlov/models/ranking/metrics.py +++ b/deeppavlov/models/ranking/metrics.py @@ -28,12 +28,14 @@ def rank_response(y_true, y_pred): if x == 0: rank_tot += i break - return float(rank_tot)/num_examples + return float(rank_tot) / num_examples + @register_metric('r@1_insQA') def r_at_1_insQA(y_true, y_pred): return recall_at_k_insQA(y_true, y_pred, k=1) + def recall_at_k_insQA(y_true, y_pred, k): labels = np.repeat(np.expand_dims(np.asarray(y_true), axis=1), k, axis=1) predictions = np.array(y_pred) @@ -44,4 +46,3 @@ def recall_at_k_insQA(y_true, y_pred, k): if predictions[i][j] in np.arange(labels[i][j]): flags[i][j] = 1. return np.mean((np.sum(flags, -1) >= 1.).astype(float)) - diff --git a/deeppavlov/models/ranking/mpm_siamese_network.py b/deeppavlov/models/ranking/mpm_siamese_network.py index 61e685d433..e629372e03 100644 --- a/deeppavlov/models/ranking/mpm_siamese_network.py +++ b/deeppavlov/models/ranking/mpm_siamese_network.py @@ -31,7 +31,6 @@ @register('mpm_nn') class MPMSiameseNetwork(BiLSTMSiameseNetwork): - """The class implementing a siamese neural network with bilateral multi-Perspective matching. The network architecture is based on https://arxiv.org/abs/1702.03814. @@ -89,7 +88,7 @@ def create_lstm_layer_2(self): ker_in = glorot_uniform(seed=self.seed) rec_in = Orthogonal(seed=self.seed) bioutp = Bidirectional(LSTM(self.aggregation_dim, - input_shape=(self.max_sequence_length, 8*self.perspective_num,), + input_shape=(self.max_sequence_length, 8 * self.perspective_num,), kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, @@ -123,11 +122,11 @@ def create_model(self) -> Model: f_layer_b = FullMatchingLayer(self.perspective_num) f_a_forw = f_layer_f([lstm_a[0], lstm_b[0]])[0] f_a_back = f_layer_b([Lambda(lambda x: K.reverse(x, 1))(lstm_a[1]), - Lambda(lambda x: K.reverse(x, 1))(lstm_b[1])])[0] + Lambda(lambda x: K.reverse(x, 1))(lstm_b[1])])[0] f_a_back = Lambda(lambda x: K.reverse(x, 1))(f_a_back) f_b_forw = f_layer_f([lstm_b[0], lstm_a[0]])[0] f_b_back = f_layer_b([Lambda(lambda x: K.reverse(x, 1))(lstm_b[1]), - Lambda(lambda x: K.reverse(x, 1))(lstm_a[1])])[0] + Lambda(lambda x: K.reverse(x, 1))(lstm_a[1])])[0] f_b_back = Lambda(lambda x: K.reverse(x, 1))(f_b_back) mp_layer_f = MaxpoolingMatchingLayer(self.perspective_num) @@ -179,4 +178,4 @@ def create_model(self) -> Model: dense = Dense(self.dense_dim, kernel_initializer=ker_in)(reduced) dist = Dense(1, activation='sigmoid', name="score_model")(dense) model = Model([context, response], dist) - return model \ No newline at end of file + return model diff --git a/deeppavlov/models/ranking/sequential_matching_network.py b/deeppavlov/models/ranking/sequential_matching_network.py index 4b006caf7b..a9222897af 100644 --- a/deeppavlov/models/ranking/sequential_matching_network.py +++ b/deeppavlov/models/ranking/sequential_matching_network.py @@ -54,7 +54,6 @@ def __init__(self, *args, **kwargs): - self.max_sentence_len = max_sequence_length self.word_embedding_size = embedding_dim self.trainable = trainable_embeddings diff --git a/deeppavlov/models/ranking/siamese_model.py b/deeppavlov/models/ranking/siamese_model.py index b32558c705..64eb8b2d7f 100644 --- a/deeppavlov/models/ranking/siamese_model.py +++ b/deeppavlov/models/ranking/siamese_model.py @@ -87,7 +87,7 @@ def __call__(self, samples_generator: Iterable[List[np.ndarray]]) -> Union[np.nd n_responses = self._append_sample_to_batch_buffer(sample, buf) if len(buf) >= self.batch_size: for i in range(len(buf) // self.batch_size): - b = self._make_batch(buf[i*self.batch_size:(i+1)*self.batch_size]) + b = self._make_batch(buf[i * self.batch_size:(i + 1) * self.batch_size]) yp = self._predict_on_batch(b) y_pred += list(yp) lenb = len(buf) % self.batch_size @@ -133,4 +133,3 @@ def _make_batch(self, x: List[List[np.ndarray]]) -> List[np.ndarray]: z = [el[i] for el in x] b.append(np.asarray(z)) return b - diff --git a/deeppavlov/models/ranking/siamese_predictor.py b/deeppavlov/models/ranking/siamese_predictor.py index f09c7157ed..a42dccc22b 100644 --- a/deeppavlov/models/ranking/siamese_predictor.py +++ b/deeppavlov/models/ranking/siamese_predictor.py @@ -24,6 +24,7 @@ log = getLogger(__name__) + @register('siamese_predictor') class SiamesePredictor(Component): """The class for ranking or paraphrase identification using the trained siamese network in the ``interact`` mode. @@ -77,7 +78,7 @@ def __init__(self, if not self.attention: self._build_response_embeddings() - def __call__(self, batch: Iterable[List[np.ndarray]]) -> List[Union[List[str],str]]: + def __call__(self, batch: Iterable[List[np.ndarray]]) -> List[Union[List[str], str]]: context = next(batch) try: next(batch) @@ -85,13 +86,12 @@ def __call__(self, batch: Iterable[List[np.ndarray]]) -> List[Union[List[str],st except StopIteration: pass - if self.ranking: if len(context) == self.num_context_turns: scores = [] if self.attention: for i in range(len(self.preproc_responses) // self.batch_size + 1): - responses = self.preproc_responses[i*self.batch_size: (i+1)*self.batch_size] + responses = self.preproc_responses[i * self.batch_size: (i + 1) * self.batch_size] b = [context + el for el in responses] b = self.model._make_batch(b) sc = self.model._predict_on_batch(b) @@ -126,7 +126,7 @@ def process_event(self) -> None: def _build_response_embeddings(self) -> None: resp_vecs = [] for i in range(len(self.preproc_responses) // self.batch_size + 1): - resp_preproc = self.preproc_responses[i*self.batch_size: (i+1)*self.batch_size] + resp_preproc = self.preproc_responses[i * self.batch_size: (i + 1) * self.batch_size] resp_preproc = self.model._make_batch(resp_preproc) resp_preproc = resp_preproc resp_vecs.append(self.model._predict_response_on_batch(resp_preproc)) @@ -135,7 +135,7 @@ def _build_response_embeddings(self) -> None: def _build_preproc_responses(self) -> None: responses = list(self.responses.values()) for i in range(len(responses) // self.batch_size + 1): - el = self.preproc_func(responses[i*self.batch_size: (i+1)*self.batch_size]) + el = self.preproc_func(responses[i * self.batch_size: (i + 1) * self.batch_size]) self.preproc_responses += list(el) def rebuild_responses(self, candidates) -> None: @@ -144,6 +144,3 @@ def rebuild_responses(self, candidates) -> None: self.preproc_responses = list() self.responses = {idx: sentence for idx, sentence in enumerate(candidates)} self._build_preproc_responses() - - - diff --git a/deeppavlov/models/ranking/tf_base_matching_model.py b/deeppavlov/models/ranking/tf_base_matching_model.py index debf88c653..8255777143 100644 --- a/deeppavlov/models/ranking/tf_base_matching_model.py +++ b/deeppavlov/models/ranking/tf_base_matching_model.py @@ -52,7 +52,8 @@ def __init__(self, *args, **kwargs) self.use_logits = use_logits if mean_oov: - self.emb_matrix[1] = np.mean(self.emb_matrix[2:], axis=0) # set mean embedding for OOV token at the 2nd index + self.emb_matrix[1] = np.mean(self.emb_matrix[2:], + axis=0) # set mean embedding for OOV token at the 2nd index def _append_sample_to_batch_buffer(self, sample: List[np.ndarray], buf: List[Tuple]) -> int: """ @@ -65,9 +66,9 @@ def _append_sample_to_batch_buffer(self, sample: List[np.ndarray], buf: List[Tup a number of candidate responses """ # - batch_buffer_context = [] # [batch_size, 10, 50] - batch_buffer_context_len = [] # [batch_size, 10] - batch_buffer_response = [] # [batch_size, 50] + batch_buffer_context = [] # [batch_size, 10, 50] + batch_buffer_context_len = [] # [batch_size, 10] + batch_buffer_response = [] # [batch_size, 50] batch_buffer_response_len = [] # [batch_size] context_sentences = sample[:self.num_context_turns] @@ -160,4 +161,4 @@ def _train_on_batch(self, batch: Dict, y: List[int]) -> float: float: value of mean loss on the batch """ batch.update({self.y_true: np.array(y)}) - return self.sess.run([self.loss, self.train_op], feed_dict=batch)[0] # return the first item aka loss \ No newline at end of file + return self.sess.run([self.loss, self.train_op], feed_dict=batch)[0] # return the first item aka loss diff --git a/deeppavlov/models/seq2seq_go_bot/bot.py b/deeppavlov/models/seq2seq_go_bot/bot.py index 60bf6108b7..9a309c0dfd 100644 --- a/deeppavlov/models/seq2seq_go_bot/bot.py +++ b/deeppavlov/models/seq2seq_go_bot/bot.py @@ -46,6 +46,7 @@ class Seq2SeqGoalOrientedBot(NNModel): **kwargs: parameters passed to parent :class:`~deeppavlov.core.models.nn_model.NNModel` class. """ + def __init__(self, network_parameters: Dict, embedder: Component, @@ -95,7 +96,7 @@ def _init_network(self, params): return Seq2SeqGoalOrientedBotNetwork(**params) def _embed_kb_key(self, key): -# TODO: fasttext embedder to work with tokens + # TODO: fasttext embedder to work with tokens emb = np.array(self.embedder([key.split('_')], mean=True)[0]) if self.debug: log.debug("embedding key tokens='{}', embedding shape = {}" @@ -124,10 +125,10 @@ def train_on_batch(self, utters, history_list, kb_entry_list, responses): # np.ones((batch_size, max_src_len), dtype=np.float32) b_enc_ins_np = np.zeros((batch_size, max_src_len, self.embedding_size), dtype=np.float32) - b_dec_ins_np = self.tgt_vocab[self.eos_token] *\ - np.ones((batch_size, max_tgt_len), dtype=np.float32) - b_dec_outs_np = self.tgt_vocab[self.eos_token] *\ - np.ones((batch_size, max_tgt_len), dtype=np.float32) + b_dec_ins_np = self.tgt_vocab[self.eos_token] * \ + np.ones((batch_size, max_tgt_len), dtype=np.float32) + b_dec_outs_np = self.tgt_vocab[self.eos_token] * \ + np.ones((batch_size, max_tgt_len), dtype=np.float32) b_tgt_weights_np = np.zeros((batch_size, max_tgt_len), dtype=np.float32) b_kb_masks_np = np.zeros((batch_size, self.kb_size), np.float32) for i, (src_len, tgt_len, kb_entries) in \ @@ -184,6 +185,7 @@ def _idx2token(idxs): yield token else: yield self.kb_keys[idx - self.tgt_vocab_size] + return [list(_idx2token(utter_idxs)) for utter_idxs in token_idxs] def __call__(self, *batch): @@ -225,4 +227,3 @@ def save(self): def load(self): pass - diff --git a/deeppavlov/models/seq2seq_go_bot/dialog_state.py b/deeppavlov/models/seq2seq_go_bot/dialog_state.py index 49f8f0e463..48e1f6b4c7 100644 --- a/deeppavlov/models/seq2seq_go_bot/dialog_state.py +++ b/deeppavlov/models/seq2seq_go_bot/dialog_state.py @@ -30,4 +30,3 @@ def __call__(self, user_ids, utterances=None, *args, **kwargs): for user, utter in zip(user_ids, utterances): self.states[user] = self.states.get(user, []) + utter return - diff --git a/deeppavlov/models/seq2seq_go_bot/kb.py b/deeppavlov/models/seq2seq_go_bot/kb.py index 5b2cc252c1..aecde5626b 100644 --- a/deeppavlov/models/seq2seq_go_bot/kb.py +++ b/deeppavlov/models/seq2seq_go_bot/kb.py @@ -53,6 +53,7 @@ class KnowledgeBase(Estimator): **kwargs: parameters passed to parent :class:`~deeppavlov.core.models.estimator.Estimator`. """ + def __init__(self, save_path: str, load_path: str = None, @@ -82,6 +83,7 @@ def _update(self, keys, kb_columns_list, kb_items_list, update_primary_keys=True def _key_value_entries(self, kb_item, kb_columns, update=True): def _format(s): return re.sub('\s+', '_', s.lower().strip()) + first_key = _format(kb_item[kb_columns[0]]) for col in kb_columns: key = first_key + '_' + _format(col) @@ -182,18 +184,18 @@ def normalize(self, tokens, entries): ent_num_tokens = len(ent_tokens) if ' '.join(ent_tokens).strip(): for i in range(len(tokens)): - if tokens[i:i+ent_num_tokens] == ent_tokens: + if tokens[i:i + ent_num_tokens] == ent_tokens: if self.remove: - tokens = tokens[:i] + tokens[i+ent_num_tokens:] + tokens = tokens[:i] + tokens[i + ent_num_tokens:] else: - tokens = tokens[:i] + [entity] + tokens[i+ent_num_tokens:] + tokens = tokens[:i] + [entity] + tokens[i + ent_num_tokens:] return tokens def denormalize(self, tokens, entries): for entity, ent_tokens in entries: while (entity in tokens): ent_pos = tokens.index(entity) - tokens = tokens[:ent_pos] + ent_tokens + tokens[ent_pos+1:] + tokens = tokens[:ent_pos] + ent_tokens + tokens[ent_pos + 1:] return tokens def __call__(self, diff --git a/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py b/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py index c93465be84..3f9c6be5d1 100644 --- a/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py +++ b/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py @@ -21,7 +21,7 @@ class KBAttention(base.Layer): -# TODO: update class doc + # TODO: update class doc """Densely-connected layer class. Arguments: units: Integer or Long, dimensionality of the output space. @@ -104,10 +104,10 @@ def __init__(self, units, hidden_sizes, "_reuse": reuse } # print("KB shape =", self.kb_input_shape) - + def build(self, input_shape): # if in_shape[:-1] != self.kb_inputs.shape -# TODO: check input shape + # TODO: check input shape # print("in build") in_shape = input_shape[:1].concatenate(self.kb_input_shape) in_shape = in_shape[:-1].concatenate(in_shape[-1] + input_shape[-1]) @@ -120,7 +120,7 @@ def build(self, input_shape): layer = tf.layers.Dense(size, name=name, _scope=name, **self.dense_params) layer.build(in_shape) in_shape = layer.compute_output_shape(in_shape) - + self.layers.append(layer) # print("input_shape =", input_shape) @@ -131,10 +131,10 @@ def build(self, input_shape): self.output_layer.build(input_shape) # print("build = True") self.built = True - + def call(self, inputs): # print("in call") -# TODO: check input dtype + # TODO: check input dtype # Tile kb_inputs kb_inputs = self.kb_inputs diff --git a/deeppavlov/models/seq2seq_go_bot/network.py b/deeppavlov/models/seq2seq_go_bot/network.py index fecb982f5f..758006e804 100644 --- a/deeppavlov/models/seq2seq_go_bot/network.py +++ b/deeppavlov/models/seq2seq_go_bot/network.py @@ -179,7 +179,7 @@ def _build_graph(self): tf.verify_tensor_all_finite(_loss_tensor, "Non finite values in loss tensor.") self._loss = tf.reduce_sum(_loss_tensor) / tf.cast(self._batch_size, tf.float32) # self._loss = tf.reduce_mean(_loss_tensor, name='loss') -# TODO: tune clip_norm + # TODO: tune clip_norm self._train_op = \ self.get_train_op(self._loss, learning_rate=self._learning_rate, @@ -221,7 +221,7 @@ def _add_placeholders(self): [None, None], name='decoder_outputs') # _kb_embedding: [kb_size, embedding_size] -# TODO: try training embeddings + # TODO: try training embeddings kb_W = np.array(self.kb_embedding)[:, :self.embedding_size] self._kb_embedding = tf.get_variable("kb_embedding", shape=(kb_W.shape[0], kb_W.shape[1]), @@ -231,7 +231,7 @@ def _add_placeholders(self): # _kb_mask: [batch_size, kb_size] self._kb_mask = tf.placeholder(tf.float32, [None, None], name='kb_mask') -# TODO: compute sequence lengths on the go + # TODO: compute sequence lengths on the go # _src_sequence_lengths, _tgt_sequence_lengths: [batch_size] self._src_sequence_lengths = tf.placeholder(tf.int32, [None], @@ -272,7 +272,7 @@ def _build_encoder(self): # Run Dynamic RNN # _encoder_outputs: [max_time, batch_size, hidden_size] # _encoder_state: [batch_size, hidden_size] -# input_states? + # input_states? _encoder_outputs, _encoder_state = tf.nn.dynamic_rnn( _encoder_cell, _encoder_emb_inp, dtype=tf.float32, sequence_length=self._src_sequence_lengths, time_major=False) @@ -346,8 +346,8 @@ def build_dec_cell(enc_out, enc_seq_len, reuse=None): _decoder_emb_inp, self._tgt_sequence_lengths, time_major=False) # Copy encoder hidden state to decoder inital state _decoder_init_state = \ - _decoder_cell_tr.zero_state(self._batch_size, dtype=tf.float32)\ - .clone(cell_state=self._encoder_state) + _decoder_cell_tr.zero_state(self._batch_size, dtype=tf.float32) \ + .clone(cell_state=self._encoder_state) _decoder_tr = \ tf.contrib.seq2seq.BasicDecoder(_decoder_cell_tr, _helper_tr, initial_state=_decoder_init_state, @@ -377,21 +377,21 @@ def build_dec_cell(enc_out, enc_seq_len, reuse=None): # Decoder Init State _decoder_init_state = \ _decoder_cell_inf.zero_state(tf.shape(_tiled_encoder_outputs)[0], - dtype=tf.float32)\ - .clone(cell_state=_tiled_encoder_state) + dtype=tf.float32) \ + .clone(cell_state=_tiled_encoder_state) # Define a beam-search decoder _start_tokens = tf.tile(tf.constant([self.tgt_sos_id], tf.int32), [self._batch_size]) # _start_tokens = tf.fill([self._batch_size], self.tgt_sos_id) _decoder_inf = tf.contrib.seq2seq.BeamSearchDecoder( - cell=_decoder_cell_inf, - embedding=self._decoder_embedding, - start_tokens=_start_tokens, - end_token=self.tgt_eos_id, - initial_state=_decoder_init_state, - beam_width=self.beam_width, - output_layer=_kb_attn_layer, - length_penalty_weight=0.0) + cell=_decoder_cell_inf, + embedding=self._decoder_embedding, + start_tokens=_start_tokens, + end_token=self.tgt_eos_id, + initial_state=_decoder_init_state, + beam_width=self.beam_width, + output_layer=_kb_attn_layer, + length_penalty_weight=0.0) # Wrap into variable scope to share attention parameters # Required! @@ -421,7 +421,7 @@ def __call__(self, enc_inputs, src_seq_lengths, kb_masks, prob=False): self._kb_mask: kb_masks } ) -# TODO: implement infer probabilities + # TODO: implement infer probabilities if prob: raise NotImplementedError("Probs not available for now.") return predictions @@ -449,8 +449,8 @@ def get_learning_rate(self): # polynomial decay global_step = min(self.global_step, self.decay_steps) decayed_learning_rate = \ - (self.learning_rate - self.end_learning_rate) *\ - (1 - global_step / self.decay_steps) ** self.decay_power +\ + (self.learning_rate - self.end_learning_rate) * \ + (1 - global_step / self.decay_steps) ** self.decay_power + \ self.end_learning_rate return decayed_learning_rate @@ -465,9 +465,9 @@ def load_params(self): params = json.load(fp) for p in self.GRAPH_PARAMS: if self.opt.get(p) != params.get(p): - if p in ('kb_embedding_control_sum') and\ + if p in ('kb_embedding_control_sum') and \ (math.abs(self.opt.get(p, 0.) - params.get(p, 0.)) < 1e-3): - continue + continue raise ConfigError("`{}` parameter must be equal to saved model" " parameter value `{}`, but is equal to `{}`" .format(p, params.get(p), self.opt.get(p))) diff --git a/deeppavlov/models/sklearn/sklearn_component.py b/deeppavlov/models/sklearn/sklearn_component.py index ce81175602..3bca87c18e 100644 --- a/deeppavlov/models/sklearn/sklearn_component.py +++ b/deeppavlov/models/sklearn/sklearn_component.py @@ -60,6 +60,7 @@ class SklearnComponent(Estimator): e.g. ``predict``, ``predict_proba``, ``predict_log_proba``, ``transform`` ensure_list_output: whether to ensure that output for each sample is iterable (but not string) """ + def __init__(self, model_class: str, save_path: Union[str, Path] = None, load_path: Union[str, Path] = None, diff --git a/deeppavlov/models/slotfill/slotfill.py b/deeppavlov/models/slotfill/slotfill.py index 87e8ce6d74..b977225e87 100644 --- a/deeppavlov/models/slotfill/slotfill.py +++ b/deeppavlov/models/slotfill/slotfill.py @@ -29,6 +29,7 @@ @register('dstc_slotfilling') class DstcSlotFillingNetwork(Component, Serializable): """Slot filling for DSTC2 task with neural network""" + def __init__(self, threshold: float = 0.8, **kwargs): super().__init__(**kwargs) self.threshold = threshold diff --git a/deeppavlov/models/slotfill/slotfill_raw.py b/deeppavlov/models/slotfill/slotfill_raw.py index 2689eb2349..9320cb7ff0 100644 --- a/deeppavlov/models/slotfill/slotfill_raw.py +++ b/deeppavlov/models/slotfill/slotfill_raw.py @@ -29,6 +29,7 @@ @register('slotfill_raw') class SlotFillingComponent(Component, Serializable): """Slot filling using Fuzzy search""" + def __init__(self, threshold: float = 0.7, return_all: bool = False, **kwargs): super().__init__(**kwargs) self.threshold = threshold diff --git a/deeppavlov/models/spelling_correction/brillmoore/error_model.py b/deeppavlov/models/spelling_correction/brillmoore/error_model.py index fae86445ca..9d533e9556 100644 --- a/deeppavlov/models/spelling_correction/brillmoore/error_model.py +++ b/deeppavlov/models/spelling_correction/brillmoore/error_model.py @@ -47,7 +47,7 @@ class ErrorModel(Estimator): candidates_count: maximum number of replacement candidates to return for every token in the input """ - def __init__(self, dictionary: StaticDictionary, window: int=1, candidates_count: int=1, *args, **kwargs): + def __init__(self, dictionary: StaticDictionary, window: int = 1, candidates_count: int = 1, *args, **kwargs): super().__init__(*args, **kwargs) self.costs = defaultdict(itertools.repeat(float('-inf')).__next__) self.dictionary = dictionary diff --git a/deeppavlov/models/spelling_correction/electors/kenlm_elector.py b/deeppavlov/models/spelling_correction/electors/kenlm_elector.py index deb3107ad7..4eb7978be6 100644 --- a/deeppavlov/models/spelling_correction/electors/kenlm_elector.py +++ b/deeppavlov/models/spelling_correction/electors/kenlm_elector.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kenlm from logging import getLogger from pathlib import Path from typing import List, Tuple +import kenlm + from deeppavlov.core.commands.utils import expand_path from deeppavlov.core.common.registry import register from deeppavlov.core.models.component import Component @@ -36,7 +37,8 @@ class KenlmElector(Component): lm: kenlm object beam_size: beam size for highest probability search """ - def __init__(self, load_path: Path, beam_size: int=4, *args, **kwargs): + + def __init__(self, load_path: Path, beam_size: int = 4, *args, **kwargs): self.lm = kenlm.Model(str(expand_path(load_path))) self.beam_size = beam_size diff --git a/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py b/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py index 3799ec16e0..887ac927f0 100644 --- a/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py +++ b/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py @@ -13,6 +13,7 @@ class LevenshteinSearcher: в соответствии с расстоянием Левенштейна """ + def __init__(self, alphabet, dictionary, operation_costs=None, allow_spaces=False, euristics='none'): self.alphabet = alphabet @@ -65,7 +66,7 @@ def _trie_search(self, word, d, transducer=None, trie = self.dictionary # инициализация переменных used_agenda_keys = set() - agenda = SortedListWithKey(key=(lambda x:x[1])) + agenda = SortedListWithKey(key=(lambda x: x[1])) h = self.h_func(word, trie.root) # agenda[self.agenda_key("", 0, trie.root)] = (0.0, 0.0, h) key, value = ("", 0, trie.root), (0.0, 0.0, h) @@ -91,7 +92,7 @@ def _trie_search(self, word, d, transducer=None, continue for curr_low, curr_cost in transducer.operation_costs[curr_up].items(): new_g = g + curr_cost - if new_g > d: #если g > d, то h можно не вычислять + if new_g > d: # если g > d, то h можно не вычислять continue if curr_low == " ": if allow_spaces and trie.is_final(index): @@ -103,7 +104,7 @@ def _trie_search(self, word, d, transducer=None, if new_index is Trie.NO_NODE: continue new_low = low + curr_low - new_h = self.h_func(word[new_pos: ], new_index) + new_h = self.h_func(word[new_pos:], new_index) new_cost = new_g + new_h if new_cost > d: continue @@ -129,8 +130,8 @@ def _precompute_euristics(self): return # вычисление минимальной стоимости операции, # приводящей к появлению ('+') или исчезновению ('-') данного символа - removal_costs = {a : np.inf for a in self.alphabet} - insertion_costs = {a : np.inf for a in self.alphabet} + removal_costs = {a: np.inf for a in self.alphabet} + insertion_costs = {a: np.inf for a in self.alphabet} if self.allow_spaces: removal_costs[' '] = np.inf insertion_costs[' '] = np.inf @@ -250,10 +251,10 @@ def _precompute_absense_costs(dictionary, removal_costs, insertion_costs, n, curr_node_removal_costs[0] = min(removal_costs[symbol] for symbol in node[0]) for j, symbols in enumerate(node[1:], 1): if len(symbols) == 0: - curr_node_removal_costs[j:] = curr_node_removal_costs[j-1] + curr_node_removal_costs[j:] = curr_node_removal_costs[j - 1] break curr_cost = min(removal_costs[symbol] for symbol in symbols) - curr_node_removal_costs[j] = min(curr_node_removal_costs[j-1], curr_cost) + curr_node_removal_costs[j] = min(curr_node_removal_costs[j - 1], curr_cost) else: curr_node_removal_costs[:] = np.inf # определение минимальной стоимости вставки @@ -288,6 +289,7 @@ class SegmentTransducer: и они равны значению по умолчанию) """ + def __init__(self, alphabet, operation_costs=None, allow_spaces=False): self.alphabet = alphabet if operation_costs is None: @@ -300,10 +302,10 @@ def __init__(self, alphabet, operation_costs=None, allow_spaces=False): self._make_maximal_key_lengths() # self.maximal_value_lengths = {} # for up, probs in self.operation_costs.items(): - # СЛИШКОМ МНОГО ВЫЗОВОВ, НАДО КАК-ТО ЗАПОМНИТЬ - # МАКСИМАЛЬНЫЕ ДЛИНЫ КЛЮЧЕЙ ПРИ ОБРАЩЕНИИ - # max_low_length = max(len(low) for low in probs) if (len(probs) > 0) else -1 - # self.maximal_value_lengths[up] = self.maximal_key_length + # СЛИШКОМ МНОГО ВЫЗОВОВ, НАДО КАК-ТО ЗАПОМНИТЬ + # МАКСИМАЛЬНЫЕ ДЛИНЫ КЛЮЧЕЙ ПРИ ОБРАЩЕНИИ + # max_low_length = max(len(low) for low in probs) if (len(probs) > 0) else -1 + # self.maximal_value_lengths[up] = self.maximal_key_length def get_operation_cost(self, up, low): """ @@ -341,7 +343,7 @@ def inverse(self): inversed_transducer.max_up_lengths_by_low = self.max_low_lengths_by_up return inversed_transducer - def distance(self, first, second, return_transduction = False): + def distance(self, first, second, return_transduction=False): """ Вычисляет трансдукцию минимальной стоимости, отображающую first в second @@ -374,7 +376,7 @@ def distance(self, first, second, return_transduction = False): clear_pred = (lambda x, y: x < y < np.inf) update_func = lambda x, y: min(x, y) costs, backtraces = self._fill_levenshtein_table(first, second, - update_func, add_pred, clear_pred) + update_func, add_pred, clear_pred) final_cost = costs[-1][-1] if final_cost == np.inf: transductions = [None] @@ -397,11 +399,11 @@ def transduce(self, first, second, threshold): список вида [(трансдукция, стоимость)] """ add_pred = (lambda x, y: x <= threshold) - clear_pred =(lambda x, y: False) + clear_pred = (lambda x, y: False) update_func = (lambda x, y: min(x, y)) costs, backtraces = self._fill_levenshtein_table(first, second, - update_func, add_pred, clear_pred, - threshold=threshold) + update_func, add_pred, clear_pred, + threshold=threshold) result = self._backtraces_to_transductions(first, second, backtraces, threshold, return_cost=True) return result @@ -430,7 +432,7 @@ def lower_transductions(self, word, max_cost, return_cost=True): for transduction, cost in prefixes[pos]: new_cost = cost + low_cost if new_cost <= max_cost: - new_transduction = transduction +(up, low) + new_transduction = transduction + (up, low) prefixes[pos + upperside_length].append((new_transduction, new_cost)) answer = sorted(prefixes[-1], key=(lambda x: x[0])) if return_cost: @@ -461,7 +463,7 @@ def upper_transductions(self, word, max_cost, return_cost=True): return inversed_transducer.lower_transductions(word, max_cost, return_cost) def _fill_levenshtein_table(self, first, second, update_func, add_pred, clear_pred, - threshold=None): + threshold=None): """ Функция, динамически заполняющая таблицу costs стоимости трансдукций, costs[i][j] --- минимальная стоимость трансдукции, @@ -502,10 +504,10 @@ def _fill_levenshtein_table(self, first, second, update_func, add_pred, clear_pr for a, b in zip(first, second): threshold += self.get_operation_cost(a, b) if m > n: - for a in first[n: ]: + for a in first[n:]: threshold += self.get_operation_cost(a, '') elif m < n: - for b in second[m: ]: + for b in second[m:]: threshold += self.get_operation_cost('', b) threshold *= 2 # инициализация возвращаемых массивов @@ -519,14 +521,14 @@ def _fill_levenshtein_table(self, first, second, update_func, add_pred, clear_pr for i_right in range(i, min(i + self.max_up_length, m) + 1): up = first[i: i_right] max_low_length = self.max_low_lengths_by_up.get(up, -1) - if max_low_length == -1: # no up key in transduction + if max_low_length == -1: # no up key in transduction continue up_costs = self.operation_costs[up] for j in range(n + 1): if costs[i][j] > threshold: continue if len(backtraces[i][j]) == 0 and i + j > 0: - continue # не нашлось обратных ссылок + continue # не нашлось обратных ссылок for j_right in range((j if i_right > i else j + 1), min(j + max_low_length, n) + 1): low = second[j: j_right] @@ -562,18 +564,18 @@ def _make_maximal_key_lengths(self): и максимальную длину элемента up в элементарной трансдукции (up, low) для каждого low """ - self.max_up_length =\ + self.max_up_length = \ (max(len(up) for up in self.operation_costs) if len(self.operation_costs) > 0 else -1) - self.max_low_length =\ + self.max_low_length = \ (max(len(low) for low in self._reversed_operation_costs) if len(self._reversed_operation_costs) > 0 else -1) self.max_low_lengths_by_up, self.max_up_lengths_by_low = dict(), dict() for up, costs in self.operation_costs.items(): - self.max_low_lengths_by_up[up] =\ + self.max_low_lengths_by_up[up] = \ max(len(low) for low in costs) if len(costs) > 0 else -1 for low, costs in self._reversed_operation_costs.items(): - self.max_up_lengths_by_low[low] =\ + self.max_up_lengths_by_low[low] = \ max(len(up) for up in costs) if len(costs) > 0 else -1 def _backtraces_to_transductions(self, first, second, backtraces, threshold, return_cost=False): @@ -603,7 +605,7 @@ def _backtraces_to_transductions(self, first, second, backtraces, threshold, ret m, n = len(first), len(second) agenda = [None] * (m + 1) for i in range(m + 1): - agenda[i] = [[] for j in range(n+1)] + agenda[i] = [[] for j in range(n + 1)] agenda[m][n] = [((), 0.0)] for i_right in range(m, -1, -1): for j_right in range(n, -1, -1): @@ -615,7 +617,7 @@ def _backtraces_to_transductions(self, first, second, backtraces, threshold, ret add_cost = self.operation_costs[up][low] for elem, cost in current_agenda: new_cost = cost + add_cost - if new_cost <= threshold: # удаление трансдукций большой стоимости + if new_cost <= threshold: # удаление трансдукций большой стоимости agenda[i][j].append((((up, low),) + elem, new_cost)) if return_cost: return agenda[0][0] diff --git a/deeppavlov/models/spelling_correction/levenshtein/tabled_trie.py b/deeppavlov/models/spelling_correction/levenshtein/tabled_trie.py index c6d91ce439..5aace19731 100644 --- a/deeppavlov/models/spelling_correction/levenshtein/tabled_trie.py +++ b/deeppavlov/models/spelling_correction/levenshtein/tabled_trie.py @@ -159,7 +159,6 @@ def words(self): branch.append(next_child) letters_with_children.append(self._get_children_and_letters(branch[-1])) - def is_final(self, index): """ Аргументы @@ -189,13 +188,13 @@ def find_partitions(self, s, max_count=1): continue next_agenda.append((child, borders, cost)) if self.is_final(child): - next_agenda.append((self.root, borders + [i+1], cost+1)) + next_agenda.append((self.root, borders + [i + 1], cost + 1)) curr_agenda = next_agenda answer = [] for curr, borders, cost in curr_agenda: if curr == self.root: borders = [0] + borders - answer.append([s[left:borders[i+1]] for i, left in enumerate(borders[:-1])]) + answer.append([s[left:borders[i + 1]] for i, left in enumerate(borders[:-1])]) return answer def __len__(self): @@ -275,8 +274,8 @@ def _get_letters(self, index, return_indexes=False): if self.dict_storage: answer = list(self.graph[index].keys()) else: - answer = [i for i, elem in enumerate(self.graph[index]) - if elem != Trie.NO_NODE] + answer = [i for i, elem in enumerate(self.graph[index]) + if elem != Trie.NO_NODE] if not return_indexes: answer = [(self.alphabet[i] if i >= 0 else " ") for i in answer] return answer @@ -285,8 +284,8 @@ def _get_children_and_letters(self, index, return_indexes=False): if self.dict_storage: answer = list(self.graph[index].items()) else: - answer = [elem for elem in enumerate(self.graph[index]) - if elem[1] != Trie.NO_NODE] + answer = [elem for elem in enumerate(self.graph[index]) + if elem[1] != Trie.NO_NODE] if not return_indexes: for i, (letter_index, child) in enumerate(answer): answer[i] = (self.alphabet[letter_index], child) @@ -318,7 +317,7 @@ def minimize(self, trie, dict_storage=False, make_cashed=False, make_numpied=Fal node_classes[index] = 0 class_representatives = [index] node_key = ((), (), trie.is_final(index)) - classes, class_keys = {node_key : 0}, [node_key] + classes, class_keys = {node_key: 0}, [node_key] curr_index = 1 for index in order[1:]: letter_indexes = tuple(trie._get_letters(index, return_indexes=True)) @@ -348,9 +347,9 @@ def minimize(self, trie, dict_storage=False, make_cashed=False, make_numpied=Fal new_final = np.array(new_final, dtype=bool) else: new_graph = [[Trie.NO_NODE for a in trie.alphabet] for i in range(L)] - for (indexes, children, final), class_index in\ + for (indexes, children, final), class_index in \ sorted(classes.items(), key=(lambda x: x[1])): - row = new_graph[L-class_index-1] + row = new_graph[L - class_index - 1] for i, child_index in zip(indexes, children): row[i] = L - child_index - 1 compressed.graph = new_graph @@ -386,7 +385,7 @@ def generate_postorder(self, trie): while len(stack) > 0: index = stack[-1] color = colors[index] - if color == 'white': # вершина ещё не обрабатывалась + if color == 'white': # вершина ещё не обрабатывалась colors[index] = 'grey' for child in trie._get_children(index): # проверяем, посещали ли мы ребёнка раньше @@ -403,7 +402,7 @@ def generate_postorder(self, trie): def load_trie(infile): with open(infile, "r", encoding="utf8") as fin: line = fin.readline().strip() - flags = [x=='T' for x in line.split()] + flags = [x == 'T' for x in line.split()] if len(flags) != len(Trie.ATTRS) + 1: raise ValueError("Wrong file format") nodes_number, root = map(int, fin.readline().strip().split()) @@ -413,7 +412,7 @@ def load_trie(infile): setattr(trie, attr, flags[i]) read_data = flags[-1] final = [False] * nodes_number - #print(len(alphabet), nodes_number) + # print(len(alphabet), nodes_number) if trie.dict_storage: graph = [defaultdict(lambda: -1) for _ in range(nodes_number)] elif trie.is_numpied: diff --git a/deeppavlov/models/squad/squad.py b/deeppavlov/models/squad/squad.py index 3a50eb430a..55c548cbf1 100644 --- a/deeppavlov/models/squad/squad.py +++ b/deeppavlov/models/squad/squad.py @@ -51,6 +51,7 @@ class SquadModel(LRScheduledTFModel): min_learning_rate: minimal learning rate, is used in learning rate decay noans_token: boolean, flags whether to use special no_ans token to make model able not to answer on question """ + def __init__(self, word_emb: np.ndarray, char_emb: np.ndarray, context_limit: int = 450, question_limit: int = 150, char_limit: int = 16, train_char_emb: bool = True, char_hidden_size: int = 100, encoder_hidden_size: int = 75, attention_hidden_size: int = 75, keep_prob: float = 0.7, @@ -217,8 +218,8 @@ def _init_placeholders(self): self.cc_ph = tf.placeholder(shape=(None, None, self.char_limit), dtype=tf.int32, name='cc_ph') self.q_ph = tf.placeholder(shape=(None, None), dtype=tf.int32, name='q_ph') self.qc_ph = tf.placeholder(shape=(None, None, self.char_limit), dtype=tf.int32, name='qc_ph') - self.y1_ph = tf.placeholder(shape=(None, ), dtype=tf.int32, name='y1_ph') - self.y2_ph = tf.placeholder(shape=(None, ), dtype=tf.int32, name='y2_ph') + self.y1_ph = tf.placeholder(shape=(None,), dtype=tf.int32, name='y1_ph') + self.y2_ph = tf.placeholder(shape=(None,), dtype=tf.int32, name='y2_ph') self.lear_rate_ph = tf.placeholder_with_default(0.0, shape=[], name='learning_rate') self.keep_prob_ph = tf.placeholder_with_default(1.0, shape=[], name='keep_prob_ph') diff --git a/deeppavlov/models/squad/utils.py b/deeppavlov/models/squad/utils.py index f04ab2b88a..d9ac2e4d92 100644 --- a/deeppavlov/models/squad/utils.py +++ b/deeppavlov/models/squad/utils.py @@ -48,11 +48,11 @@ def __call__(self, inputs, seq_len, keep_prob=1.0, is_train=None, concat_layers= init_fw, init_bw = self.inits[layer] mask_fw, mask_bw = self.dropout_mask[layer] with tf.variable_scope('fw_{}'.format(layer), reuse=tf.AUTO_REUSE): - out_fw, _ = gru_fw(outputs[-1] * mask_fw, (init_fw, )) + out_fw, _ = gru_fw(outputs[-1] * mask_fw, (init_fw,)) with tf.variable_scope('bw_{}'.format(layer), reuse=tf.AUTO_REUSE): inputs_bw = tf.reverse_sequence( outputs[-1] * mask_bw, seq_lengths=seq_len, seq_dim=0, batch_dim=1) - out_bw, _ = gru_bw(inputs_bw, (init_bw, )) + out_bw, _ = gru_bw(inputs_bw, (init_bw,)) out_bw = tf.reverse_sequence( out_bw, seq_lengths=seq_len, seq_dim=0, batch_dim=1) outputs.append(tf.concat([out_fw, out_bw], axis=2)) diff --git a/deeppavlov/models/tokenizers/lazy_tokenizer.py b/deeppavlov/models/tokenizers/lazy_tokenizer.py index 21af61a65f..f437bcbfb8 100644 --- a/deeppavlov/models/tokenizers/lazy_tokenizer.py +++ b/deeppavlov/models/tokenizers/lazy_tokenizer.py @@ -26,6 +26,7 @@ @register('lazy_tokenizer') class LazyTokenizer(Component): """Tokenizes if there is something to tokenize.""" + def __init__(self, **kwargs): pass diff --git a/deeppavlov/models/tokenizers/nltk_moses_tokenizer.py b/deeppavlov/models/tokenizers/nltk_moses_tokenizer.py index c42fb723c9..64c34b7fcf 100644 --- a/deeppavlov/models/tokenizers/nltk_moses_tokenizer.py +++ b/deeppavlov/models/tokenizers/nltk_moses_tokenizer.py @@ -32,7 +32,7 @@ class NLTKMosesTokenizer(Component): escape: whether escape characters for use in html markup """ - def __init__(self, escape: bool=False, *args, **kwargs): + def __init__(self, escape: bool = False, *args, **kwargs): self.escape = escape self.tokenizer = MosesTokenizer() self.detokenizer = MosesDetokenizer() diff --git a/deeppavlov/models/tokenizers/nltk_tokenizer.py b/deeppavlov/models/tokenizers/nltk_tokenizer.py index 12be4a5444..08a2072ee5 100644 --- a/deeppavlov/models/tokenizers/nltk_tokenizer.py +++ b/deeppavlov/models/tokenizers/nltk_tokenizer.py @@ -31,6 +31,7 @@ class NLTKTokenizer(Component): Attributes: tokenizer: tokenizer instance from nltk.tokenizers """ + def __init__(self, tokenizer: str = "wordpunct_tokenize", download: bool = False, *args, **kwargs): if download: diff --git a/deeppavlov/models/tokenizers/ru_sent_tokenizer.py b/deeppavlov/models/tokenizers/ru_sent_tokenizer.py index f8e38d470b..15055d5c37 100644 --- a/deeppavlov/models/tokenizers/ru_sent_tokenizer.py +++ b/deeppavlov/models/tokenizers/ru_sent_tokenizer.py @@ -34,11 +34,11 @@ class RuSentTokenizer(Component): Use default value if working on news or fiction texts """ + def __init__(self, shortenings: Set[str] = SHORTENINGS, joining_shortenings: Set[str] = JOINING_SHORTENINGS, paired_shortenings: Set[Tuple[str, str]] = PAIRED_SHORTENINGS, **kwargs): - self.shortenings = shortenings self.joining_shortenings = joining_shortenings self.paired_shortenings = paired_shortenings diff --git a/deeppavlov/models/tokenizers/ru_tokenizer.py b/deeppavlov/models/tokenizers/ru_tokenizer.py index 3017dce007..e51478a079 100644 --- a/deeppavlov/models/tokenizers/ru_tokenizer.py +++ b/deeppavlov/models/tokenizers/ru_tokenizer.py @@ -99,7 +99,7 @@ def __call__(self, batch: Union[List[str], List[List[str]]]) -> \ raise TypeError( "StreamSpacyTokenizer.__call__() is not implemented for `{}`".format(type(batch[0]))) - def _tokenize(self, data: List[str], ngram_range: Tuple[int, int]=(1, 1), lowercase: bool=True)\ + def _tokenize(self, data: List[str], ngram_range: Tuple[int, int] = (1, 1), lowercase: bool = True) \ -> Generator[List[str], Any, None]: """Tokenize a list of documents. @@ -135,7 +135,7 @@ def _tokenize(self, data: List[str], ngram_range: Tuple[int, int]=(1, 1), lowerc processed_doc = ngramize(filtered, ngram_range=_ngram_range) yield from processed_doc - def _lemmatize(self, data: List[str], ngram_range: Tuple[int, int]=(1, 1)) -> \ + def _lemmatize(self, data: List[str], ngram_range: Tuple[int, int] = (1, 1)) -> \ Generator[List[str], Any, None]: """Lemmatize a list of documents. @@ -171,7 +171,7 @@ def _lemmatize(self, data: List[str], ngram_range: Tuple[int, int]=(1, 1)) -> \ processed_doc = ngramize(filtered, ngram_range=_ngram_range) yield from processed_doc - def _filter(self, items: List[str], alphas_only: bool=True) -> List[str]: + def _filter(self, items: List[str], alphas_only: bool = True) -> List[str]: """Filter a list of tokens/lemmas. Args: @@ -205,5 +205,3 @@ def set_stopwords(self, stopwords: List[str]) -> None: """ self.stopwords = stopwords - - diff --git a/deeppavlov/models/tokenizers/spacy_tokenizer.py b/deeppavlov/models/tokenizers/spacy_tokenizer.py index 567d8204af..11521a4f4f 100644 --- a/deeppavlov/models/tokenizers/spacy_tokenizer.py +++ b/deeppavlov/models/tokenizers/spacy_tokenizer.py @@ -178,7 +178,7 @@ def _lemmatize(self, data: List[str], ngram_range: Optional[Tuple[int, int]] = N processed_doc = ngramize(filtered, ngram_range=_ngram_range) yield from processed_doc - def _filter(self, items: List[str], alphas_only: bool=True) -> List[str]: + def _filter(self, items: List[str], alphas_only: bool = True) -> List[str]: """Filter a list of tokens/lemmas. Args: diff --git a/deeppavlov/models/tokenizers/split_tokenizer.py b/deeppavlov/models/tokenizers/split_tokenizer.py index 4e543784ab..79a15a677a 100644 --- a/deeppavlov/models/tokenizers/split_tokenizer.py +++ b/deeppavlov/models/tokenizers/split_tokenizer.py @@ -25,6 +25,7 @@ class SplitTokenizer(Component): Doesn't have any parameters. """ + def __init__(self, **kwargs) -> None: pass diff --git a/deeppavlov/models/vectorizers/word_vectorizer.py b/deeppavlov/models/vectorizers/word_vectorizer.py index 7d04ee8b92..7f93c94556 100644 --- a/deeppavlov/models/vectorizers/word_vectorizer.py +++ b/deeppavlov/models/vectorizers/word_vectorizer.py @@ -79,6 +79,7 @@ class DictionaryVectorizer(WordIndexVectorizer): min_freq: minimal frequency of tag to memorize this tag, unk_token: unknown token to be yielded for unknown words """ + def __init__(self, save_path: str, load_path: Union[str, List[str]], min_freq: int = 1, unk_token: str = None, **kwargs) -> None: super().__init__(save_path, load_path, **kwargs) @@ -116,7 +117,7 @@ def load(self) -> None: labels_by_words[word].update(labels.split()) self._initialize(labels_by_words) - def _initialize(self, labels_by_words : Dict): + def _initialize(self, labels_by_words: Dict): self._i2t = [self.unk_token] if self.unk_token is not None else [] self._t2i = defaultdict(lambda: self.unk_token) freq = defaultdict(int) @@ -286,4 +287,3 @@ def _get_tag_indexes(self, pymorphy_tag): tag = self.converter(str(pymorphy_tag)) answer = self.memorized_tag_indexes[pymorphy_tag] = self.find_compatible(tag) return answer - diff --git a/deeppavlov/skills/aiml_skill/aiml_skill.py b/deeppavlov/skills/aiml_skill/aiml_skill.py index 384730ddff..72c69113ca 100644 --- a/deeppavlov/skills/aiml_skill/aiml_skill.py +++ b/deeppavlov/skills/aiml_skill/aiml_skill.py @@ -16,6 +16,8 @@ from pathlib import Path from typing import Tuple, Optional, List from logging import getLogger +from pathlib import Path +from typing import Tuple, Optional, List import aiml diff --git a/deeppavlov/skills/dsl_skill/context.py b/deeppavlov/skills/dsl_skill/context.py index b1dc1b3cf4..acbfc6c5b9 100644 --- a/deeppavlov/skills/dsl_skill/context.py +++ b/deeppavlov/skills/dsl_skill/context.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Union, Dict - import json +from typing import Optional, Union, Dict from deeppavlov.skills.dsl_skill.utils import UserId diff --git a/deeppavlov/utils/pip_wrapper/__init__.py b/deeppavlov/utils/pip_wrapper/__init__.py index 24cb413c4d..e2d482331e 100644 --- a/deeppavlov/utils/pip_wrapper/__init__.py +++ b/deeppavlov/utils/pip_wrapper/__init__.py @@ -1 +1 @@ -from .pip_wrapper import * \ No newline at end of file +from .pip_wrapper import * diff --git a/deeppavlov/utils/pip_wrapper/pip_wrapper.py b/deeppavlov/utils/pip_wrapper/pip_wrapper.py index 175abfdc46..a829af0cc6 100644 --- a/deeppavlov/utils/pip_wrapper/pip_wrapper.py +++ b/deeppavlov/utils/pip_wrapper/pip_wrapper.py @@ -13,13 +13,13 @@ def install(*packages): - if any(_tf_re.match(package) for package in packages)\ + if any(_tf_re.match(package) for package in packages) \ and b'tensorflow-gpu' in subprocess.check_output([sys.executable, '-m', 'pip', 'freeze'], env=os.environ.copy()): log.warn('found tensorflow-gpu installed, so upgrading it instead of tensorflow') packages = [_tf_re.sub(r'tensorflow-gpu\1', package) for package in packages] result = subprocess.check_call([sys.executable, '-m', 'pip', 'install', - *[re.sub(r'\s', '', package) for package in packages]], + *[re.sub(r'\s', '', package) for package in packages]], env=os.environ.copy()) return result diff --git a/deeppavlov/utils/server/server.py b/deeppavlov/utils/server/server.py index fb8e12333f..c47710c0b2 100644 --- a/deeppavlov/utils/server/server.py +++ b/deeppavlov/utils/server/server.py @@ -25,8 +25,8 @@ from pydantic import BaseConfig, BaseModel, Schema from pydantic.fields import Field from pydantic.main import MetaModel -from starlette.responses import RedirectResponse from starlette.middleware.cors import CORSMiddleware +from starlette.responses import RedirectResponse from deeppavlov.core.commands.infer import build_model from deeppavlov.core.commands.utils import parse_config @@ -42,6 +42,7 @@ class ProbeFilter(logging.Filter): """ProbeFilter class is used to filter POST requests to /probe endpoint from logs.""" + def filter(self, record: logging.LogRecord) -> bool: """To log the record method should return True.""" return 'POST /probe HTTP' not in record.getMessage() @@ -116,6 +117,7 @@ def get_ssl_params(server_params: dict, def redirect_root_to_docs(fast_app: FastAPI, func_name: str, endpoint: str, method: str) -> None: """Adds api route to server that redirects user from root to docs with opened `endpoint` description.""" + @fast_app.get('/', include_in_schema=False) async def redirect_to_docs() -> RedirectResponse: operation_id = generate_operation_id_for_path(name=func_name, path=endpoint, method=method) @@ -192,6 +194,7 @@ class Batch(BaseModel): redirect_root_to_docs(app, 'answer', model_endpoint, 'post') model_endpoint_post_example = {arg_name: ['string'] for arg_name in model_args_names} + @app.post(model_endpoint, summary='A model endpoint') async def answer(item: Batch = Body(..., example=model_endpoint_post_example)) -> List: loop = asyncio.get_event_loop() diff --git a/deeppavlov/utils/settings/log_config.json b/deeppavlov/utils/settings/log_config.json index 0f25e18038..0b5ed38b0e 100644 --- a/deeppavlov/utils/settings/log_config.json +++ b/deeppavlov/utils/settings/log_config.json @@ -23,7 +23,7 @@ "datefmt": "%Y-%m-%d %H:%M:%S" }, "uvicorn_fmt": { - "format":"%(asctime)s %(message)s", + "format": "%(asctime)s %(message)s", "datefmt": "%Y-%m-%d %H:%M:%S" } }, diff --git a/deeppavlov/utils/settings/socket_config.json b/deeppavlov/utils/settings/socket_config.json index 53c6b638dd..17d32dde47 100644 --- a/deeppavlov/utils/settings/socket_config.json +++ b/deeppavlov/utils/settings/socket_config.json @@ -1,5 +1,5 @@ { - "common_defaults":{ + "common_defaults": { "host": "0.0.0.0", "port": 5001, "unix_socket_file": "/tmp/deeppavlov_socket.s", diff --git a/deeppavlov/vocabs/typos.py b/deeppavlov/vocabs/typos.py index a1fb1b98b0..c6266f655f 100644 --- a/deeppavlov/vocabs/typos.py +++ b/deeppavlov/vocabs/typos.py @@ -45,7 +45,7 @@ class StaticDictionary: words_trie: trie structure of all the words """ - def __init__(self, data_dir: [Path, str]='', *args, dictionary_name: str='dictionary', **kwargs): + def __init__(self, data_dir: [Path, str] = '', *args, dictionary_name: str = 'dictionary', **kwargs): data_dir = expand_path(data_dir) / dictionary_name alphabet_path = data_dir / 'alphabet.pkl' @@ -71,7 +71,7 @@ def __init__(self, data_dir: [Path, str]='', *args, dictionary_name: str='dictio words_trie = defaultdict(set) for word in words: for i in range(len(word)): - words_trie[word[:i]].add(word[:i+1]) + words_trie[word[:i]].add(word[:i + 1]) words_trie[word] = set() words_trie = {k: sorted(v) for k, v in words_trie.items()} @@ -113,7 +113,7 @@ class RussianWordsVocab(StaticDictionary): words_trie: trie structure of all the words """ - def __init__(self, data_dir: [Path, str]='', *args, **kwargs): + def __init__(self, data_dir: [Path, str] = '', *args, **kwargs): kwargs['dictionary_name'] = 'russian_words_vocab' super().__init__(data_dir, *args, **kwargs) @@ -140,7 +140,8 @@ class Wiki100KDictionary(StaticDictionary): words_set: set of all the words words_trie: trie structure of all the words """ - def __init__(self, data_dir: [Path, str]='', *args, **kwargs): + + def __init__(self, data_dir: [Path, str] = '', *args, **kwargs): kwargs['dictionary_name'] = 'wikipedia_100K_vocab' super().__init__(data_dir, *args, **kwargs) diff --git a/setup.py b/setup.py index d7a13015d8..2f19c3de5b 100644 --- a/setup.py +++ b/setup.py @@ -58,15 +58,15 @@ def readme(): keywords=deeppavlov.__keywords__, include_package_data=True, extras_require={ - 'tests': [ - 'flake8', - 'pytest', - 'pexpect'], - 'docs': [ - 'sphinx>=1.7.9', - 'sphinx_rtd_theme>=0.4.0', - 'nbsphinx>=0.3.4', - 'ipykernel>=4.8.0' - ]}, + 'tests': [ + 'flake8', + 'pytest', + 'pexpect'], + 'docs': [ + 'sphinx>=1.7.9', + 'sphinx_rtd_theme>=0.4.0', + 'nbsphinx>=0.3.4', + 'ipykernel>=4.8.0' + ]}, **read_requirements() ) diff --git a/tests/test_quick_start.py b/tests/test_quick_start.py index 59cc944b39..2b131b6b57 100644 --- a/tests/test_quick_start.py +++ b/tests/test_quick_start.py @@ -214,20 +214,20 @@ }, "seq2seq_go_bot": { ("seq2seq_go_bot/bot_kvret_train.json", "seq2seq_go_bot", ('TI',)): - [ - ("will it snow on tuesday?", - "f78cf0f9-7d1e-47e9-aa45-33f9942c94be", - "", - "", - "", - None) - ], + [ + ("will it snow on tuesday?", + "f78cf0f9-7d1e-47e9-aa45-33f9942c94be", + "", + "", + "", + None) + ], ("seq2seq_go_bot/bot_kvret.json", "seq2seq_go_bot", ('IP',)): - [ - ("will it snow on tuesday?", - "f78cf0f9-7d1e-47e9-aa45-33f9942c94be", - None) - ] + [ + ("will it snow on tuesday?", + "f78cf0f9-7d1e-47e9-aa45-33f9942c94be", + None) + ] }, "odqa": { ("odqa/en_odqa_infer_wiki_test.json", "odqa", ('IP',)): [ONE_ARGUMENT_INFER_CHECK], @@ -373,7 +373,7 @@ def interact(config_path, model_directory, qr_list=None): p.expect(">> ") if expected_response is not None: actual_response = p.readline().decode().strip() - assert expected_response == actual_response,\ + assert expected_response == actual_response, \ f"Error in interacting with {model_directory} ({config_path}): {query}" p.expect("::") @@ -474,7 +474,7 @@ def interact_socket(config_path, socket_type): resp = json.loads(data) except json.decoder.JSONDecodeError: raise ValueError(f"Can't decode model response {data}") - assert resp['status'] == 'OK', f"{socket_type} socket request returned status: {resp['status']}"\ + assert resp['status'] == 'OK', f"{socket_type} socket request returned status: {resp['status']}" \ f" with {config_path}\n{logfile.getvalue().decode()}" except pexpect.exceptions.EOF: @@ -544,7 +544,7 @@ def test_consecutive_training_and_interacting(self, model, conf_file, model_dir, config_path = str(test_configs_path.joinpath(conf_file)) install_config(config_path) deep_download(config_path) - shutil.rmtree(str(model_path), ignore_errors=True) + shutil.rmtree(str(model_path), ignore_errors=True) logfile = io.BytesIO(b'') p = pexpect.popen_spawn.PopenSpawn(sys.executable + " -m deeppavlov train " + str(c), timeout=None, @@ -571,7 +571,7 @@ def test_crossvalidation(): install_config(c) deep_download(c) - shutil.rmtree(str(model_path), ignore_errors=True) + shutil.rmtree(str(model_path), ignore_errors=True) logfile = io.BytesIO(b'') p = pexpect.popen_spawn.PopenSpawn(sys.executable + f" -m deeppavlov crossval {c} --folds 2", @@ -596,7 +596,7 @@ def test_param_search(): install_config(c) deep_download(c) - shutil.rmtree(str(model_path), ignore_errors=True) + shutil.rmtree(str(model_path), ignore_errors=True) logfile = io.BytesIO(b'') p = pexpect.popen_spawn.PopenSpawn(sys.executable + f" -m deeppavlov.paramsearch {c} --folds 2", diff --git a/tests/test_tf_layers.py b/tests/test_tf_layers.py index e67ff61c8d..7ce26e31ca 100644 --- a/tests/test_tf_layers.py +++ b/tests/test_tf_layers.py @@ -146,7 +146,6 @@ def load(self, path): class TestTFLayers: - allowed_error_lvl = 0.01 * 2 ** 0.5 @staticmethod diff --git a/utils/prepare/hashes.py b/utils/prepare/hashes.py index dbcf149fda..e021beb6a3 100644 --- a/utils/prepare/hashes.py +++ b/utils/prepare/hashes.py @@ -24,7 +24,7 @@ from deeppavlov.core.data.utils import file_md5 -def tar_md5(fpath: Union[str, Path], chunk_size: int = 2**16) -> Dict[str, str]: +def tar_md5(fpath: Union[str, Path], chunk_size: int = 2 ** 16) -> Dict[str, str]: tar = tarfile.open(fpath) res = {} while True: @@ -41,7 +41,7 @@ def tar_md5(fpath: Union[str, Path], chunk_size: int = 2**16) -> Dict[str, str]: return res -def gzip_md5(fpath: Union[str, Path], chunk_size: int = 2**16) -> str: +def gzip_md5(fpath: Union[str, Path], chunk_size: int = 2 ** 16) -> str: file_hash = md5() with gzip.open(fpath, 'rb') as f: for chunk in iter(lambda: f.read(chunk_size), b""): @@ -49,7 +49,7 @@ def gzip_md5(fpath: Union[str, Path], chunk_size: int = 2**16) -> str: return file_hash.hexdigest() -def zip_md5(fpath: Union[str, Path], chunk_size: int = 2**16) -> Dict[str, str]: +def zip_md5(fpath: Union[str, Path], chunk_size: int = 2 ** 16) -> Dict[str, str]: res = {} with ZipFile(fpath) as zip_f: for item in zip_f.infolist(): diff --git a/utils/prepare/registry.py b/utils/prepare/registry.py index ee1f8eb8d6..2571ede8d3 100644 --- a/utils/prepare/registry.py +++ b/utils/prepare/registry.py @@ -24,7 +24,7 @@ C_REGISTRY.clear() M_REGISTRY.clear() - for _, pkg_name, _ in pkgutil.walk_packages(deeppavlov.__path__, deeppavlov.__name__+'.'): + for _, pkg_name, _ in pkgutil.walk_packages(deeppavlov.__path__, deeppavlov.__name__ + '.'): if pkg_name not in ('deeppavlov.core.common.registry', 'deeppavlov.core.common.metrics_registry'): reload(import_module(pkg_name)) From 7347faf9963dd2cc603f20d0648edebdab8fd094 Mon Sep 17 00:00:00 2001 From: Fedor Ignatov Date: Thu, 31 Oct 2019 18:11:22 +0300 Subject: [PATCH 18/28] fix: allow multiple messages in risesocket (#1056) * fix: #1054 * refactor: socket server * fix: socket tests * docs: updated breaking changes * fix: test socket response correct handling * feat: test socket connection delaty to prevent error due to socket lag * fix: not imported time.sleep in tests --- README.md | 1 + deeppavlov/utils/settings/socket_config.json | 3 +- deeppavlov/utils/socket/__init__.py | 2 +- deeppavlov/utils/socket/socket.py | 220 +++++++++++-------- docs/integrations/socket_api.rst | 13 +- tests/test_quick_start.py | 35 +-- 6 files changed, 159 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index cf4ca3c8b4..55527a2e84 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,7 @@ and others in the Integrations section for more info. [Microsoft Bot Framework](http://docs.deeppavlov.ai/en/0.7.0/integrations/ms_bot.html) and [Telegram integration](http://docs.deeppavlov.ai/en/0.7.0/integrations/telegram.html) interfaces were changed - `/start` and `/help` Telegram messages were moved from `models_info.json` to [connector_config.json](/deeppavlov/utils/settings/connector_config.json) +- [risesocket](http://docs.deeppavlov.ai/en/0.7.0/integrations/socket_api.html) request and response format was changed **Breaking changes in version 0.6.0** - [REST API](http://docs.deeppavlov.ai/en/0.6.0/integrations/rest_api.html): diff --git a/deeppavlov/utils/settings/socket_config.json b/deeppavlov/utils/settings/socket_config.json index 17d32dde47..67d45cfccc 100644 --- a/deeppavlov/utils/settings/socket_config.json +++ b/deeppavlov/utils/settings/socket_config.json @@ -5,8 +5,7 @@ "unix_socket_file": "/tmp/deeppavlov_socket.s", "socket_type": "TCP", "model_args_names": "", - "bufsize": 1024, - "binding_message": "binding socket to" + "launch_message": "launching socket server at" }, "model_defaults": { "SquadModel": {} diff --git a/deeppavlov/utils/socket/__init__.py b/deeppavlov/utils/socket/__init__.py index 843dfda563..f222759a23 100644 --- a/deeppavlov/utils/socket/__init__.py +++ b/deeppavlov/utils/socket/__init__.py @@ -1 +1 @@ -from .socket import start_socket_server, SOCKET_CONFIG_FILENAME +from .socket import encode, start_socket_server, SOCKET_CONFIG_FILENAME diff --git a/deeppavlov/utils/socket/socket.py b/deeppavlov/utils/socket/socket.py index 55c66a8020..61dc0298c9 100644 --- a/deeppavlov/utils/socket/socket.py +++ b/deeppavlov/utils/socket/socket.py @@ -14,10 +14,10 @@ import asyncio import json -import socket from logging import getLogger from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union +from struct import pack, unpack +from typing import Any, List, Optional, Tuple, Union from deeppavlov.core.commands.infer import build_model from deeppavlov.core.common.chainer import Chainer @@ -27,129 +27,171 @@ from deeppavlov.utils.server import get_server_params SOCKET_CONFIG_FILENAME = 'socket_config.json' +HEADER_FORMAT = ' bytes: + """Сonverts data to the socket server input formatted bytes array. + + Serializes ``data`` to the JSON formatted bytes array and adds 4 bytes to the beginning of the array - packed + to bytes length of the JSON formatted bytes array. Header format is ">> from deeppavlov.utils.socket import encode + >>> encode({'a':1}) + b'\x08\x00\x00\x00{"a": 1} + >>> encode([42]) + b'\x04\x00\x00\x00[42]' + + """ + json_data = jsonify_data(data) + bytes_data = json.dumps(json_data).encode() + response = pack(HEADER_FORMAT, len(bytes_data)) + bytes_data + return response + + class SocketServer: """Creates socket server that sends the received data to the DeepPavlov model and returns model response. - The server receives dictionary serialized to JSON formatted bytes array and sends it to the model. The dictionary + The server receives bytes array consists of the `header` and the `body`. The `header` is the first 4 bytes + of the array - `body` length in bytes represented by a packed unsigned int (byte order is little-endian). + `body` is dictionary serialized to JSON formatted bytes array that server sends to the model. The dictionary keys should match model arguments names, the values should be lists or tuples of inferenced values. - Example: - {“context”:[“Elon Musk launched his cherry Tesla roadster to the Mars orbit”]} + Socket server request creation example: + >>> from deeppavlov.utils.socket import encode + >>> request = encode({"context":["Elon Musk launched his cherry Tesla roadster to the Mars orbit"]}) + >>> request + b'I\x00\x00\x00{"x": ["Elon Musk launched his cherry Tesla roadster to the Mars orbit"]}' - Socket server returns dictionary {'status': status, 'payload': payload} serialized to a JSON formatted byte array, - where: + Socket server response, like the request, consists of the header and the body. Response body is dictionary + {'status': status, 'payload': payload} serialized to a JSON formatted byte array, where: status (str): 'OK' if the model successfully processed the data, else - error message. - payload: (Optional[List[Tuple]]): The model result if no error has occurred, otherwise None + payload: (Optional[List[Tuple]]): The model result if no error has occurred, otherwise None. """ - _address_family: socket.AddressFamily - _bind_address: Union[Tuple[str, int], str] _launch_msg: str _loop: asyncio.AbstractEventLoop _model: Chainer - _params: Dict - _socket: socket.socket - _socket_type: str + _model_args_names: List - def __init__(self, model_config: Path, socket_type: str, port: Optional[int] = None, + def __init__(self, + model_config: Path, + socket_type: str, + port: Optional[int] = None, socket_file: Optional[Union[str, Path]] = None) -> None: - """Initialize socket server. + """Initializes socket server. Args: model_config: Path to the config file. - socket_type: Socket family. "TCP" for the AF_INET socket, "UNIX" for the AF_UNIX. + socket_type: Socket family. "TCP" for the AF_INET socket server, "UNIX" for UNIX Domain Socket server. port: Port number for the AF_INET address family. If parameter is not defined, the port number from the - model_config is used. - socket_file: Path to the file to which server of the AF_UNIX address family connects. If parameter - is not defined, the path from the model_config is used. + utils/settings/socket_config.json is used. + socket_file: Path to the file to which UNIX Domain Socket server connects. If parameter is not defined, + the path from the utils/settings/socket_config.json is used. + + Raises: + ValueError: If ``socket_type`` parameter is neither "TCP" nor "UNIX". """ socket_config_path = get_settings_path() / SOCKET_CONFIG_FILENAME - self._params = get_server_params(model_config, socket_config_path) - self._socket_type = socket_type or self._params['socket_type'] - - if self._socket_type == 'TCP': - host = self._params['host'] - port = port or self._params['port'] - self._address_family = socket.AF_INET - self._launch_msg = f'{self._params["binding_message"]} http://{host}:{port}' - self._bind_address = (host, port) - elif self._socket_type == 'UNIX': - self._address_family = socket.AF_UNIX - bind_address = socket_file or self._params['unix_socket_file'] - bind_address = Path(bind_address).resolve() - if bind_address.exists(): - bind_address.unlink() - self._bind_address = str(bind_address) - self._launch_msg = f'{self._params["binding_message"]} {self._bind_address}' + server_params = get_server_params(model_config, socket_config_path) + socket_type = socket_type or server_params['socket_type'] + self._loop = asyncio.get_event_loop() + + if socket_type == 'TCP': + host = server_params['host'] + port = port or server_params['port'] + self._launch_msg = f'{server_params["launch_message"]} http://{host}:{port}' + self._loop.create_task(asyncio.start_server(self._handle_client, host, port)) + elif socket_type == 'UNIX': + socket_file = socket_file or server_params['unix_socket_file'] + socket_path = Path(socket_file).resolve() + if socket_path.exists(): + socket_path.unlink() + self._launch_msg = f'{server_params["launch_message"]} {socket_file}' + self._loop.create_task(asyncio.start_unix_server(self._handle_client, socket_file)) else: - raise ValueError(f'socket type "{self._socket_type}" is not supported') + raise ValueError(f'socket type "{socket_type}" is not supported') - self._log = getLogger(__name__) - self._loop = asyncio.get_event_loop() self._model = build_model(model_config) - self._socket = socket.socket(self._address_family, socket.SOCK_STREAM) - - self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self._socket.setblocking(False) + self._model_args_names = server_params['model_args_names'] def start(self) -> None: - """Binds the socket to the address and enables the server to accept connections""" - self._socket.bind(self._bind_address) - self._socket.listen() - self._log.info(self._launch_msg) + """Launches socket server""" + log.info(self._launch_msg) try: - self._loop.run_until_complete(self._server()) + self._loop.run_forever() + except KeyboardInterrupt: + pass except Exception as e: - self._log.error(f'got exception {e} while running server') + log.error(f'got exception {e} while running server') finally: self._loop.close() - self._socket.close() - async def _server(self) -> None: - while True: - conn, addr = await self._loop.sock_accept(self._socket) - self._loop.create_task(self._handle_connection(conn, addr)) + async def _handle_client(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """Handles connection from a client. - async def _handle_connection(self, conn: socket.socket, addr: Tuple) -> None: - self._log.info(f'handling connection from {addr}') - conn.setblocking(False) - recv_data = b'' - try: - while True: - chunk = await self._loop.run_in_executor(None, conn.recv, self._params['bufsize']) - if chunk: - recv_data += chunk - else: - break - except BlockingIOError: - pass - try: - data = json.loads(recv_data) - except ValueError: - await self._wrap_error(conn, f'request "{recv_data}" type is not json') - return + Validates requests, sends request body to DeepPavlov model, sends responses to client. + + """ + addr = writer.get_extra_info('peername') + log.info(f'handling connection from {addr}') + while True: + header = await reader.read(4) + if not header: + log.info(f'closing connection from {addr}') + writer.close() + break + elif len(header) != 4: + error_msg = f'header "{header}" length less than 4 bytes' + log.error(error_msg) + response = self._response(error_msg) + else: + data_len = unpack(HEADER_FORMAT, header)[0] + request_body = await reader.read(data_len) + try: + data = json.loads(request_body) + response = await self._interact(data) + except ValueError: + error_msg = f'request "{request_body}" type is not json' + log.error(error_msg) + response = self._response(error_msg) + writer.write(response) + await writer.drain() + + async def _interact(self, data: dict) -> bytes: dialog_logger.log_in(data) model_args = [] - for param_name in self._params['model_args_names']: + for param_name in self._model_args_names: param_value = data.get(param_name) if param_value is None or (isinstance(param_value, list) and len(param_value) > 0): model_args.append(param_value) else: - await self._wrap_error(conn, f"nonempty array expected but got '{param_name}'={repr(param_value)}") - return + error_msg = f"nonempty array expected but got '{param_name}'={repr(param_value)}" + log.error(error_msg) + return self._response(error_msg) lengths = {len(i) for i in model_args if i is not None} if not lengths: - await self._wrap_error(conn, 'got empty request') - return + error_msg = 'got empty request' + log.error(error_msg) + return self._response(error_msg) elif len(lengths) > 1: - await self._wrap_error(conn, f'got several different batch sizes: {lengths}') - return + error_msg = f'got several different batch sizes: {lengths}' + log.error(error_msg) + return self._response(error_msg) + batch_size = list(lengths)[0] model_args = [arg or [None] * batch_size for arg in model_args] @@ -160,29 +202,23 @@ async def _handle_connection(self, conn: socket.socket, addr: Tuple) -> None: if len(self._model.out_params) == 1: prediction = [prediction] prediction = list(zip(*prediction)) - result = await self._response('OK', prediction) - dialog_logger.log_out(result) - await self._loop.sock_sendall(conn, result) - - async def _wrap_error(self, conn: socket.socket, error: str) -> None: - self._log.error(error) - await self._loop.sock_sendall(conn, await self._response(error, None)) + dialog_logger.log_out(prediction) + return self._response(payload=prediction) @staticmethod - async def _response(status: str, payload: Optional[List[Tuple]]) -> bytes: - """Puts arguments into dict and serialize it to JSON formatted byte array. + def _response(status: str = 'OK', payload: Optional[List[Tuple]] = None) -> bytes: + """Puts arguments into dict and serialize it to JSON formatted byte array with header. Args: status: Response status. 'OK' if no error has occurred, otherwise error message. payload: DeepPavlov model result if no error has occurred, otherwise None. Returns: - dict({'status': status, 'payload': payload}) serialized to a JSON formatted byte array. + dict({'status': status, 'payload': payload}) serialized to a JSON formatted byte array starting with the + 4-byte header - the length of serialized dict in bytes. """ - resp_dict = jsonify_data({'status': status, 'payload': payload}) - resp_str = json.dumps(resp_dict) - return resp_str.encode('utf-8') + return encode({'status': status, 'payload': payload}) def start_socket_server(model_config: Path, socket_type: str, port: Optional[int], diff --git a/docs/integrations/socket_api.rst b/docs/integrations/socket_api.rst index c4526d94e1..1c4f8dcc63 100644 --- a/docs/integrations/socket_api.rst +++ b/docs/integrations/socket_api.rst @@ -90,6 +90,9 @@ two elements: import json import socket + from struct import unpack + + from deeppavlov.utils.socket import encode socket_payload = { "context_raw": [ @@ -101,13 +104,15 @@ two elements: "Who I used to be?" ] } - dumped_socket_payload = json.dumps(socket_payload) + serialized_socket_payload = encode(socket_payload) with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect(('0.0.0.0', 5000)) - s.sendall(dumped_socket_payload.encode('utf-8')) - serialized_payload = s.recv(1024) - json_payload = json.loads(serialized_payload) + s.sendall(serialized_socket_payload) + header = s.recv(4) + body_len = unpack(' Date: Tue, 5 Nov 2019 11:08:41 +0300 Subject: [PATCH 19/28] fix: allow special characters in sqlite requests values (#1060) * fix: allow special characters in sqlite requests values * fix: join where fields with `AND` instead of `,` --- deeppavlov/core/data/sqlite_database.py | 29 ++++++++++++++----------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/deeppavlov/core/data/sqlite_database.py b/deeppavlov/core/data/sqlite_database.py index 4d575c69b2..144cda2fbe 100644 --- a/deeppavlov/core/data/sqlite_database.py +++ b/deeppavlov/core/data/sqlite_database.py @@ -74,7 +74,7 @@ def __call__(self, batch: List[Dict], ascending: bool = False) -> List[List[Dict]]: order = 'ASC' if ascending else 'DESC' if not self._check_if_table_exists(): - log.warn("Database is empty, call fit() before using.") + log.warning("Database is empty, call fit() before using.") return [[] for i in range(len(batch))] return [self._search(b, order_by=order_by, order=order) for b in batch] @@ -156,26 +156,29 @@ def _insert_many(self, data): self.conn.commit() def _get_record(self, primary_values): - ffields = ', '.join(self.keys) or '*' - where_expr = " AND ".join(f"{pk} = '{v}'" - for pk, v in zip(self.primary_keys, - primary_values)) + ffields = ", ".join(self.keys) or "*" + where_expr = " AND ".join(f"{pk}=?" for pk in self.primary_keys) fetched = self.cursor.execute(f"SELECT {ffields} FROM {self.tname}" + - f" WHERE {where_expr}").fetchone() + f" WHERE {where_expr}", primary_values).fetchone() if not fetched: return None return fetched def _update_one(self, record): - set_expr = ', '.join(f"{k} = '{v}'" - for k, v in zip(self.keys, record) - if k not in self.primary_keys) - where_expr = " AND ".join(f"{k} = '{v}'" - for k, v in zip(self.keys, record) - if k in self.primary_keys) + set_values, where_values = [], [] + set_fields, where_fields = [], [] + for k, v in zip(self.keys, record): + if k in self.primary_keys: + where_fields.append(f"{k}=?") + where_values.append(v) + else: + set_fields.append(f"{k}=?") + set_values.append(v) + set_expr = ", ".join(set_fields) + where_expr = " AND ".join(where_fields) self.cursor.execute(f"UPDATE {self.tname}" + f" SET {set_expr}" + - f" WHERE {where_expr}") + f" WHERE {where_expr}", set_values+where_values) def save(self): pass From c11f40a5161e635a3c8cd879a4ae477a809a8492 Mon Sep 17 00:00:00 2001 From: "M.Cemil Guney" Date: Tue, 5 Nov 2019 12:17:38 +0300 Subject: [PATCH 20/28] docs: fix colab link. (#1061) --- examples/gobot_extended_tutorial.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gobot_extended_tutorial.ipynb b/examples/gobot_extended_tutorial.ipynb index d2ae991dae..5c31af8174 100644 --- a/examples/gobot_extended_tutorial.ipynb +++ b/examples/gobot_extended_tutorial.ipynb @@ -7,7 +7,7 @@ "id": "K7nBJnADTgUw" }, "source": [ - "### You can also run the notebook in [COLAB](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/gobot_tutorial.ipynb)." + "### You can also run the notebook in [COLAB](https://colab.research.google.com/github/deepmipt/DeepPavlov/blob/master/examples/gobot_extended_tutorial.ipynb)." ] }, { From 3b01b895c8922664cc26f9ca2ac5754209b7037f Mon Sep 17 00:00:00 2001 From: Fedor Ignatov Date: Fri, 8 Nov 2019 11:52:31 +0300 Subject: [PATCH 21/28] feat: remove sever_utils from configs (#1063) * feat: server config and socket config are joined * refactor: license information * feat: connector_config moved to server_settings * docs: changed docs according config changes + small typing changes * fix: docs build * fix: some warnings * refactor: removed unnecessary import in server init * Update deeppavlov/utils/telegram/telegram_ui.py Co-Authored-By: Aleksei Lymar * refactor: get_server_params metadata --- README.md | 11 ++- .../configs/classifiers/insults_kaggle.json | 4 - .../classifiers/insults_kaggle_bert.json | 4 - .../classifiers/insults_kaggle_conv_bert.json | 4 - .../configs/classifiers/intents_dstc2.json | 4 - .../classifiers/intents_dstc2_bert.json | 4 - .../classifiers/intents_dstc2_big.json | 3 - .../classifiers/intents_sample_csv.json | 4 - .../classifiers/intents_sample_json.json | 4 - .../configs/classifiers/intents_snips.json | 4 - .../classifiers/intents_snips_big.json | 4 - .../classifiers/intents_snips_sklearn.json | 4 - .../intents_snips_tfidf_weighted.json | 4 - .../configs/classifiers/rusentiment_bert.json | 4 - .../rusentiment_bigru_superconv.json | 4 - .../configs/classifiers/rusentiment_cnn.json | 4 - .../classifiers/rusentiment_convers_bert.json | 4 - .../rusentiment_elmo_twitter_cnn.json | 4 - .../classifiers/sentiment_sst_conv_bert.json | 4 - .../classifiers/sentiment_sst_multi_bert.json | 4 - .../classifiers/sentiment_twitter.json | 4 - .../sentiment_twitter_preproc.json | 4 - .../classifiers/sentiment_yelp_conv_bert.json | 4 - .../sentiment_yelp_multi_bert.json | 4 - .../configs/classifiers/topic_ag_news.json | 4 - .../classifiers/yahoo_convers_vs_info.json | 4 - .../yahoo_convers_vs_info_bert.json | 4 - .../en_ranker_pop_enwiki20180211.json | 3 - .../en_ranker_tfidf_enwiki20161221.json | 3 - .../doc_retrieval/en_ranker_tfidf_wiki.json | 3 - .../doc_retrieval/ru_ranker_tfidf_wiki.json | 3 - .../ecommerce_skill/bleu_retrieve.json | 4 - .../ecommerce_skill/tfidf_retrieve.json | 4 - .../evolution/evolve_intents_snips.json | 4 - .../evolution/evolve_rusentiment_cnn.json | 4 - deeppavlov/configs/go_bot/gobot_dstc2.json | 4 - .../configs/go_bot/gobot_dstc2_best.json | 4 - .../configs/go_bot/gobot_dstc2_minimal.json | 4 - .../configs/go_bot/gobot_simple_dstc2.json | 4 - .../morpho_tagger/UD2.0/morpho_ar.json | 5 +- .../morpho_tagger/UD2.0/morpho_cs.json | 5 +- .../morpho_tagger/UD2.0/morpho_de.json | 5 +- .../morpho_tagger/UD2.0/morpho_en.json | 5 +- .../morpho_tagger/UD2.0/morpho_es_ancora.json | 5 +- .../morpho_tagger/UD2.0/morpho_fr.json | 5 +- .../morpho_tagger/UD2.0/morpho_hi.json | 5 +- .../morpho_tagger/UD2.0/morpho_hu.json | 5 +- .../morpho_tagger/UD2.0/morpho_it.json | 5 +- .../UD2.0/morpho_ru_syntagrus.json | 5 +- .../UD2.0/morpho_ru_syntagrus_pymorphy.json | 5 +- ...orpho_ru_syntagrus_pymorphy_lemmatize.json | 5 +- .../morpho_tagger/UD2.0/morpho_tr.json | 5 +- deeppavlov/configs/ner/ner_conll2003.json | 4 - .../configs/ner/ner_conll2003_bert.json | 4 - deeppavlov/configs/ner/ner_conll2003_pos.json | 4 - deeppavlov/configs/ner/ner_dstc2.json | 4 - deeppavlov/configs/ner/ner_few_shot_ru.json | 4 - .../configs/ner/ner_few_shot_ru_simulate.json | 4 - deeppavlov/configs/ner/ner_kb_rus.json | 4 - deeppavlov/configs/ner/ner_ontonotes.json | 4 - .../configs/ner/ner_ontonotes_bert.json | 4 - .../configs/ner/ner_ontonotes_bert_mult.json | 4 - deeppavlov/configs/ner/ner_rus.json | 4 - deeppavlov/configs/ner/ner_rus_bert.json | 4 - deeppavlov/configs/ner/slotfill_dstc2.json | 4 - .../configs/ner/slotfill_dstc2_raw.json | 3 - .../ner/slotfill_simple_dstc2_raw.json | 3 - .../odqa/en_odqa_infer_enwiki20161221.json | 3 - .../configs/odqa/en_odqa_infer_wiki.json | 3 - .../en_odqa_pop_infer_enwiki20180211.json | 3 - .../configs/odqa/ru_odqa_infer_wiki.json | 3 - .../odqa/ru_odqa_infer_wiki_retr_noans.json | 3 - .../odqa/ru_odqa_infer_wiki_rubert.json | 3 - .../odqa/ru_odqa_infer_wiki_rubert_noans.json | 3 - .../paraphrase_ident_elmo_interact.json | 4 - .../ranking/paraphrase_ident_paraphraser.json | 4 - .../paraphrase_ident_paraphraser_elmo.json | 4 - ...paraphrase_ident_paraphraser_interact.json | 4 - ...paraphrase_ident_paraphraser_pretrain.json | 4 - .../paraphrase_ident_paraphraser_tune.json | 4 - .../configs/ranking/paraphrase_ident_qqp.json | 4 - .../ranking/paraphrase_ident_qqp_bilstm.json | 4 - .../paraphrase_ident_qqp_bilstm_interact.json | 4 - .../paraphrase_ident_qqp_interact.json | 4 - .../paraphrase_ident_tune_interact.json | 4 - .../configs/ranking/ranking_default.json | 4 - .../ranking/ranking_default_triplet.json | 4 - .../configs/ranking/ranking_insurance.json | 4 - .../ranking/ranking_insurance_interact.json | 4 - .../ranking_ubuntu_v1_mt_word2vec_dam.json | 4 - ...ubuntu_v1_mt_word2vec_dam_transformer.json | 4 - .../ranking_ubuntu_v1_mt_word2vec_smn.json | 4 - .../configs/ranking/ranking_ubuntu_v2.json | 4 - .../ranking/ranking_ubuntu_v2_interact.json | 4 - .../configs/ranking/ranking_ubuntu_v2_mt.json | 4 - .../ranking_ubuntu_v2_mt_interact.json | 4 - .../ranking_ubuntu_v2_mt_word2vec_dam.json | 4 - ...ubuntu_v2_mt_word2vec_dam_transformer.json | 4 - .../ranking_ubuntu_v2_mt_word2vec_smn.json | 4 - .../configs/seq2seq_go_bot/bot_kvret.json | 4 - .../seq2seq_go_bot/bot_kvret_train.json | 4 - .../brillmoore_kartaslov_ru.json | 4 - .../brillmoore_kartaslov_ru_custom_vocab.json | 4 - .../brillmoore_kartaslov_ru_nolm.json | 4 - .../brillmoore_wikitypos_en.json | 4 - .../levenshtein_corrector_ru.json | 4 - .../configs/squad/multi_squad_noans.json | 4 - .../squad/multi_squad_noans_infer.json | 4 - .../configs/squad/multi_squad_retr_noans.json | 4 - .../squad/multi_squad_ru_retr_noans.json | 4 - .../multi_squad_ru_retr_noans_rubert.json | 4 - ...ulti_squad_ru_retr_noans_rubert_infer.json | 4 - deeppavlov/configs/squad/squad.json | 4 - deeppavlov/configs/squad/squad_bert.json | 4 - .../configs/squad/squad_bert_infer.json | 4 - .../squad_bert_multilingual_freezed_emb.json | 4 - .../configs/squad/squad_bert_uncased.json | 4 - deeppavlov/configs/squad/squad_ru.json | 4 - deeppavlov/configs/squad/squad_ru_bert.json | 4 - .../configs/squad/squad_ru_bert_infer.json | 4 - deeppavlov/configs/squad/squad_ru_rubert.json | 4 - .../configs/squad/squad_ru_rubert_infer.json | 4 - deeppavlov/configs/squad/squad_zh_bert.json | 4 - deeppavlov/core/commands/train.py | 4 +- deeppavlov/core/common/metrics_registry.py | 14 ++++ deeppavlov/core/trainers/nn_trainer.py | 2 +- deeppavlov/dataset_readers/kvret_reader.py | 4 +- deeppavlov/deep.py | 8 +- .../models/api_requester/api_requester.py | 14 ++++ deeppavlov/models/api_requester/api_router.py | 14 ++++ .../classifiers/ru_obscenity_classifier.py | 14 ++++ deeppavlov/models/go_bot/network.py | 4 +- deeppavlov/models/kbqa/entity_linking.py | 2 +- .../models/kbqa/kb_answer_parser_wikidata.py | 4 +- .../models/ranking/tf_base_matching_model.py | 28 +++---- .../models/seq2seq_go_bot/dialog_state.py | 28 +++---- .../models/seq2seq_go_bot/kb_attn_layer.py | 28 +++---- .../levenshtein/levenshtein_searcher.py | 14 ++++ .../levenshtein/tabled_trie.py | 14 ++++ deeppavlov/skills/aiml_skill/aiml_skill.py | 2 - deeppavlov/utils/alexa/server.py | 2 +- deeppavlov/utils/connector/bot.py | 12 +-- deeppavlov/utils/connector/conversation.py | 8 +- deeppavlov/utils/ms_bot_framework/server.py | 2 +- deeppavlov/utils/pip_wrapper/pip_wrapper.py | 18 ++++- deeppavlov/utils/server/server.py | 12 +-- .../utils/settings/connector_config.json | 59 -------------- deeppavlov/utils/settings/server_config.json | 79 ++++++++----------- deeppavlov/utils/settings/socket_config.json | 13 --- deeppavlov/utils/socket/__init__.py | 2 +- deeppavlov/utils/socket/socket.py | 13 ++- deeppavlov/utils/telegram/telegram_ui.py | 4 +- deeppavlov/vocabs/wiki_sqlite.py | 2 +- docs/features/models/classifiers.rst | 2 +- docs/features/models/slot_filling.rst | 27 +++++-- docs/features/overview.rst | 4 +- docs/integrations/amazon_alexa.rst | 3 +- docs/integrations/ms_bot.rst | 3 +- docs/integrations/rest_api.rst | 36 +++++---- docs/integrations/socket_api.rst | 39 ++++----- docs/integrations/telegram.rst | 21 ++--- docs/integrations/yandex_alice.rst | 2 +- docs/intro/quick_start.rst | 4 +- .../classifiers/intents_snips_bigru.json | 3 - .../classifiers/intents_snips_bilstm.json | 3 - .../intents_snips_bilstm_bilstm.json | 3 - .../classifiers/intents_snips_bilstm_cnn.json | 3 - .../intents_snips_bilstm_proj_layer.json | 3 - ...tents_snips_bilstm_self_add_attention.json | 3 - ...ents_snips_bilstm_self_mult_attention.json | 3 - .../classifiers/intents_snips_cnn_bilstm.json | 3 - .../en_ranker_pop_wiki_test.json | 3 - .../en_ranker_tfidf_wiki_test.json | 3 - .../ru_ranker_tfidf_wiki_test.json | 3 - .../odqa/en_odqa_infer_wiki_test.json | 3 - .../odqa/en_odqa_pop_infer_wiki_test.json | 3 - .../odqa/ru_odqa_infer_wiki_test.json | 3 - ...hrase_ident_paraphraser_interact_test.json | 4 - .../paraphrase_ident_paraphraser_test.json | 4 - ...phrase_ident_qqp_bilstm_interact_test.json | 4 - .../paraphrase_ident_qqp_bilstm_test.json | 4 - .../paraphrase_ident_qqp_interact_test.json | 4 - .../ranking/paraphrase_ident_qqp_test.json | 4 - .../ranking_insurance_interact_test.json | 4 - .../ranking/ranking_insurance_test.json | 4 - .../ranking_ubuntu_v2_mt_interact_test.json | 4 - .../ranking/ranking_ubuntu_v2_mt_test.json | 4 - .../ranking/ranking_ubuntu_v2_test.json | 4 - tests/test_quick_start.py | 9 +-- 189 files changed, 313 insertions(+), 845 deletions(-) delete mode 100644 deeppavlov/utils/settings/connector_config.json delete mode 100644 deeppavlov/utils/settings/socket_config.json diff --git a/README.md b/README.md index 55527a2e84..0f450480bb 100644 --- a/README.md +++ b/README.md @@ -153,9 +153,9 @@ python -m deeppavlov [-d] * `interact` to interact via CLI, * `riseapi` to run a REST API server (see [doc](http://docs.deeppavlov.ai/en/master/integrations/rest_api.html)), - * `interactbot` to run as a Telegram bot (see + * `telegram` to run as a Telegram bot (see [doc](http://docs.deeppavlov.ai/en/master/integrations/telegram.html)), - * `interactmsbot` to run a Miscrosoft Bot Framework server (see + * `msbot` to run a Miscrosoft Bot Framework server (see [doc](http://docs.deeppavlov.ai/en/master/integrations/ms_bot.html)), * `predict` to get prediction for samples from *stdin* or from ** if `-f ` is specified. @@ -219,15 +219,18 @@ and others in the Integrations section for more info. **Breaking changes in version 0.7.0** - in dialog logger config file [dialog_logger_config.json](deeppavlov/utils/settings/dialog_logger_config.json) `agent_name` parameter was renamed to `logger_name`, the default value was changed -- Agent, Skill, eCommerce Bot and Pattern Matching classes were moved to [deeppavlov.deprecated](deeppavlov/deprecated/) +- Agent, Skill, eCommerce Bot and Pattern Matching classes were moved to [deeppavlov.deprecated](deeppavlov/deprecated) - [AIML Skill](http://docs.deeppavlov.ai/en/0.7.0/features/skills/aiml_skill.html), [RASA Skill](http://docs.deeppavlov.ai/en/0.7.0/features/skills/rasa_skill.html), [Yandex Alice](http://docs.deeppavlov.ai/en/0.7.0/integrations/yandex_alice.html), [Amazon Alexa](http://docs.deeppavlov.ai/en/0.7.0/integrations/amazon_alexa.html), [Microsoft Bot Framework](http://docs.deeppavlov.ai/en/0.7.0/integrations/ms_bot.html) and [Telegram integration](http://docs.deeppavlov.ai/en/0.7.0/integrations/telegram.html) interfaces were changed -- `/start` and `/help` Telegram messages were moved from `models_info.json` to [connector_config.json](/deeppavlov/utils/settings/connector_config.json) +- `/start` and `/help` Telegram messages were moved from `models_info.json` to [server_config.json](deeppavlov/utils/settings/server_config.json) - [risesocket](http://docs.deeppavlov.ai/en/0.7.0/integrations/socket_api.html) request and response format was changed +- [riseapi](http://docs.deeppavlov.ai/en/0.7.0/integrations/rest_api.html#advanced-configuration) and + [risesocket](http://docs.deeppavlov.ai/en/0.7.0/integrations/socket_api.html#advanced-configuration) model-specific + properties parametrization was changed **Breaking changes in version 0.6.0** - [REST API](http://docs.deeppavlov.ai/en/0.6.0/integrations/rest_api.html): diff --git a/deeppavlov/configs/classifiers/insults_kaggle.json b/deeppavlov/configs/classifiers/insults_kaggle.json index a3ca0f7238..34e1b800ea 100644 --- a/deeppavlov/configs/classifiers/insults_kaggle.json +++ b/deeppavlov/configs/classifiers/insults_kaggle.json @@ -141,10 +141,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/insults_data.tar.gz", diff --git a/deeppavlov/configs/classifiers/insults_kaggle_bert.json b/deeppavlov/configs/classifiers/insults_kaggle_bert.json index 83974a46a1..2206c698d2 100644 --- a/deeppavlov/configs/classifiers/insults_kaggle_bert.json +++ b/deeppavlov/configs/classifiers/insults_kaggle_bert.json @@ -123,10 +123,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/insults_data.tar.gz", diff --git a/deeppavlov/configs/classifiers/insults_kaggle_conv_bert.json b/deeppavlov/configs/classifiers/insults_kaggle_conv_bert.json index 0884166985..c819a6e034 100644 --- a/deeppavlov/configs/classifiers/insults_kaggle_conv_bert.json +++ b/deeppavlov/configs/classifiers/insults_kaggle_conv_bert.json @@ -139,10 +139,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/insults_data.tar.gz", diff --git a/deeppavlov/configs/classifiers/intents_dstc2.json b/deeppavlov/configs/classifiers/intents_dstc2.json index dc118c37a4..48d0776fad 100644 --- a/deeppavlov/configs/classifiers/intents_dstc2.json +++ b/deeppavlov/configs/classifiers/intents_dstc2.json @@ -146,10 +146,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/embeddings/dstc2_fastText_model.bin", diff --git a/deeppavlov/configs/classifiers/intents_dstc2_bert.json b/deeppavlov/configs/classifiers/intents_dstc2_bert.json index 83de46665c..cb8599da08 100644 --- a/deeppavlov/configs/classifiers/intents_dstc2_bert.json +++ b/deeppavlov/configs/classifiers/intents_dstc2_bert.json @@ -106,10 +106,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/cased_L-12_H-768_A-12.zip", diff --git a/deeppavlov/configs/classifiers/intents_dstc2_big.json b/deeppavlov/configs/classifiers/intents_dstc2_big.json index 4a08197c2e..8359c57937 100644 --- a/deeppavlov/configs/classifiers/intents_dstc2_big.json +++ b/deeppavlov/configs/classifiers/intents_dstc2_big.json @@ -145,9 +145,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/embeddings/wiki.en.bin", diff --git a/deeppavlov/configs/classifiers/intents_sample_csv.json b/deeppavlov/configs/classifiers/intents_sample_csv.json index b3eeee623e..e2530c95c2 100644 --- a/deeppavlov/configs/classifiers/intents_sample_csv.json +++ b/deeppavlov/configs/classifiers/intents_sample_csv.json @@ -146,10 +146,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/snips_intents/train.csv", diff --git a/deeppavlov/configs/classifiers/intents_sample_json.json b/deeppavlov/configs/classifiers/intents_sample_json.json index 23f2bdd8e9..2b5e0318e5 100644 --- a/deeppavlov/configs/classifiers/intents_sample_json.json +++ b/deeppavlov/configs/classifiers/intents_sample_json.json @@ -141,10 +141,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/snips_intents/train.json", diff --git a/deeppavlov/configs/classifiers/intents_snips.json b/deeppavlov/configs/classifiers/intents_snips.json index de684dd21e..914c0f41d9 100644 --- a/deeppavlov/configs/classifiers/intents_snips.json +++ b/deeppavlov/configs/classifiers/intents_snips.json @@ -131,10 +131,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/embeddings/dstc2_fastText_model.bin", diff --git a/deeppavlov/configs/classifiers/intents_snips_big.json b/deeppavlov/configs/classifiers/intents_snips_big.json index e113c9f86c..6b638d1ec8 100644 --- a/deeppavlov/configs/classifiers/intents_snips_big.json +++ b/deeppavlov/configs/classifiers/intents_snips_big.json @@ -131,10 +131,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/embeddings/wiki.en.bin", diff --git a/deeppavlov/configs/classifiers/intents_snips_sklearn.json b/deeppavlov/configs/classifiers/intents_snips_sklearn.json index 61012f7958..a407d23fb6 100644 --- a/deeppavlov/configs/classifiers/intents_snips_sklearn.json +++ b/deeppavlov/configs/classifiers/intents_snips_sklearn.json @@ -158,10 +158,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/classifiers/intents_snips_sklearn_v11.tar.gz", diff --git a/deeppavlov/configs/classifiers/intents_snips_tfidf_weighted.json b/deeppavlov/configs/classifiers/intents_snips_tfidf_weighted.json index b767a26a20..580ca692d8 100644 --- a/deeppavlov/configs/classifiers/intents_snips_tfidf_weighted.json +++ b/deeppavlov/configs/classifiers/intents_snips_tfidf_weighted.json @@ -172,10 +172,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/classifiers/intents_snips_sklearn_v12.tar.gz", diff --git a/deeppavlov/configs/classifiers/rusentiment_bert.json b/deeppavlov/configs/classifiers/rusentiment_bert.json index 4de0d36918..25f54c8fa2 100644 --- a/deeppavlov/configs/classifiers/rusentiment_bert.json +++ b/deeppavlov/configs/classifiers/rusentiment_bert.json @@ -136,10 +136,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/multi_cased_L-12_H-768_A-12.zip", diff --git a/deeppavlov/configs/classifiers/rusentiment_bigru_superconv.json b/deeppavlov/configs/classifiers/rusentiment_bigru_superconv.json index 4a7fbf490a..eef5f71889 100644 --- a/deeppavlov/configs/classifiers/rusentiment_bigru_superconv.json +++ b/deeppavlov/configs/classifiers/rusentiment_bigru_superconv.json @@ -155,10 +155,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/embeddings/ft_native_300_ru_twitter_nltk_word_tokenize.bin", diff --git a/deeppavlov/configs/classifiers/rusentiment_cnn.json b/deeppavlov/configs/classifiers/rusentiment_cnn.json index 3b4f9e9c75..a2c1ff5de8 100644 --- a/deeppavlov/configs/classifiers/rusentiment_cnn.json +++ b/deeppavlov/configs/classifiers/rusentiment_cnn.json @@ -157,10 +157,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/embeddings/ft_native_300_ru_wiki_lenta_nltk_wordpunct_tokenize/ft_native_300_ru_wiki_lenta_nltk_wordpunct_tokenize.bin", diff --git a/deeppavlov/configs/classifiers/rusentiment_convers_bert.json b/deeppavlov/configs/classifiers/rusentiment_convers_bert.json index ccc9012847..f02f5ec5ee 100644 --- a/deeppavlov/configs/classifiers/rusentiment_convers_bert.json +++ b/deeppavlov/configs/classifiers/rusentiment_convers_bert.json @@ -136,10 +136,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/ru_conversational_cased_L-12_H-768_A-12.tar.gz", diff --git a/deeppavlov/configs/classifiers/rusentiment_elmo_twitter_cnn.json b/deeppavlov/configs/classifiers/rusentiment_elmo_twitter_cnn.json index 038f4104bb..db4cd5455c 100644 --- a/deeppavlov/configs/classifiers/rusentiment_elmo_twitter_cnn.json +++ b/deeppavlov/configs/classifiers/rusentiment_elmo_twitter_cnn.json @@ -164,10 +164,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/tf-hub.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/classifiers/rusentiment_v10.tar.gz", diff --git a/deeppavlov/configs/classifiers/sentiment_sst_conv_bert.json b/deeppavlov/configs/classifiers/sentiment_sst_conv_bert.json index e907b240cd..6a33aabbee 100644 --- a/deeppavlov/configs/classifiers/sentiment_sst_conv_bert.json +++ b/deeppavlov/configs/classifiers/sentiment_sst_conv_bert.json @@ -125,10 +125,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/stanfordSentimentTreebank.zip", diff --git a/deeppavlov/configs/classifiers/sentiment_sst_multi_bert.json b/deeppavlov/configs/classifiers/sentiment_sst_multi_bert.json index 95e7a59ef0..4a2ec2e088 100644 --- a/deeppavlov/configs/classifiers/sentiment_sst_multi_bert.json +++ b/deeppavlov/configs/classifiers/sentiment_sst_multi_bert.json @@ -125,10 +125,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/stanfordSentimentTreebank.zip", diff --git a/deeppavlov/configs/classifiers/sentiment_twitter.json b/deeppavlov/configs/classifiers/sentiment_twitter.json index 4d8e3ce4b1..aa2af5b46e 100644 --- a/deeppavlov/configs/classifiers/sentiment_twitter.json +++ b/deeppavlov/configs/classifiers/sentiment_twitter.json @@ -132,10 +132,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/sentiment_twitter_data.tar.gz", diff --git a/deeppavlov/configs/classifiers/sentiment_twitter_preproc.json b/deeppavlov/configs/classifiers/sentiment_twitter_preproc.json index cf127367c7..b62ddc7cad 100644 --- a/deeppavlov/configs/classifiers/sentiment_twitter_preproc.json +++ b/deeppavlov/configs/classifiers/sentiment_twitter_preproc.json @@ -145,10 +145,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/sentiment_twitter_data.tar.gz", diff --git a/deeppavlov/configs/classifiers/sentiment_yelp_conv_bert.json b/deeppavlov/configs/classifiers/sentiment_yelp_conv_bert.json index ac7ff0156b..f0b9b8a045 100644 --- a/deeppavlov/configs/classifiers/sentiment_yelp_conv_bert.json +++ b/deeppavlov/configs/classifiers/sentiment_yelp_conv_bert.json @@ -139,10 +139,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/yelp_review_full_csv.tar.gz", diff --git a/deeppavlov/configs/classifiers/sentiment_yelp_multi_bert.json b/deeppavlov/configs/classifiers/sentiment_yelp_multi_bert.json index fd93b3ecb6..630319f160 100644 --- a/deeppavlov/configs/classifiers/sentiment_yelp_multi_bert.json +++ b/deeppavlov/configs/classifiers/sentiment_yelp_multi_bert.json @@ -139,10 +139,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/yelp_review_full_csv.tar.gz", diff --git a/deeppavlov/configs/classifiers/topic_ag_news.json b/deeppavlov/configs/classifiers/topic_ag_news.json index 8def9bcace..93152e3491 100644 --- a/deeppavlov/configs/classifiers/topic_ag_news.json +++ b/deeppavlov/configs/classifiers/topic_ag_news.json @@ -140,10 +140,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/ag_news_data.tar.gz", diff --git a/deeppavlov/configs/classifiers/yahoo_convers_vs_info.json b/deeppavlov/configs/classifiers/yahoo_convers_vs_info.json index 04c5a54d9a..8542aa9f72 100644 --- a/deeppavlov/configs/classifiers/yahoo_convers_vs_info.json +++ b/deeppavlov/configs/classifiers/yahoo_convers_vs_info.json @@ -157,10 +157,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/tf-hub.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/embeddings/yahooo-sber-questions_epoches_n_15.tar.gz", diff --git a/deeppavlov/configs/classifiers/yahoo_convers_vs_info_bert.json b/deeppavlov/configs/classifiers/yahoo_convers_vs_info_bert.json index a33a240b9f..04dd679783 100644 --- a/deeppavlov/configs/classifiers/yahoo_convers_vs_info_bert.json +++ b/deeppavlov/configs/classifiers/yahoo_convers_vs_info_bert.json @@ -150,10 +150,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/conversational_cased_L-12_H-768_A-12.tar.gz", diff --git a/deeppavlov/configs/doc_retrieval/en_ranker_pop_enwiki20180211.json b/deeppavlov/configs/doc_retrieval/en_ranker_pop_enwiki20180211.json index 741c2e8b98..8d072fbf39 100644 --- a/deeppavlov/configs/doc_retrieval/en_ranker_pop_enwiki20180211.json +++ b/deeppavlov/configs/doc_retrieval/en_ranker_pop_enwiki20180211.json @@ -78,9 +78,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/wikipedia/enwiki.tar.gz", diff --git a/deeppavlov/configs/doc_retrieval/en_ranker_tfidf_enwiki20161221.json b/deeppavlov/configs/doc_retrieval/en_ranker_tfidf_enwiki20161221.json index 0fb2f24970..7635c89ded 100644 --- a/deeppavlov/configs/doc_retrieval/en_ranker_tfidf_enwiki20161221.json +++ b/deeppavlov/configs/doc_retrieval/en_ranker_tfidf_enwiki20161221.json @@ -70,9 +70,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/wikipedia/enwiki20161221.tar.gz", diff --git a/deeppavlov/configs/doc_retrieval/en_ranker_tfidf_wiki.json b/deeppavlov/configs/doc_retrieval/en_ranker_tfidf_wiki.json index a4515ece56..03c0ab250c 100644 --- a/deeppavlov/configs/doc_retrieval/en_ranker_tfidf_wiki.json +++ b/deeppavlov/configs/doc_retrieval/en_ranker_tfidf_wiki.json @@ -70,9 +70,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/wikipedia/enwiki.tar.gz", diff --git a/deeppavlov/configs/doc_retrieval/ru_ranker_tfidf_wiki.json b/deeppavlov/configs/doc_retrieval/ru_ranker_tfidf_wiki.json index 4dda2a06d4..629152ad6c 100644 --- a/deeppavlov/configs/doc_retrieval/ru_ranker_tfidf_wiki.json +++ b/deeppavlov/configs/doc_retrieval/ru_ranker_tfidf_wiki.json @@ -67,9 +67,6 @@ "MODELS_PATH": "{ROOT_PATH}/models" }, "requirements": [], - "labels": { - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/wikipedia/ruwiki.tar.gz", diff --git a/deeppavlov/configs/ecommerce_skill/bleu_retrieve.json b/deeppavlov/configs/ecommerce_skill/bleu_retrieve.json index 6a949b630b..e754becbfe 100644 --- a/deeppavlov/configs/ecommerce_skill/bleu_retrieve.json +++ b/deeppavlov/configs/ecommerce_skill/bleu_retrieve.json @@ -67,10 +67,6 @@ "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", "MODELS_PATH": "{ROOT_PATH}/models" }, - "labels": { - "telegram_utils": "EcommerceSkill", - "server_utils": "EcommerceSkill" - }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" diff --git a/deeppavlov/configs/ecommerce_skill/tfidf_retrieve.json b/deeppavlov/configs/ecommerce_skill/tfidf_retrieve.json index 8dc09bc774..72b9632abe 100644 --- a/deeppavlov/configs/ecommerce_skill/tfidf_retrieve.json +++ b/deeppavlov/configs/ecommerce_skill/tfidf_retrieve.json @@ -98,10 +98,6 @@ "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", "MODELS_PATH": "{ROOT_PATH}/models" }, - "labels": { - "telegram_utils": "EcommerceSkill", - "server_utils": "EcommerceSkill" - }, "requirements": [ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" diff --git a/deeppavlov/configs/evolution/evolve_intents_snips.json b/deeppavlov/configs/evolution/evolve_intents_snips.json index ca37789fbd..d6ee797560 100644 --- a/deeppavlov/configs/evolution/evolve_intents_snips.json +++ b/deeppavlov/configs/evolution/evolve_intents_snips.json @@ -194,10 +194,6 @@ "MODELS_PATH": "{ROOT_PATH}/models", "MODEL_PATH": "{MODELS_PATH}/classifiers/intents_snips_evolution" }, - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/embeddings/wiki.en.bin", diff --git a/deeppavlov/configs/evolution/evolve_rusentiment_cnn.json b/deeppavlov/configs/evolution/evolve_rusentiment_cnn.json index 6ff2b47e19..595c0b9785 100644 --- a/deeppavlov/configs/evolution/evolve_rusentiment_cnn.json +++ b/deeppavlov/configs/evolution/evolve_rusentiment_cnn.json @@ -199,10 +199,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel", - "server_utils": "KerasIntentModel" - }, "download": [ { "url": "https://github.com/text-machine-lab/rusentiment/raw/master/Dataset/rusentiment_random_posts.csv", diff --git a/deeppavlov/configs/go_bot/gobot_dstc2.json b/deeppavlov/configs/go_bot/gobot_dstc2.json index d2f7707654..7a658a634b 100644 --- a/deeppavlov/configs/go_bot/gobot_dstc2.json +++ b/deeppavlov/configs/go_bot/gobot_dstc2.json @@ -109,10 +109,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "telegram_utils": "GoalOrientedBot", - "server_utils": "GoalOrientedBot" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/gobot_dstc2_v9.tar.gz", diff --git a/deeppavlov/configs/go_bot/gobot_dstc2_best.json b/deeppavlov/configs/go_bot/gobot_dstc2_best.json index d0d3842694..2f81cbf211 100644 --- a/deeppavlov/configs/go_bot/gobot_dstc2_best.json +++ b/deeppavlov/configs/go_bot/gobot_dstc2_best.json @@ -117,10 +117,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "telegram_utils": "GoalOrientedBot", - "server_utils": "GoalOrientedBot" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/gobot_dstc2_best_v4.tar.gz", diff --git a/deeppavlov/configs/go_bot/gobot_dstc2_minimal.json b/deeppavlov/configs/go_bot/gobot_dstc2_minimal.json index eeadeadc31..f022c47ffb 100644 --- a/deeppavlov/configs/go_bot/gobot_dstc2_minimal.json +++ b/deeppavlov/configs/go_bot/gobot_dstc2_minimal.json @@ -99,10 +99,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "telegram_utils": "GoalOrientedBot", - "server_utils": "GoalOrientedBot" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/gobot_dstc2_v9.tar.gz", diff --git a/deeppavlov/configs/go_bot/gobot_simple_dstc2.json b/deeppavlov/configs/go_bot/gobot_simple_dstc2.json index 01ba40b1f3..f0df51cf36 100644 --- a/deeppavlov/configs/go_bot/gobot_simple_dstc2.json +++ b/deeppavlov/configs/go_bot/gobot_simple_dstc2.json @@ -109,10 +109,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "telegram_utils": "GoalOrientedBot", - "server_utils": "GoalOrientedBot" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/gobot_dstc2_v9.tar.gz", diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ar.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ar.json index 5a6dedf6ec..22d2623f8c 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ar.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ar.json @@ -170,9 +170,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/ar.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/ar" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_cs.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_cs.json index 0171da3de9..ec1318001d 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_cs.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_cs.json @@ -170,9 +170,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/cs.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/cs" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_de.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_de.json index c6e304f164..0faeac1b13 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_de.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_de.json @@ -170,9 +170,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/de.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/de" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_en.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_en.json index 441f6b27df..c847f5e5b9 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_en.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_en.json @@ -170,9 +170,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/en.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/en" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_es_ancora.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_es_ancora.json index 1bb78b7bbb..a50cb801b7 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_es_ancora.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_es_ancora.json @@ -170,9 +170,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/es_ancora.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/es_ancora" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_fr.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_fr.json index d48241e913..256977765e 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_fr.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_fr.json @@ -170,9 +170,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/fr.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/fr" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hi.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hi.json index 120b18f75d..ac79415851 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hi.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hi.json @@ -170,9 +170,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/hi.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/hi" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hu.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hu.json index 623e7548fd..1dc47596a5 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hu.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hu.json @@ -170,9 +170,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/hu.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/hu" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_it.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_it.json index dfcd799d05..d545af336f 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_it.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_it.json @@ -170,9 +170,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/it.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/it" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus.json index 1aaa8ca70b..3a12f35948 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus.json @@ -170,9 +170,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/ru_syntagrus.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/ru_syntagrus" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy.json index b4f6da6f6a..0f0ec0dd29 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy.json @@ -191,9 +191,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/ru_syntagrus.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/ru_syntagrus" } - ], - "labels": { - "telegram_utils": "MorphoTaggerPymorphyModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy_lemmatize.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy_lemmatize.json index 34a8c55aec..664ddfd317 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy_lemmatize.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy_lemmatize.json @@ -199,9 +199,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/ru_syntagrus.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/ru_syntagrus" } - ], - "labels": { - "telegram_utils": "MorphoTaggerPymorphyLemmatizedModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_tr.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_tr.json index b4e96569cc..602c241236 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_tr.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_tr.json @@ -171,9 +171,6 @@ "url": "http://files.deeppavlov.ai/datasets/UD2.0_source/tr.tar.gz", "subdir": "{DOWNLOADS_PATH}/UD2.0_source/tr" } - ], - "labels": { - "telegram_utils": "MorphoTaggerModel" - } + ] } } \ No newline at end of file diff --git a/deeppavlov/configs/ner/ner_conll2003.json b/deeppavlov/configs/ner/ner_conll2003.json index 3cf3c8ef45..caaa220be7 100644 --- a/deeppavlov/configs/ner/ner_conll2003.json +++ b/deeppavlov/configs/ner/ner_conll2003.json @@ -167,10 +167,6 @@ "{DEEPPAVLOV_PATH}/requirements/gensim.txt", "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ner_conll2003_v5.tar.gz", diff --git a/deeppavlov/configs/ner/ner_conll2003_bert.json b/deeppavlov/configs/ner/ner_conll2003_bert.json index 18d8abb26e..7166cc31f4 100644 --- a/deeppavlov/configs/ner/ner_conll2003_bert.json +++ b/deeppavlov/configs/ner/ner_conll2003_bert.json @@ -107,10 +107,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ner_conll2003_bert_v1.tar.gz", diff --git a/deeppavlov/configs/ner/ner_conll2003_pos.json b/deeppavlov/configs/ner/ner_conll2003_pos.json index 09eced3d29..63b27a1810 100644 --- a/deeppavlov/configs/ner/ner_conll2003_pos.json +++ b/deeppavlov/configs/ner/ner_conll2003_pos.json @@ -183,10 +183,6 @@ "{DEEPPAVLOV_PATH}/requirements/gensim.txt", "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { "url": "http://files.deeppavlov.ai/embeddings/glove.6B.100d.txt", diff --git a/deeppavlov/configs/ner/ner_dstc2.json b/deeppavlov/configs/ner/ner_dstc2.json index 9840c2fb3a..fbe958f664 100644 --- a/deeppavlov/configs/ner/ner_dstc2.json +++ b/deeppavlov/configs/ner/ner_dstc2.json @@ -115,10 +115,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "NERModel", - "server_utils": "NER" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/dstc_slot_vals.tar.gz", diff --git a/deeppavlov/configs/ner/ner_few_shot_ru.json b/deeppavlov/configs/ner/ner_few_shot_ru.json index 3097bfcd4e..393f440b03 100644 --- a/deeppavlov/configs/ner/ner_few_shot_ru.json +++ b/deeppavlov/configs/ner/ner_few_shot_ru.json @@ -99,10 +99,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/tf-hub.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/elmo_ru-news_wmt11-16_1.5M_steps.tar.gz", diff --git a/deeppavlov/configs/ner/ner_few_shot_ru_simulate.json b/deeppavlov/configs/ner/ner_few_shot_ru_simulate.json index 840a4b732c..5c880357d5 100644 --- a/deeppavlov/configs/ner/ner_few_shot_ru_simulate.json +++ b/deeppavlov/configs/ner/ner_few_shot_ru_simulate.json @@ -135,10 +135,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/tf-hub.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/elmo_ru-news_wmt11-16_1.5M_steps.tar.gz", diff --git a/deeppavlov/configs/ner/ner_kb_rus.json b/deeppavlov/configs/ner/ner_kb_rus.json index 2cb3045127..210ca90ab8 100644 --- a/deeppavlov/configs/ner/ner_kb_rus.json +++ b/deeppavlov/configs/ner/ner_kb_rus.json @@ -156,10 +156,6 @@ "{DEEPPAVLOV_PATH}/requirements/fasttext.txt", "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ner_rus_v2_cpu_compatible.tar.gz", diff --git a/deeppavlov/configs/ner/ner_ontonotes.json b/deeppavlov/configs/ner/ner_ontonotes.json index b1a6339622..a0a2a3114f 100644 --- a/deeppavlov/configs/ner/ner_ontonotes.json +++ b/deeppavlov/configs/ner/ner_ontonotes.json @@ -154,10 +154,6 @@ "{DEEPPAVLOV_PATH}/requirements/gensim.txt", "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ner_ontonotes_v3_cpu_compatible.tar.gz", diff --git a/deeppavlov/configs/ner/ner_ontonotes_bert.json b/deeppavlov/configs/ner/ner_ontonotes_bert.json index 6707f00f5b..8d5dcf8e34 100644 --- a/deeppavlov/configs/ner/ner_ontonotes_bert.json +++ b/deeppavlov/configs/ner/ner_ontonotes_bert.json @@ -107,10 +107,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { diff --git a/deeppavlov/configs/ner/ner_ontonotes_bert_mult.json b/deeppavlov/configs/ner/ner_ontonotes_bert_mult.json index 2f6373d650..a9a9066553 100644 --- a/deeppavlov/configs/ner/ner_ontonotes_bert_mult.json +++ b/deeppavlov/configs/ner/ner_ontonotes_bert_mult.json @@ -107,10 +107,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { diff --git a/deeppavlov/configs/ner/ner_rus.json b/deeppavlov/configs/ner/ner_rus.json index f9e7e6cfec..bf7b3771e0 100644 --- a/deeppavlov/configs/ner/ner_rus.json +++ b/deeppavlov/configs/ner/ner_rus.json @@ -167,10 +167,6 @@ "{DEEPPAVLOV_PATH}/requirements/fasttext.txt", "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ner_rus_v3_cpu_compatible.tar.gz", diff --git a/deeppavlov/configs/ner/ner_rus_bert.json b/deeppavlov/configs/ner/ner_rus_bert.json index 855beaa9ab..31ba3c96b3 100644 --- a/deeppavlov/configs/ner/ner_rus_bert.json +++ b/deeppavlov/configs/ner/ner_rus_bert.json @@ -107,10 +107,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "NERCoNLL2003Model", - "server_utils": "NER" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ner_rus_bert_v1.tar.gz", diff --git a/deeppavlov/configs/ner/slotfill_dstc2.json b/deeppavlov/configs/ner/slotfill_dstc2.json index 217f2b2bfc..2f760863ba 100644 --- a/deeppavlov/configs/ner/slotfill_dstc2.json +++ b/deeppavlov/configs/ner/slotfill_dstc2.json @@ -53,10 +53,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "NERModel", - "server_utils": "DstcSlotFillingNetwork" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/dstc_slot_vals.tar.gz", diff --git a/deeppavlov/configs/ner/slotfill_dstc2_raw.json b/deeppavlov/configs/ner/slotfill_dstc2_raw.json index f66293b8bc..1f52e81f00 100644 --- a/deeppavlov/configs/ner/slotfill_dstc2_raw.json +++ b/deeppavlov/configs/ner/slotfill_dstc2_raw.json @@ -47,9 +47,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "NERModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/dstc_slot_vals.tar.gz", diff --git a/deeppavlov/configs/ner/slotfill_simple_dstc2_raw.json b/deeppavlov/configs/ner/slotfill_simple_dstc2_raw.json index 47f176394c..1b3389b81d 100644 --- a/deeppavlov/configs/ner/slotfill_simple_dstc2_raw.json +++ b/deeppavlov/configs/ner/slotfill_simple_dstc2_raw.json @@ -47,9 +47,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "NERModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/dstc_slot_vals.tar.gz", diff --git a/deeppavlov/configs/odqa/en_odqa_infer_enwiki20161221.json b/deeppavlov/configs/odqa/en_odqa_infer_enwiki20161221.json index ac46766de5..2247f01731 100644 --- a/deeppavlov/configs/odqa/en_odqa_infer_enwiki20161221.json +++ b/deeppavlov/configs/odqa/en_odqa_infer_enwiki20161221.json @@ -68,9 +68,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "server_utils": "ODQA" - }, "download": [ ] } diff --git a/deeppavlov/configs/odqa/en_odqa_infer_wiki.json b/deeppavlov/configs/odqa/en_odqa_infer_wiki.json index 00e07493d4..c8aac1f860 100644 --- a/deeppavlov/configs/odqa/en_odqa_infer_wiki.json +++ b/deeppavlov/configs/odqa/en_odqa_infer_wiki.json @@ -68,9 +68,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "server_utils": "ODQA" - }, "download": [ ] } diff --git a/deeppavlov/configs/odqa/en_odqa_pop_infer_enwiki20180211.json b/deeppavlov/configs/odqa/en_odqa_pop_infer_enwiki20180211.json index 825a287aa6..9d034898fc 100644 --- a/deeppavlov/configs/odqa/en_odqa_pop_infer_enwiki20180211.json +++ b/deeppavlov/configs/odqa/en_odqa_pop_infer_enwiki20180211.json @@ -80,9 +80,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "server_utils": "ODQA" - }, "download": [ ] } diff --git a/deeppavlov/configs/odqa/ru_odqa_infer_wiki.json b/deeppavlov/configs/odqa/ru_odqa_infer_wiki.json index 64b3e113af..5ec716cbea 100644 --- a/deeppavlov/configs/odqa/ru_odqa_infer_wiki.json +++ b/deeppavlov/configs/odqa/ru_odqa_infer_wiki.json @@ -66,9 +66,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "server_utils": "ODQA" - }, "download": [ ] } diff --git a/deeppavlov/configs/odqa/ru_odqa_infer_wiki_retr_noans.json b/deeppavlov/configs/odqa/ru_odqa_infer_wiki_retr_noans.json index 31b2aba3aa..1f553addf3 100644 --- a/deeppavlov/configs/odqa/ru_odqa_infer_wiki_retr_noans.json +++ b/deeppavlov/configs/odqa/ru_odqa_infer_wiki_retr_noans.json @@ -66,9 +66,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "server_utils": "ODQA" - }, "download": [ ] } diff --git a/deeppavlov/configs/odqa/ru_odqa_infer_wiki_rubert.json b/deeppavlov/configs/odqa/ru_odqa_infer_wiki_rubert.json index b774e10292..2174dd4de2 100644 --- a/deeppavlov/configs/odqa/ru_odqa_infer_wiki_rubert.json +++ b/deeppavlov/configs/odqa/ru_odqa_infer_wiki_rubert.json @@ -68,9 +68,6 @@ "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt", "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "server_utils": "ODQA" - }, "download": [ ] } diff --git a/deeppavlov/configs/odqa/ru_odqa_infer_wiki_rubert_noans.json b/deeppavlov/configs/odqa/ru_odqa_infer_wiki_rubert_noans.json index eb4533919f..be6a883d58 100644 --- a/deeppavlov/configs/odqa/ru_odqa_infer_wiki_rubert_noans.json +++ b/deeppavlov/configs/odqa/ru_odqa_infer_wiki_rubert_noans.json @@ -68,9 +68,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "server_utils": "ODQA" - }, "download": [ ] } diff --git a/deeppavlov/configs/ranking/paraphrase_ident_elmo_interact.json b/deeppavlov/configs/ranking/paraphrase_ident_elmo_interact.json index b96c2b9c3d..2bfb4727cf 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_elmo_interact.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_elmo_interact.json @@ -106,10 +106,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/tf-hub.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/paraphraser_elmo_ft_pre_1_model.tar.gz", diff --git a/deeppavlov/configs/ranking/paraphrase_ident_paraphraser.json b/deeppavlov/configs/ranking/paraphrase_ident_paraphraser.json index 9ed50a9575..13d20ccdb1 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_paraphraser.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_paraphraser.json @@ -94,10 +94,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/paraphraser.zip", diff --git a/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_elmo.json b/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_elmo.json index 26415640ef..1592ab463b 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_elmo.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_elmo.json @@ -99,10 +99,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/tf-hub.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/paraphraser.zip", diff --git a/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_interact.json b/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_interact.json index b0ad855564..7bae82f9b3 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_interact.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_interact.json @@ -103,10 +103,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/paraphrase_ident_paraphraser.tar.gz", diff --git a/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_pretrain.json b/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_pretrain.json index 5fab2d8c57..146a9844cb 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_pretrain.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_pretrain.json @@ -94,10 +94,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/paraphraser_pretrain_train.zip", diff --git a/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_tune.json b/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_tune.json index 729fa41d01..6b535a2452 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_tune.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_paraphraser_tune.json @@ -94,10 +94,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/paraphraser.zip", diff --git a/deeppavlov/configs/ranking/paraphrase_ident_qqp.json b/deeppavlov/configs/ranking/paraphrase_ident_qqp.json index e3279a8e6f..3cd3e0b6fb 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_qqp.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_qqp.json @@ -97,10 +97,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/quora_question_pairs.zip", diff --git a/deeppavlov/configs/ranking/paraphrase_ident_qqp_bilstm.json b/deeppavlov/configs/ranking/paraphrase_ident_qqp_bilstm.json index a6e05839bf..14a8a10dbe 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_qqp_bilstm.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_qqp_bilstm.json @@ -97,10 +97,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/quora_question_pairs.zip", diff --git a/deeppavlov/configs/ranking/paraphrase_ident_qqp_bilstm_interact.json b/deeppavlov/configs/ranking/paraphrase_ident_qqp_bilstm_interact.json index e5ce42772c..f9671281a6 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_qqp_bilstm_interact.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_qqp_bilstm_interact.json @@ -105,10 +105,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/paraphrase_ident_qqp_bilstm.tar.gz", diff --git a/deeppavlov/configs/ranking/paraphrase_ident_qqp_interact.json b/deeppavlov/configs/ranking/paraphrase_ident_qqp_interact.json index 8d83fc0164..611f965f1c 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_qqp_interact.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_qqp_interact.json @@ -105,10 +105,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/paraphrase_ident_qqp.tar.gz", diff --git a/deeppavlov/configs/ranking/paraphrase_ident_tune_interact.json b/deeppavlov/configs/ranking/paraphrase_ident_tune_interact.json index bc032abeea..853c7f0562 100644 --- a/deeppavlov/configs/ranking/paraphrase_ident_tune_interact.json +++ b/deeppavlov/configs/ranking/paraphrase_ident_tune_interact.json @@ -103,10 +103,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/paraphrase_ident_paraphraser_tuned.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_default.json b/deeppavlov/configs/ranking/ranking_default.json index 163a531c27..6f674ecaac 100644 --- a/deeppavlov/configs/ranking/ranking_default.json +++ b/deeppavlov/configs/ranking/ranking_default.json @@ -96,10 +96,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/default_ranking_data.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_default_triplet.json b/deeppavlov/configs/ranking/ranking_default_triplet.json index 4097071fe6..7416d58970 100644 --- a/deeppavlov/configs/ranking/ranking_default_triplet.json +++ b/deeppavlov/configs/ranking/ranking_default_triplet.json @@ -98,10 +98,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/default_ranking_data_triplet.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_insurance.json b/deeppavlov/configs/ranking/ranking_insurance.json index e789f7047e..4dcd2bc27d 100644 --- a/deeppavlov/configs/ranking/ranking_insurance.json +++ b/deeppavlov/configs/ranking/ranking_insurance.json @@ -102,10 +102,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/insuranceQA-master.zip", diff --git a/deeppavlov/configs/ranking/ranking_insurance_interact.json b/deeppavlov/configs/ranking/ranking_insurance_interact.json index ca7a614e6c..6e160f17de 100644 --- a/deeppavlov/configs/ranking/ranking_insurance_interact.json +++ b/deeppavlov/configs/ranking/ranking_insurance_interact.json @@ -110,10 +110,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/insurance_ranking.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam.json b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam.json index c8b52b7d85..33600486eb 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam.json @@ -117,10 +117,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/gensim.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ubuntu_v1_mt_word2vec_dam.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam_transformer.json b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam_transformer.json index 943aae71a2..f4f381a2d1 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam_transformer.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_dam_transformer.json @@ -121,10 +121,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf-hub.txt", "{DEEPPAVLOV_PATH}/requirements/gensim.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ubuntu_v1_mt_word2vec_dam_transformer.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_smn.json b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_smn.json index b357133f0f..2864e86337 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_smn.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v1_mt_word2vec_smn.json @@ -114,10 +114,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/gensim.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ubuntu_v1_mt_word2vec_smn.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2.json index 55787c84ef..6b738ff847 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2.json @@ -94,10 +94,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/ubuntu_v2_data.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_interact.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_interact.json index 5c9a9c4dc9..d57dafb433 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_interact.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_interact.json @@ -102,10 +102,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ubuntu_v2_ranking.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt.json index 0c190b1463..add8bdc538 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt.json @@ -97,10 +97,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/ubuntu_v2_data.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_interact.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_interact.json index d45cb3f52a..835a30cb57 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_interact.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_interact.json @@ -107,10 +107,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ubuntu_v2_mt_ranking.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam.json index c1824b7d6f..8af5b566d9 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam.json @@ -117,10 +117,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/gensim.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ubuntu_v2_mt_word2vec_dam.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam_transformer.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam_transformer.json index 61132f7859..aaaca46106 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam_transformer.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_dam_transformer.json @@ -121,10 +121,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf-hub.txt", "{DEEPPAVLOV_PATH}/requirements/gensim.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ubuntu_v2_mt_word2vec_dam_transformer.tar.gz", diff --git a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_smn.json b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_smn.json index 781baa175b..4e6eece3a5 100644 --- a/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_smn.json +++ b/deeppavlov/configs/ranking/ranking_ubuntu_v2_mt_word2vec_smn.json @@ -113,10 +113,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/gensim.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ubuntu_v2_mt_word2vec_smn.tar.gz", diff --git a/deeppavlov/configs/seq2seq_go_bot/bot_kvret.json b/deeppavlov/configs/seq2seq_go_bot/bot_kvret.json index c7f2b469cf..1498263f90 100644 --- a/deeppavlov/configs/seq2seq_go_bot/bot_kvret.json +++ b/deeppavlov/configs/seq2seq_go_bot/bot_kvret.json @@ -125,10 +125,6 @@ "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "Seq2SeqGoalOrientedBot", - "server_utils": "GoalOrientedBot" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs.tar.gz", diff --git a/deeppavlov/configs/seq2seq_go_bot/bot_kvret_train.json b/deeppavlov/configs/seq2seq_go_bot/bot_kvret_train.json index 1febea3fbb..391cca444d 100644 --- a/deeppavlov/configs/seq2seq_go_bot/bot_kvret_train.json +++ b/deeppavlov/configs/seq2seq_go_bot/bot_kvret_train.json @@ -158,10 +158,6 @@ "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "Seq2SeqGoalOrientedBot", - "server_utils": "GoalOrientedBot" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs.tar.gz", diff --git a/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru.json b/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru.json index 2ef47fac0a..75cc4fb8d4 100644 --- a/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru.json +++ b/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru.json @@ -72,10 +72,6 @@ "{DEEPPAVLOV_PATH}/requirements/spelling.txt", "{DEEPPAVLOV_PATH}/requirements/kenlm.txt" ], - "labels": { - "telegram_utils": "ErrorModel", - "server_utils": "ErrorModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/error_model.tar.gz", diff --git a/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru_custom_vocab.json b/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru_custom_vocab.json index c70a4238de..e5fe011ddb 100644 --- a/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru_custom_vocab.json +++ b/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru_custom_vocab.json @@ -74,10 +74,6 @@ "{DEEPPAVLOV_PATH}/requirements/spelling.txt", "{DEEPPAVLOV_PATH}/requirements/kenlm.txt" ], - "labels": { - "telegram_utils": "ErrorModel", - "server_utils": "ErrorModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/error_model.tar.gz", diff --git a/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru_nolm.json b/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru_nolm.json index 2263c63618..9c93e92e30 100644 --- a/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru_nolm.json +++ b/deeppavlov/configs/spelling_correction/brillmoore_kartaslov_ru_nolm.json @@ -70,10 +70,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/spelling.txt" ], - "labels": { - "telegram_utils": "ErrorModel", - "server_utils": "ErrorModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/error_model.tar.gz", diff --git a/deeppavlov/configs/spelling_correction/brillmoore_wikitypos_en.json b/deeppavlov/configs/spelling_correction/brillmoore_wikitypos_en.json index d25fb248a2..e75cc4215a 100644 --- a/deeppavlov/configs/spelling_correction/brillmoore_wikitypos_en.json +++ b/deeppavlov/configs/spelling_correction/brillmoore_wikitypos_en.json @@ -69,10 +69,6 @@ "{DEEPPAVLOV_PATH}/requirements/spelling.txt", "{DEEPPAVLOV_PATH}/requirements/kenlm.txt" ], - "labels": { - "telegram_utils": "ErrorModel", - "server_utils": "ErrorModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/error_model.tar.gz", diff --git a/deeppavlov/configs/spelling_correction/levenshtein_corrector_ru.json b/deeppavlov/configs/spelling_correction/levenshtein_corrector_ru.json index ca85a8c6c3..7355930b33 100644 --- a/deeppavlov/configs/spelling_correction/levenshtein_corrector_ru.json +++ b/deeppavlov/configs/spelling_correction/levenshtein_corrector_ru.json @@ -50,10 +50,6 @@ "{DEEPPAVLOV_PATH}/requirements/spelling.txt", "{DEEPPAVLOV_PATH}/requirements/kenlm.txt" ], - "labels": { - "telegram_utils": "ErrorModel", - "server_utils": "ErrorModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs/russian_words_vocab.dict.gz", diff --git a/deeppavlov/configs/squad/multi_squad_noans.json b/deeppavlov/configs/squad/multi_squad_noans.json index 33b80d0f17..90c9aff905 100644 --- a/deeppavlov/configs/squad/multi_squad_noans.json +++ b/deeppavlov/configs/squad/multi_squad_noans.json @@ -133,10 +133,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/multi_squad_model_noans_1.1.tar.gz", diff --git a/deeppavlov/configs/squad/multi_squad_noans_infer.json b/deeppavlov/configs/squad/multi_squad_noans_infer.json index 4786af61f0..00fbfd1f72 100644 --- a/deeppavlov/configs/squad/multi_squad_noans_infer.json +++ b/deeppavlov/configs/squad/multi_squad_noans_infer.json @@ -133,10 +133,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/multi_squad_model_noans_1.1.tar.gz", diff --git a/deeppavlov/configs/squad/multi_squad_retr_noans.json b/deeppavlov/configs/squad/multi_squad_retr_noans.json index c99c7b2e7a..e46842ece1 100644 --- a/deeppavlov/configs/squad/multi_squad_retr_noans.json +++ b/deeppavlov/configs/squad/multi_squad_retr_noans.json @@ -144,10 +144,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/multi_squad_model_noans_1.1.tar.gz", diff --git a/deeppavlov/configs/squad/multi_squad_ru_retr_noans.json b/deeppavlov/configs/squad/multi_squad_ru_retr_noans.json index 4f794a65c8..67e5602afd 100644 --- a/deeppavlov/configs/squad/multi_squad_ru_retr_noans.json +++ b/deeppavlov/configs/squad/multi_squad_ru_retr_noans.json @@ -144,10 +144,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/multi_squad_model_ru_1.0.tar.gz", diff --git a/deeppavlov/configs/squad/multi_squad_ru_retr_noans_rubert.json b/deeppavlov/configs/squad/multi_squad_ru_retr_noans_rubert.json index 2f62410b3a..9009c78f6e 100644 --- a/deeppavlov/configs/squad/multi_squad_ru_retr_noans_rubert.json +++ b/deeppavlov/configs/squad/multi_squad_ru_retr_noans_rubert.json @@ -95,10 +95,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/rubert_cased_L-12_H-768_A-12_v1.tar.gz", diff --git a/deeppavlov/configs/squad/multi_squad_ru_retr_noans_rubert_infer.json b/deeppavlov/configs/squad/multi_squad_ru_retr_noans_rubert_infer.json index 702cc0fde2..9d25378746 100644 --- a/deeppavlov/configs/squad/multi_squad_ru_retr_noans_rubert_infer.json +++ b/deeppavlov/configs/squad/multi_squad_ru_retr_noans_rubert_infer.json @@ -59,10 +59,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/rubert_cased_L-12_H-768_A-12_v1.tar.gz", diff --git a/deeppavlov/configs/squad/squad.json b/deeppavlov/configs/squad/squad.json index dd186ece6c..acf66ffc4e 100644 --- a/deeppavlov/configs/squad/squad.json +++ b/deeppavlov/configs/squad/squad.json @@ -123,10 +123,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/squad_model_1.4_cpu_compatible.tar.gz", diff --git a/deeppavlov/configs/squad/squad_bert.json b/deeppavlov/configs/squad/squad_bert.json index bb8dfe196c..4cd960d990 100644 --- a/deeppavlov/configs/squad/squad_bert.json +++ b/deeppavlov/configs/squad/squad_bert.json @@ -94,10 +94,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/cased_L-12_H-768_A-12.zip", diff --git a/deeppavlov/configs/squad/squad_bert_infer.json b/deeppavlov/configs/squad/squad_bert_infer.json index 797151f993..5f4a1920f3 100644 --- a/deeppavlov/configs/squad/squad_bert_infer.json +++ b/deeppavlov/configs/squad/squad_bert_infer.json @@ -65,10 +65,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [{ "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/cased_L-12_H-768_A-12.zip", "subdir": "{DOWNLOADS_PATH}/bert_models" diff --git a/deeppavlov/configs/squad/squad_bert_multilingual_freezed_emb.json b/deeppavlov/configs/squad/squad_bert_multilingual_freezed_emb.json index 29a52beaae..5a73bdd4e1 100644 --- a/deeppavlov/configs/squad/squad_bert_multilingual_freezed_emb.json +++ b/deeppavlov/configs/squad/squad_bert_multilingual_freezed_emb.json @@ -55,10 +55,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/multi_cased_L-12_H-768_A-12.zip", diff --git a/deeppavlov/configs/squad/squad_bert_uncased.json b/deeppavlov/configs/squad/squad_bert_uncased.json index 6b205f8c3f..ff09ca809a 100644 --- a/deeppavlov/configs/squad/squad_bert_uncased.json +++ b/deeppavlov/configs/squad/squad_bert_uncased.json @@ -96,10 +96,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/uncased_L-12_H-768_A-12.zip", diff --git a/deeppavlov/configs/squad/squad_ru.json b/deeppavlov/configs/squad/squad_ru.json index 6961b7de82..28c1aac323 100644 --- a/deeppavlov/configs/squad/squad_ru.json +++ b/deeppavlov/configs/squad/squad_ru.json @@ -124,10 +124,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/squad_model_ru_1.4_cpu_compatible.tar.gz", diff --git a/deeppavlov/configs/squad/squad_ru_bert.json b/deeppavlov/configs/squad/squad_ru_bert.json index 91db1ebcd6..cdb55121a4 100644 --- a/deeppavlov/configs/squad/squad_ru_bert.json +++ b/deeppavlov/configs/squad/squad_ru_bert.json @@ -97,10 +97,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/multi_cased_L-12_H-768_A-12.zip", diff --git a/deeppavlov/configs/squad/squad_ru_bert_infer.json b/deeppavlov/configs/squad/squad_ru_bert_infer.json index aa10a3ff26..a6b69cff58 100644 --- a/deeppavlov/configs/squad/squad_ru_bert_infer.json +++ b/deeppavlov/configs/squad/squad_ru_bert_infer.json @@ -67,10 +67,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/multi_cased_L-12_H-768_A-12.zip", diff --git a/deeppavlov/configs/squad/squad_ru_rubert.json b/deeppavlov/configs/squad/squad_ru_rubert.json index 1290998388..3dd1de2d93 100644 --- a/deeppavlov/configs/squad/squad_ru_rubert.json +++ b/deeppavlov/configs/squad/squad_ru_rubert.json @@ -97,10 +97,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/rubert_cased_L-12_H-768_A-12_v1.tar.gz", diff --git a/deeppavlov/configs/squad/squad_ru_rubert_infer.json b/deeppavlov/configs/squad/squad_ru_rubert_infer.json index 76b89fb927..2999038b64 100644 --- a/deeppavlov/configs/squad/squad_ru_rubert_infer.json +++ b/deeppavlov/configs/squad/squad_ru_rubert_infer.json @@ -67,10 +67,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/rubert_cased_L-12_H-768_A-12_v1.tar.gz", diff --git a/deeppavlov/configs/squad/squad_zh_bert.json b/deeppavlov/configs/squad/squad_zh_bert.json index 9fc95aa8b5..5349a69532 100644 --- a/deeppavlov/configs/squad/squad_zh_bert.json +++ b/deeppavlov/configs/squad/squad_zh_bert.json @@ -97,10 +97,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt" ], - "labels": { - "telegram_utils": "SquadModel", - "server_utils": "SquadModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/multi_cased_L-12_H-768_A-12.zip", diff --git a/deeppavlov/core/commands/train.py b/deeppavlov/core/commands/train.py index 7cfb675727..c3466fc403 100644 --- a/deeppavlov/core/commands/train.py +++ b/deeppavlov/core/commands/train.py @@ -131,8 +131,8 @@ def train_evaluate_model_from_config(config: Union[str, Path, dict], if to_validate: evaluation_targets.append('valid') else: - log.warn('Both "evaluation_targets" and "to_validate" parameters are specified.' - ' "to_validate" is deprecated and will be ignored') + log.warning('Both "evaluation_targets" and "to_validate" parameters are specified.' + ' "to_validate" is deprecated and will be ignored') res = trainer.evaluate(iterator, evaluation_targets, print_reports=True) trainer.get_chainer().destroy() diff --git a/deeppavlov/core/common/metrics_registry.py b/deeppavlov/core/common/metrics_registry.py index 28bf3a6e1b..55d7d34f9d 100644 --- a/deeppavlov/core/common/metrics_registry.py +++ b/deeppavlov/core/common/metrics_registry.py @@ -1,3 +1,17 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import importlib import json from logging import getLogger diff --git a/deeppavlov/core/trainers/nn_trainer.py b/deeppavlov/core/trainers/nn_trainer.py index f70509598c..7dcdeaf881 100644 --- a/deeppavlov/core/trainers/nn_trainer.py +++ b/deeppavlov/core/trainers/nn_trainer.py @@ -337,7 +337,7 @@ def train(self, iterator: DataLearningIterator) -> None: except KeyboardInterrupt: log.info('Stopped training') else: - log.warn(f'Using {self.__class__.__name__} for a pipeline without batched training') + log.warning(f'Using {self.__class__.__name__} for a pipeline without batched training') # Run the at-train-exit model-saving logic if self.validation_number < 1: diff --git a/deeppavlov/dataset_readers/kvret_reader.py b/deeppavlov/dataset_readers/kvret_reader.py index 275a5f0f5d..c218dc4298 100644 --- a/deeppavlov/dataset_readers/kvret_reader.py +++ b/deeppavlov/dataset_readers/kvret_reader.py @@ -143,8 +143,8 @@ def _iter_file(cls, file_path): if cls._check_dialog(dialog): yield dialog, sample['scenario'] else: - log.warn("Skipping {}th dialogue with uuid={}: wrong format." \ - .format(i, sample['scenario']['uuid'])) + log.warning("Skipping {}th dialogue with uuid={}: wrong format." \ + .format(i, sample['scenario']['uuid'])) @staticmethod def _get_turns(data, with_indices=False): diff --git a/deeppavlov/deep.py b/deeppavlov/deep.py index 27f5912987..29e6cca4a7 100644 --- a/deeppavlov/deep.py +++ b/deeppavlov/deep.py @@ -33,8 +33,8 @@ parser = argparse.ArgumentParser() parser.add_argument("mode", help="select a mode, train or interact", type=str, - choices={'train', 'evaluate', 'interact', 'predict', 'interactbot', 'interactmsbot', - 'alexa', 'alice', 'riseapi', 'risesocket', 'download', 'install', 'crossval'}) + choices={'train', 'evaluate', 'interact', 'predict', 'telegram', 'msbot', 'alexa', 'alice', + 'riseapi', 'risesocket', 'download', 'install', 'crossval'}) parser.add_argument("config_path", help="path to a pipeline json config", type=str) parser.add_argument("-e", "--start-epoch-num", dest="start_epoch_num", default=None, @@ -77,9 +77,9 @@ def main(): train_evaluate_model_from_config(pipeline_config_path, to_train=False, start_epoch_num=args.start_epoch_num) elif args.mode == 'interact': interact_model(pipeline_config_path) - elif args.mode == 'interactbot': + elif args.mode == 'telegram': interact_model_by_telegram(model_config=pipeline_config_path, token=args.token) - elif args.mode == 'interactmsbot': + elif args.mode == 'msbot': start_ms_bf_server(model_config=pipeline_config_path, app_id=args.ms_id, app_secret=args.ms_secret, diff --git a/deeppavlov/models/api_requester/api_requester.py b/deeppavlov/models/api_requester/api_requester.py index df87db4523..ee1e9217af 100644 --- a/deeppavlov/models/api_requester/api_requester.py +++ b/deeppavlov/models/api_requester/api_requester.py @@ -1,3 +1,17 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import asyncio from typing import Any, List, Dict, AsyncIterable diff --git a/deeppavlov/models/api_requester/api_router.py b/deeppavlov/models/api_requester/api_router.py index 71f636cd2b..75f4a368b6 100644 --- a/deeppavlov/models/api_requester/api_router.py +++ b/deeppavlov/models/api_requester/api_router.py @@ -1,3 +1,17 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import concurrent from concurrent.futures import ProcessPoolExecutor from logging import getLogger diff --git a/deeppavlov/models/classifiers/ru_obscenity_classifier.py b/deeppavlov/models/classifiers/ru_obscenity_classifier.py index 2c3ef26d36..6c17ae2ae8 100644 --- a/deeppavlov/models/classifiers/ru_obscenity_classifier.py +++ b/deeppavlov/models/classifiers/ru_obscenity_classifier.py @@ -1,3 +1,17 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import re from logging import getLogger diff --git a/deeppavlov/models/go_bot/network.py b/deeppavlov/models/go_bot/network.py index 1401aaeae6..d5774519c3 100644 --- a/deeppavlov/models/go_bot/network.py +++ b/deeppavlov/models/go_bot/network.py @@ -408,7 +408,7 @@ def prepare_data(self, x: List[dict], y: List[dict]) -> List[np.ndarray]: if self.debug: log.debug(f"True response = '{response['text']}'.") if d_a_masks[-1][action_id] != 1.: - log.warn("True action forbidden by action mask.") + log.warning("True action forbidden by action mask.") # padding to max_num_utter num_padds = max_num_utter - len(d_contexts) @@ -481,7 +481,7 @@ def make_api_call(self, state: dict) -> dict: if len(db_results) > 1: db_results = [r for r in db_results if r != state['db_result']] else: - log.warn("No database specified.") + log.warning("No database specified.") log.info(f"Made api_call with {slots}, got {len(db_results)} results.") return {} if not db_results else db_results[0] diff --git a/deeppavlov/models/kbqa/entity_linking.py b/deeppavlov/models/kbqa/entity_linking.py index 0fd9408aba..0e76f12a7b 100644 --- a/deeppavlov/models/kbqa/entity_linking.py +++ b/deeppavlov/models/kbqa/entity_linking.py @@ -39,7 +39,7 @@ class EntityLinker(Component, Serializable): Levenstein distance between the entity and keys (titles) in the dictionary. """ - LANGUAGES = set(['rus']) + LANGUAGES = {'rus'} def __init__(self, load_path: str, wiki_filename: str, entities_filename: str, inverted_index_filename: str, id_to_name_file: str, lemmatize: bool = True, debug: bool = False, rule_filter_entities: bool = True, diff --git a/deeppavlov/models/kbqa/kb_answer_parser_wikidata.py b/deeppavlov/models/kbqa/kb_answer_parser_wikidata.py index e866b8b501..946ab32eb2 100644 --- a/deeppavlov/models/kbqa/kb_answer_parser_wikidata.py +++ b/deeppavlov/models/kbqa/kb_answer_parser_wikidata.py @@ -200,9 +200,9 @@ def entities_and_rels_from_templates(self, tokens: List[List[str]]) -> Tuple[str return ent, relation def is_kbqa_question(self, question_tokens: List[List[str]]) -> bool: - not_kbqa_question_templates = ["почему", "когда будет", "что будет", "что если", "для чего ", "как ", \ + not_kbqa_question_templates = ["почему", "когда будет", "что будет", "что если", "для чего ", "как ", "что делать", "зачем", "что может"] - kbqa_question_templates = ["как зовут", "как называется", "как звали", "как ты думаешь", "как твое мнение", \ + kbqa_question_templates = ["как зовут", "как называется", "как звали", "как ты думаешь", "как твое мнение", "как ты считаешь"] question_init = ' '.join(question_tokens) question = ''.join([ch for ch in question_init if ch not in punctuation]).lower() diff --git a/deeppavlov/models/ranking/tf_base_matching_model.py b/deeppavlov/models/ranking/tf_base_matching_model.py index 8255777143..d7aa420856 100644 --- a/deeppavlov/models/ranking/tf_base_matching_model.py +++ b/deeppavlov/models/ranking/tf_base_matching_model.py @@ -1,18 +1,16 @@ -""" -Copyright 2018 Neural Networks and Deep Learning lab, MIPT - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from logging import getLogger from typing import List, Dict, Tuple diff --git a/deeppavlov/models/seq2seq_go_bot/dialog_state.py b/deeppavlov/models/seq2seq_go_bot/dialog_state.py index 48e1f6b4c7..a316250ea4 100644 --- a/deeppavlov/models/seq2seq_go_bot/dialog_state.py +++ b/deeppavlov/models/seq2seq_go_bot/dialog_state.py @@ -1,18 +1,16 @@ -""" -Copyright 2017 Neural Networks and Deep Learning lab, MIPT - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from deeppavlov.core.common.registry import register from deeppavlov.core.models.component import Component diff --git a/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py b/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py index 3f9c6be5d1..4b362c24f7 100644 --- a/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py +++ b/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py @@ -1,18 +1,16 @@ -""" -Copyright 2017 Neural Networks and Deep Learning lab, MIPT - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import tensorflow as tf from tensorflow.python.framework import tensor_shape diff --git a/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py b/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py index 887ac927f0..3c05d6848c 100644 --- a/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py +++ b/deeppavlov/models/spelling_correction/levenshtein/levenshtein_searcher.py @@ -1,3 +1,17 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import copy import itertools diff --git a/deeppavlov/models/spelling_correction/levenshtein/tabled_trie.py b/deeppavlov/models/spelling_correction/levenshtein/tabled_trie.py index 5aace19731..cb24d12245 100644 --- a/deeppavlov/models/spelling_correction/levenshtein/tabled_trie.py +++ b/deeppavlov/models/spelling_correction/levenshtein/tabled_trie.py @@ -1,3 +1,17 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import copy from collections import defaultdict diff --git a/deeppavlov/skills/aiml_skill/aiml_skill.py b/deeppavlov/skills/aiml_skill/aiml_skill.py index 72c69113ca..51bf6f2360 100644 --- a/deeppavlov/skills/aiml_skill/aiml_skill.py +++ b/deeppavlov/skills/aiml_skill/aiml_skill.py @@ -13,8 +13,6 @@ # limitations under the License. import uuid -from pathlib import Path -from typing import Tuple, Optional, List from logging import getLogger from pathlib import Path from typing import Tuple, Optional, List diff --git a/deeppavlov/utils/alexa/server.py b/deeppavlov/utils/alexa/server.py index 70a60851a3..18805c26ea 100644 --- a/deeppavlov/utils/alexa/server.py +++ b/deeppavlov/utils/alexa/server.py @@ -34,7 +34,7 @@ def start_alexa_server(model_config: Union[str, Path, dict], port: Optional[int] = None, - https: bool = False, + https: Optional[bool] = None, ssl_key: Optional[str] = None, ssl_cert: Optional[str] = None) -> None: """Initiates FastAPI web service with Alexa skill. diff --git a/deeppavlov/utils/connector/bot.py b/deeppavlov/utils/connector/bot.py index eec9ca8429..9eb1e4eece 100644 --- a/deeppavlov/utils/connector/bot.py +++ b/deeppavlov/utils/connector/bot.py @@ -34,7 +34,7 @@ from deeppavlov.utils.connector.conversation import MSConversation, TelegramConversation from deeppavlov.utils.connector.ssl_tools import verify_cert, verify_signature -CONNECTOR_CONFIG_FILENAME = 'connector_config.json' +CONNECTOR_CONFIG_FILENAME = 'server_config.json' INPUT_QUEUE_TIMEOUT = 1 log = getLogger(__name__) @@ -139,14 +139,8 @@ def _get_connector_params(self) -> dict: connector_config: dict = read_json(connector_config_path) bot_name = type(self).__name__ - bot_defaults = connector_config['bot_defaults'].get(bot_name, {}) - - model_name = type(self._model.get_main_component()).__name__ - models_info = connector_config['models_info'] - model_msgs = models_info.get(model_name, models_info['@default']) - conversation_defaults = connector_config['conversation_defaults'] - conversation_defaults['start_message'] = model_msgs['start_message'] - conversation_defaults['help_message'] = model_msgs['help_message'] + conversation_defaults = connector_config['telegram'] + bot_defaults = connector_config['deprecated'].get(bot_name, conversation_defaults) connector_defaults = {'bot_defaults': bot_defaults, 'conversation_defaults': conversation_defaults} diff --git a/deeppavlov/utils/connector/conversation.py b/deeppavlov/utils/connector/conversation.py index 09129ed70c..ecc6b179dd 100644 --- a/deeppavlov/utils/connector/conversation.py +++ b/deeppavlov/utils/connector/conversation.py @@ -41,7 +41,7 @@ class BaseConversation: _timer: Timer _infer_utterances: list _conversation_lifetime: int - _next_utter_msg: str + _next_arg_msg: str _start_message: str def __init__(self, @@ -63,9 +63,9 @@ def __init__(self, self._conversation_id = conversation_id self._infer_utterances = list() self._conversation_lifetime = config['conversation_lifetime'] - self._next_utter_msg = config['next_utter_msg'] + self._next_arg_msg = config['next_argument_message'] self._start_message = config['start_message'] - self._unsupported_message = config['unsupported_msg'] + self._unsupported_message = config['unsupported_message'] logger_name: str = DIALOG_LOGGER_NAME_MAPPING.get(type(self).__name__, DIALOG_LOGGER_NAME_MAPPING['_unsupported']) self._dialog_logger = DialogLogger(logger_name=logger_name) @@ -177,7 +177,7 @@ def _act(self, utterance: str) -> str: response = prediction self._dialog_logger.log_out(response, self._conversation_id) else: - response = self._next_utter_msg.format(self._model.in_x[len(self._infer_utterances)]) + response = self._next_arg_msg.format(self._model.in_x[len(self._infer_utterances)]) return response diff --git a/deeppavlov/utils/ms_bot_framework/server.py b/deeppavlov/utils/ms_bot_framework/server.py index c79e14f60d..0b4d4fd868 100644 --- a/deeppavlov/utils/ms_bot_framework/server.py +++ b/deeppavlov/utils/ms_bot_framework/server.py @@ -32,7 +32,7 @@ def start_ms_bf_server(model_config: Path, app_id: Optional[str], app_secret: Optional[str], port: Optional[int] = None, - https: bool = False, + https: Optional[bool] = None, ssl_key: Optional[str] = None, ssl_cert: Optional[str] = None) -> None: diff --git a/deeppavlov/utils/pip_wrapper/pip_wrapper.py b/deeppavlov/utils/pip_wrapper/pip_wrapper.py index a829af0cc6..1f550c7b88 100644 --- a/deeppavlov/utils/pip_wrapper/pip_wrapper.py +++ b/deeppavlov/utils/pip_wrapper/pip_wrapper.py @@ -1,3 +1,17 @@ +# Copyright 2017 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import re import subprocess @@ -16,7 +30,7 @@ def install(*packages): if any(_tf_re.match(package) for package in packages) \ and b'tensorflow-gpu' in subprocess.check_output([sys.executable, '-m', 'pip', 'freeze'], env=os.environ.copy()): - log.warn('found tensorflow-gpu installed, so upgrading it instead of tensorflow') + log.warning('found tensorflow-gpu installed, so upgrading it instead of tensorflow') packages = [_tf_re.sub(r'tensorflow-gpu\1', package) for package in packages] result = subprocess.check_call([sys.executable, '-m', 'pip', 'install', *[re.sub(r'\s', '', package) for package in packages]], @@ -29,7 +43,7 @@ def install_from_config(config: [str, Path, dict]): requirements_files = config.get('metadata', {}).get('requirements', []) if not requirements_files: - log.warn('No requirements found in config') + log.warning('No requirements found in config') return requirements = [] diff --git a/deeppavlov/utils/server/server.py b/deeppavlov/utils/server/server.py index c47710c0b2..f05655ddc1 100644 --- a/deeppavlov/utils/server/server.py +++ b/deeppavlov/utils/server/server.py @@ -64,15 +64,15 @@ def filter(self, record: logging.LogRecord) -> bool: dialog_logger = DialogLogger(logger_name='rest_api') -def get_server_params(model_config: Union[str, Path], server_config_path: Path = SERVER_CONFIG_PATH) -> Dict: - server_config = read_json(server_config_path) +def get_server_params(model_config: Union[str, Path]) -> Dict: + server_config = read_json(SERVER_CONFIG_PATH) model_config = parse_config(model_config) server_params = server_config['common_defaults'] - if check_nested_dict_keys(model_config, ['metadata', 'labels', 'server_utils']): - model_tag = model_config['metadata']['labels']['server_utils'] - if model_tag in server_config['model_defaults']: + if check_nested_dict_keys(model_config, ['metadata', 'server_utils']): + model_tag = model_config['metadata']['server_utils'] + if check_nested_dict_keys(server_config, ['model_defaults', model_tag]): model_defaults = server_config['model_defaults'][model_tag] for param_name in model_defaults.keys(): if model_defaults[param_name]: @@ -164,7 +164,7 @@ def test_interact(model: Chainer, payload: Dict[str, Optional[List]]) -> List[st def start_model_server(model_config: Path, - https: bool = False, + https: Optional[bool] = None, ssl_key: Optional[str] = None, ssl_cert: Optional[str] = None, port: Optional[int] = None) -> None: diff --git a/deeppavlov/utils/settings/connector_config.json b/deeppavlov/utils/settings/connector_config.json deleted file mode 100644 index 1779438538..0000000000 --- a/deeppavlov/utils/settings/connector_config.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "bot_defaults": { - "AlexaBot": { - "amazon_cert_lifetime_secs": 3600, - "request_timestamp_tolerance_secs": 150, - "refresh_valid_certs_period_secs": 120, - "intent_name": "AskDeepPavlov", - "slot_name": "raw_input" - }, - "MSBot": { - "auth_polling_interval": 3500, - "auth_url": "https://login.microsoftonline.com/botframework.com/oauth2/v2.0/token", - "auth_headers": { - "Host": "login.microsoftonline.com", - "Content-Type": "application/x-www-form-urlencoded" - }, - "auth_payload": { - "grant_type": "client_credentials", - "scope": "https://api.botframework.com/.default", - "client_id": "", - "client_secret": "" - } - }, - "TelegramBot": { - "token": "" - } - }, - "conversation_defaults": { - "conversation_lifetime": 3600, - "next_utter_msg": "Please enter an argument '{}'", - "unsupported_msg": "Unsupported message received." - }, - "models_info": { - "@default": { - "start_message": "Welcome to DeepPavlov inference bot!", - "help_message": "Welcome to DeepPavlov inference bot!" - }, - "DstcSlotFillingNetwork": { - "start_message": "Welcome to the UI of the DSTC slot filling model. You can enter a text string and get identified slots with values. Example:\n\nFrench cusine\n\n{'food': 'french'}", - "help_message": "Welcome to the UI of the DSTC slot filling model. You can enter a text string and get identified slots with values. Example:\n\nFrench cusine\n\n{'food': 'french'}" - }, - "ErrorModel": { - "start_message": "Welcome to the automatic spelling correction component. You can enter a phrase with typos and get it back corrected. Example:\n\nhelllo\n\nhello", - "help_message": "Welcome to the automatic spelling correction component. You can enter a phrase with typos and get it back corrected. Example:\n\nhelllo\n\nhello" - }, - "GoalOrientedBot": { - "start_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in a restaurant advisory domain. Enter any phrase to start interacting with model.", - "help_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in a restaurant advisory domain. Enter any phrase to start interacting with model." - }, - "Seq2SeqGoalOrientedBot": { - "start_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in three domains: calendar scheduling, weather information retrieval, and point-of-interest navigation. Enter any phrase to start interacting with model. Example:\n\nWhere's the nearest parking garage?\n\nthe nearest is dish parking at dish_parking_address. is that okay?\n\nYes, please set directions via a route that avoids all heavy traffic if possible.\n\nit looks like there is a road block being reported on the route but i will still find the quickest route to dish_parking_address.\n\nThanks so much for your help.\n\nyou're welcome. have a good day. end_of_dialogue", - "help_message": "Welcome to the UI of the Dialogue Manager for goal-oriented task. This model can handle a goal-oriented conversation in three domains: calendar scheduling, weather information retrieval, and point-of-interest navigation. Enter any phrase to start interacting with model. Example:\n\nWhere's the nearest parking garage?\n\nthe nearest is dish parking at dish_parking_address. is that okay?\n\nYes, please set directions via a route that avoids all heavy traffic if possible.\n\nit looks like there is a road block being reported on the route but i will still find the quickest route to dish_parking_address.\n\nThanks so much for your help.\n\nyou're welcome. have a good day. end_of_dialogue" - }, - "KerasIntentModel": { - "start_message": "Welcome to the UI for the user intent classification model. You can enter enter a text string and get intents (classes which a request belongs with).", - "help_message": "Welcome to the UI for the user intent classification model. You can enter enter a text string and get intents (classes which a request belongs with)." - } - } -} \ No newline at end of file diff --git a/deeppavlov/utils/settings/server_config.json b/deeppavlov/utils/settings/server_config.json index a9f315182b..9949bf3a57 100644 --- a/deeppavlov/utils/settings/server_config.json +++ b/deeppavlov/utils/settings/server_config.json @@ -5,53 +5,40 @@ "model_args_names": "", "https": false, "https_cert_path": "", - "https_key_path": "" + "https_key_path": "", + "socket_type": "TCP", + "unix_socket_file": "/tmp/deeppavlov_socket.s", + "socket_launch_message": "launching socket server at" }, - "model_defaults": { - "DstcSlotFillingNetwork": { - "host": "", - "port": "", - "model_args_names": "" - }, - "EcommerceSkill": { - "host": "", - "port": "", - "model_args_names": "" - }, - "ErrorModel": { - "host": "", - "port": "", - "model_args_names": "" - }, - "GoalOrientedBot": { - "host": "", - "port": "", - "model_args_names": "" - }, - "KerasIntentModel": { - "host": "", - "port": "", - "model_args_names": "" - }, - "NER": { - "host": "", - "port": "", - "model_args_names": "" - }, - "SquadModel": { - "host": "", - "port": "", - "model_args_names": "" - }, - "ODQA": { - "host": "", - "port": "", - "model_args_names": "" - }, - "Ranker": { - "host": "", - "port": "", - "model_args_names": "" + "telegram": { + "token": "", + "conversation_lifetime": 3600, + "start_message": "Welcome to DeepPavlov inference bot!", + "help_message": "Welcome to DeepPavlov inference bot!", + "next_argument_message": "Please enter an argument '{}'", + "unsupported_message": "Unsupported message received." + }, + "deprecated": { + "AlexaBot": { + "amazon_cert_lifetime_secs": 3600, + "request_timestamp_tolerance_secs": 150, + "refresh_valid_certs_period_secs": 120, + "intent_name": "AskDeepPavlov", + "slot_name": "raw_input" + }, + "MSBot": { + "auth_polling_interval": 3500, + "auth_url": "https://login.microsoftonline.com/botframework.com/oauth2/v2.0/token", + "auth_headers": { + "Host": "login.microsoftonline.com", + "Content-Type": "application/x-www-form-urlencoded" + }, + "auth_payload": { + "grant_type": "client_credentials", + "scope": "https://api.botframework.com/.default", + "client_id": "", + "client_secret": "" + } } } } diff --git a/deeppavlov/utils/settings/socket_config.json b/deeppavlov/utils/settings/socket_config.json deleted file mode 100644 index 67d45cfccc..0000000000 --- a/deeppavlov/utils/settings/socket_config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "common_defaults": { - "host": "0.0.0.0", - "port": 5001, - "unix_socket_file": "/tmp/deeppavlov_socket.s", - "socket_type": "TCP", - "model_args_names": "", - "launch_message": "launching socket server at" - }, - "model_defaults": { - "SquadModel": {} - } -} \ No newline at end of file diff --git a/deeppavlov/utils/socket/__init__.py b/deeppavlov/utils/socket/__init__.py index f222759a23..aa1ab4a91a 100644 --- a/deeppavlov/utils/socket/__init__.py +++ b/deeppavlov/utils/socket/__init__.py @@ -1 +1 @@ -from .socket import encode, start_socket_server, SOCKET_CONFIG_FILENAME +from .socket import encode, start_socket_server diff --git a/deeppavlov/utils/socket/socket.py b/deeppavlov/utils/socket/socket.py index 61dc0298c9..ca0249e501 100644 --- a/deeppavlov/utils/socket/socket.py +++ b/deeppavlov/utils/socket/socket.py @@ -21,12 +21,10 @@ from deeppavlov.core.commands.infer import build_model from deeppavlov.core.common.chainer import Chainer -from deeppavlov.core.common.paths import get_settings_path from deeppavlov.core.data.utils import jsonify_data from deeppavlov.utils.connector import DialogLogger from deeppavlov.utils.server import get_server_params -SOCKET_CONFIG_FILENAME = 'socket_config.json' HEADER_FORMAT = ' None: +def interact_model_by_telegram(model_config: Union[str, Path, dict], token: Optional[str] = None) -> None: bot = TelegramBot(model_config, token) bot.start() diff --git a/deeppavlov/vocabs/wiki_sqlite.py b/deeppavlov/vocabs/wiki_sqlite.py index 9e9b79a44a..0856bacfa0 100644 --- a/deeppavlov/vocabs/wiki_sqlite.py +++ b/deeppavlov/vocabs/wiki_sqlite.py @@ -51,7 +51,7 @@ def __call__(self, doc_ids: Optional[List[List[Any]]] = None, *args, **kwargs) - """ all_contents = [] if not doc_ids: - logger.warn('No doc_ids are provided in WikiSqliteVocab, return all docs') + logger.warning('No doc_ids are provided in WikiSqliteVocab, return all docs') doc_ids = [self.get_doc_ids()] for ids in doc_ids: diff --git a/docs/features/models/classifiers.rst b/docs/features/models/classifiers.rst index 6c6f4e5ab4..513ee2e44a 100644 --- a/docs/features/models/classifiers.rst +++ b/docs/features/models/classifiers.rst @@ -29,7 +29,7 @@ or its name without an extension, for example :config:`"intents_snips" `` + ``python -m deeppavlov telegram deeppavlov/configs/go_bot/gobot_dstc2.json -d -t `` - Run goal-oriented bot with console interface: ``python -m deeppavlov interact deeppavlov/configs/go_bot/gobot_dstc2.json -d`` @@ -555,7 +555,7 @@ Examples of some models ``python -m deeppavlov riseapi deeppavlov/configs/go_bot/gobot_dstc2.json -d`` - Run slot-filling model with Telegram interface: - ``python -m deeppavlov interactbot deeppavlov/configs/ner/slotfill_dstc2.json -d -t `` + ``python -m deeppavlov telegram deeppavlov/configs/ner/slotfill_dstc2.json -d -t `` - Run slot-filling model with console interface: ``python -m deeppavlov interact deeppavlov/configs/ner/slotfill_dstc2.json -d`` diff --git a/docs/integrations/amazon_alexa.rst b/docs/integrations/amazon_alexa.rst index 10dabe3076..5cf3f2d034 100644 --- a/docs/integrations/amazon_alexa.rst +++ b/docs/integrations/amazon_alexa.rst @@ -186,7 +186,8 @@ DeepPavlov skill/model can be made available for Amazon Alexa as a REST service .. code:: bash - python -m deeppavlov alexa --https --key --cert [-d] [-p ] + python -m deeppavlov alexa [--https] [--key ] \ + [--cert ] [-d] [-p ] If you redirect requests to your skills service from some https endpoint, you may want to run it in http mode by omitting ``--https``, ``--key``, ``--cert`` keys. diff --git a/docs/integrations/ms_bot.rst b/docs/integrations/ms_bot.rst index 7ec34ad366..e27d603803 100644 --- a/docs/integrations/ms_bot.rst +++ b/docs/integrations/ms_bot.rst @@ -83,7 +83,8 @@ as a REST service by: .. code:: bash - python -m deeppavlov interactmsbot -i -s --https --key --cert [-d] [-p ] + python -m deeppavlov msbot [-i ] [-s ] \ + [--https] [--key ] [--cert ] [-d] [-p ] Use *Microsoft App ID* and *Microsoft App Secret* obtained in the **Web App Bot connection configuration** section. diff --git a/docs/integrations/rest_api.rst b/docs/integrations/rest_api.rst index 66f6dba582..1a16e62c68 100644 --- a/docs/integrations/rest_api.rst +++ b/docs/integrations/rest_api.rst @@ -6,12 +6,19 @@ inference as a REST web service. The general method is: .. code:: bash - python -m deeppavlov riseapi [-d] [-p ] + python -m deeppavlov riseapi [-d] [-p ] [--https] [--key ] \ + [--cert ] * ``-d``: downloads model specific data before starting the service. * ``-p ``: sets the port to ````. Overrides default - settings from ``deeppavlov/utils/settings/server_config.json``. + value from ``deeppavlov/utils/settings/server_config.json``. +* ``--https``: use https instead of http. Overrides default + value from ``deeppavlov/utils/settings/server_config.json``. +* ``--key ``: path to SSL key file. Overrides default + value from ``deeppavlov/utils/settings/server_config.json``. +* ``--cert ``: path to SSL certificate file. Overrides default + value from ``deeppavlov/utils/settings/server_config.json``. The command will print the used host and port. Default web service properties (host, port, POST request arguments) can be modified via changing @@ -52,19 +59,18 @@ By modifying ``deeppavlov/utils/settings/server_config.json`` you can change host, port, POST request arguments and other properties of the API service. Properties from ``common_defaults`` section are used by default unless -they are overridden by model-specific properties, provided in -``model_defaults`` section of the ``server_config.json``. -Model-specific properties are bound to the model by -``server_utils`` label in ``metadata/labels`` section of the model -config. Value of ``server_utils`` label from model config should -match with properties key from ``model_defaults`` section of -``server_config.json``. - -For example, ``metadata/labels/server_utils`` tag from -``go_bot/gobot_dstc2.json`` references to the *GoalOrientedBot* section -of ``server_config.json``. Therefore, all parameters with non empty (i.e. not -``""``, not ``[]`` etc.) values from ``model_defaults/GoalOrientedBot`` will -overwrite the parameter values in ``common_defaults``. +they are overridden by model-specific properties, provided in ``model_defaults`` +section of the ``server_config.json``. Model-specific properties are bound +to the model by ``server_utils`` label in ``metadata`` section of the model +config. Value of ``server_utils`` label from model config should match with +properties key from ``model_defaults`` section of ``server_config.json``. + +For example, adding ``metadata/server_utils`` key to ``go_bot/gobot_dstc2.json`` +with value *GoalOrientedBot* will initiate the search of *GoalOrientedBot* tag +at ``model_defaults`` section of ``server_config.json``. Therefore, if this +section is present, all parameters with non empty (i.e. not ``""``, +not ``[]`` etc.) values stored by this tag will overwrite the parameter values +in ``common_defaults``. If ``model_args_names`` parameter of ``server_config.json`` is empty string, then model argument names are provided as list from ``chainer/in`` section of diff --git a/docs/integrations/socket_api.rst b/docs/integrations/socket_api.rst index 1c4f8dcc63..48214a6196 100644 --- a/docs/integrations/socket_api.rst +++ b/docs/integrations/socket_api.rst @@ -13,40 +13,41 @@ method is: * ``-d``: downloads model specific data before starting the service. * ``--socket-type ``: sets socket address family to ``AF_INET`` if ```` is ``TCP`` or to ``AF_UNIX`` if ```` - is ``UNIX``. Overrides default settings from - ``deeppavlov/utils/settings/socket_config.json``. + is ``UNIX``. Overrides default value from + ``deeppavlov/utils/settings/server_config.json``. * ``-p ``: sets the port to ```` if socket address family is - ``AF_INET``. Overrides default settings from - ``deeppavlov/utils/settings/socket_config.json``. + ``AF_INET``. Overrides default value from + ``deeppavlov/utils/settings/server_config.json``. * ``--socket-file ``: sets the file for socket binding to ```` if socket address family is ``AF_UNIX``. Overrides - default settings from ``deeppavlov/utils/settings/socket_config.json``. + default value from ``deeppavlov/utils/settings/server_config.json``. The command will print the binding address: host and port for ``AF_INET`` socket family and path to the UNIX socket file for ``AF_UNIX`` socket family. Default service properties (socket address family, host, port, path to the UNIX socket file, socket buffer size, binding message) can be modified via changing -``deeppavlov/utils/settings/socket_config.json`` file. +``deeppavlov/utils/settings/server_config.json`` file. Advanced configuration ~~~~~~~~~~~~~~~~~~~~~~ -By modifying ``deeppavlov/utils/settings/socket_config.json`` you can change +By modifying ``deeppavlov/utils/settings/server_config.json`` you can change socket address family, host, port, path to the UNIX socket file and other properties of the API service. -Properties from ``common_defaults`` section are used by default unless they are -overridden by model-specific properties, provided in ``model_defaults`` section -of the ``socket_config.json``. Model-specific properties are bound to the model -by ``server_utils`` label in ``metadata/labels`` section of the model config. -Value of ``server_utils`` label from model config should match with properties -key from ``model_defaults`` section of ``socket_config.json``. - -For example, ``metadata/labels/server_utils`` tag from -``deeppavlov/configs/squad/squad.json`` references to the *SquadModel* section -of ``socket_config.json``. Therefore, all parameters with non empty (i.e. not -``""``, not ``[]`` etc.) values from ``model_defaults/SquadModel`` will -overwrite the parameter values in ``common_defaults``. +Properties from ``common_defaults`` section are used by default unless +they are overridden by model-specific properties, provided in ``model_defaults`` +section of the ``server_config.json``. Model-specific properties are bound +to the model by ``server_utils`` label in ``metadata`` section of the model +config. Value of ``server_utils`` label from model config should match with +properties key from ``model_defaults`` section of ``server_config.json``. + +For example, adding ``metadata/server_utils`` key to ``go_bot/gobot_dstc2.json`` +with value *GoalOrientedBot* will initiate the search of *GoalOrientedBot* tag +at ``model_defaults`` section of ``server_config.json``. Therefore, if this +section is present, all parameters with non empty (i.e. not ``""``, +not ``[]`` etc.) values stored by this tag will overwrite the parameter values +in ``common_defaults``. If ``model_args_names`` parameter of ``server_config.json`` is empty string, then model argument names are provided as list from ``chainer/in`` section of diff --git a/docs/integrations/telegram.rst b/docs/integrations/telegram.rst index bdf171c56e..561cb94891 100644 --- a/docs/integrations/telegram.rst +++ b/docs/integrations/telegram.rst @@ -13,28 +13,23 @@ with a ````: .. code:: bash - python -m deeppavlov interactbot -t [-d] + python -m deeppavlov telegram [-t ] [-d] -* ``-t ``: specifies telegram token as ````. +* ``-t ``: specifies telegram token as ````. Overrides + default value from ``deeppavlov/utils/settings/server_config.json``. * ``-d``: downloads model specific data before starting the service. -The command will print the used host and port. Default web service properties -(host, port, model endpoint, GET request arguments) can be modified via changing -``deeppavlov/utils/settings/server_config.json`` file. Advanced API -configuration is described in :doc:`REST API ` section. +The command will print info message ``Bot initiated`` when starts bot. -If you want to get custom ``/start`` and ``/help`` Telegram messages for the running model you should: - -* Add section to ``models_info`` section of ``deeppavlov/utils/settings/connector_config.json`` with your custom - Telegram messages -* In model config file specify ``metadata.labels.telegram_utils`` parameter with name which - refers to the added section of ``deeppavlov/utils/settings/connector_config.json`` ``models_info`` section. +``/start`` and ``/help`` Telegram bot messages can be modified via changing +``telegram.start_message`` and ``telegram.help_message`` +in `deeppavlov/utils/settings/server_config.json`. Python ~~~~~~ -To run a model specified by a DeepPavlov config ```` as as +To run a model specified by a DeepPavlov config ```` as Telegram bot, you have to run following code: .. code:: python diff --git a/docs/integrations/yandex_alice.rst b/docs/integrations/yandex_alice.rst index a74c3b5ba6..8a30bc6bdf 100644 --- a/docs/integrations/yandex_alice.rst +++ b/docs/integrations/yandex_alice.rst @@ -27,7 +27,7 @@ The command will print the used host and port. Default web service properties (host, port, model endpoint, GET request arguments, paths to ssl cert and key, https mode) can be modified via changing ``deeppavlov/utils/settings/server_config.json`` file. ``--https``, ``--key``, -``--cert``, ``-p`` arguments override values from ``server_config.json``. +``--cert``, ``-p`` arguments override default values from ``server_config.json``. Advanced API configuration is described in :doc:`REST API ` section. diff --git a/docs/intro/quick_start.rst b/docs/intro/quick_start.rst index e4b3b29aed..e6a7f2c6ee 100644 --- a/docs/intro/quick_start.rst +++ b/docs/intro/quick_start.rst @@ -71,9 +71,9 @@ There are even more actions you can perform with configs: `), * ``risesocket`` to run a socket API server (see :doc:`docs `), - * ``interactbot`` to run as a Telegram bot (see :doc:`docs + * ``telegram`` to run as a Telegram bot (see :doc:`docs `), - * ``interactmsbot`` to run a Miscrosoft Bot Framework server (see + * ``msbot`` to run a Miscrosoft Bot Framework server (see :doc:`docs `), * ``predict`` to get prediction for samples from `stdin` or from `` if ``-f `` is specified. diff --git a/tests/test_configs/classifiers/intents_snips_bigru.json b/tests/test_configs/classifiers/intents_snips_bigru.json index ff9e7ed430..6244f4eaaa 100644 --- a/tests/test_configs/classifiers/intents_snips_bigru.json +++ b/tests/test_configs/classifiers/intents_snips_bigru.json @@ -128,9 +128,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs.tar.gz", diff --git a/tests/test_configs/classifiers/intents_snips_bilstm.json b/tests/test_configs/classifiers/intents_snips_bilstm.json index 831be68378..58d937ea85 100644 --- a/tests/test_configs/classifiers/intents_snips_bilstm.json +++ b/tests/test_configs/classifiers/intents_snips_bilstm.json @@ -128,9 +128,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs.tar.gz", diff --git a/tests/test_configs/classifiers/intents_snips_bilstm_bilstm.json b/tests/test_configs/classifiers/intents_snips_bilstm_bilstm.json index ba4301ffba..cda73379be 100644 --- a/tests/test_configs/classifiers/intents_snips_bilstm_bilstm.json +++ b/tests/test_configs/classifiers/intents_snips_bilstm_bilstm.json @@ -129,9 +129,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs.tar.gz", diff --git a/tests/test_configs/classifiers/intents_snips_bilstm_cnn.json b/tests/test_configs/classifiers/intents_snips_bilstm_cnn.json index e2ac142a43..8fcbbd39a6 100644 --- a/tests/test_configs/classifiers/intents_snips_bilstm_cnn.json +++ b/tests/test_configs/classifiers/intents_snips_bilstm_cnn.json @@ -135,9 +135,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs.tar.gz", diff --git a/tests/test_configs/classifiers/intents_snips_bilstm_proj_layer.json b/tests/test_configs/classifiers/intents_snips_bilstm_proj_layer.json index 0c557350ab..2e5cb7b0bc 100644 --- a/tests/test_configs/classifiers/intents_snips_bilstm_proj_layer.json +++ b/tests/test_configs/classifiers/intents_snips_bilstm_proj_layer.json @@ -130,9 +130,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs.tar.gz", diff --git a/tests/test_configs/classifiers/intents_snips_bilstm_self_add_attention.json b/tests/test_configs/classifiers/intents_snips_bilstm_self_add_attention.json index 32311f4629..e6c01f518e 100644 --- a/tests/test_configs/classifiers/intents_snips_bilstm_self_add_attention.json +++ b/tests/test_configs/classifiers/intents_snips_bilstm_self_add_attention.json @@ -131,9 +131,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs.tar.gz", diff --git a/tests/test_configs/classifiers/intents_snips_bilstm_self_mult_attention.json b/tests/test_configs/classifiers/intents_snips_bilstm_self_mult_attention.json index 9d373c957c..8dc21569b0 100644 --- a/tests/test_configs/classifiers/intents_snips_bilstm_self_mult_attention.json +++ b/tests/test_configs/classifiers/intents_snips_bilstm_self_mult_attention.json @@ -131,9 +131,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs.tar.gz", diff --git a/tests/test_configs/classifiers/intents_snips_cnn_bilstm.json b/tests/test_configs/classifiers/intents_snips_cnn_bilstm.json index 30c6437277..79df581879 100644 --- a/tests/test_configs/classifiers/intents_snips_cnn_bilstm.json +++ b/tests/test_configs/classifiers/intents_snips_cnn_bilstm.json @@ -135,9 +135,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "IntentModel" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/vocabs.tar.gz", diff --git a/tests/test_configs/doc_retrieval/en_ranker_pop_wiki_test.json b/tests/test_configs/doc_retrieval/en_ranker_pop_wiki_test.json index 64f43fcab1..5003f86715 100644 --- a/tests/test_configs/doc_retrieval/en_ranker_pop_wiki_test.json +++ b/tests/test_configs/doc_retrieval/en_ranker_pop_wiki_test.json @@ -84,9 +84,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/wikipedia/enwiki_test.tar.gz", diff --git a/tests/test_configs/doc_retrieval/en_ranker_tfidf_wiki_test.json b/tests/test_configs/doc_retrieval/en_ranker_tfidf_wiki_test.json index 4eb95413ff..4822f362ed 100644 --- a/tests/test_configs/doc_retrieval/en_ranker_tfidf_wiki_test.json +++ b/tests/test_configs/doc_retrieval/en_ranker_tfidf_wiki_test.json @@ -70,9 +70,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/wikipedia/enwiki_test.tar.gz", diff --git a/tests/test_configs/doc_retrieval/ru_ranker_tfidf_wiki_test.json b/tests/test_configs/doc_retrieval/ru_ranker_tfidf_wiki_test.json index 02ffbb9ad4..d8aadb8ae0 100644 --- a/tests/test_configs/doc_retrieval/ru_ranker_tfidf_wiki_test.json +++ b/tests/test_configs/doc_retrieval/ru_ranker_tfidf_wiki_test.json @@ -67,9 +67,6 @@ "MODELS_PATH": "{ROOT_PATH}/models" }, "requirements": [], - "labels": { - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/wikipedia/ruwiki_test.tar.gz", diff --git a/tests/test_configs/odqa/en_odqa_infer_wiki_test.json b/tests/test_configs/odqa/en_odqa_infer_wiki_test.json index ddceee7b6b..323de1704e 100644 --- a/tests/test_configs/odqa/en_odqa_infer_wiki_test.json +++ b/tests/test_configs/odqa/en_odqa_infer_wiki_test.json @@ -73,9 +73,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "server_utils": "ODQA" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/odqa_test.tar.gz", diff --git a/tests/test_configs/odqa/en_odqa_pop_infer_wiki_test.json b/tests/test_configs/odqa/en_odqa_pop_infer_wiki_test.json index c7f2127e73..217ee11e49 100644 --- a/tests/test_configs/odqa/en_odqa_pop_infer_wiki_test.json +++ b/tests/test_configs/odqa/en_odqa_pop_infer_wiki_test.json @@ -78,9 +78,6 @@ "{DEEPPAVLOV_PATH}/requirements/spacy.txt", "{DEEPPAVLOV_PATH}/requirements/en_core_web_sm.txt" ], - "labels": { - "server_utils": "ODQA" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/odqa_test.tar.gz", diff --git a/tests/test_configs/odqa/ru_odqa_infer_wiki_test.json b/tests/test_configs/odqa/ru_odqa_infer_wiki_test.json index e5e4712195..8b78155e27 100644 --- a/tests/test_configs/odqa/ru_odqa_infer_wiki_test.json +++ b/tests/test_configs/odqa/ru_odqa_infer_wiki_test.json @@ -75,9 +75,6 @@ "requirements": [ "{DEEPPAVLOV_PATH}/requirements/tf.txt" ], - "labels": { - "server_utils": "ODQA" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/odqa_test.tar.gz", diff --git a/tests/test_configs/ranking/paraphrase_ident_paraphraser_interact_test.json b/tests/test_configs/ranking/paraphrase_ident_paraphraser_interact_test.json index 9648bec909..bec5f573c9 100644 --- a/tests/test_configs/ranking/paraphrase_ident_paraphraser_interact_test.json +++ b/tests/test_configs/ranking/paraphrase_ident_paraphraser_interact_test.json @@ -131,10 +131,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/paraphrase_ident_paraphraser.tar.gz", diff --git a/tests/test_configs/ranking/paraphrase_ident_paraphraser_test.json b/tests/test_configs/ranking/paraphrase_ident_paraphraser_test.json index 2c850276b7..c941b3d876 100644 --- a/tests/test_configs/ranking/paraphrase_ident_paraphraser_test.json +++ b/tests/test_configs/ranking/paraphrase_ident_paraphraser_test.json @@ -122,10 +122,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/paraphraser.zip", diff --git a/tests/test_configs/ranking/paraphrase_ident_qqp_bilstm_interact_test.json b/tests/test_configs/ranking/paraphrase_ident_qqp_bilstm_interact_test.json index 3dbbe888d7..54f1a78400 100644 --- a/tests/test_configs/ranking/paraphrase_ident_qqp_bilstm_interact_test.json +++ b/tests/test_configs/ranking/paraphrase_ident_qqp_bilstm_interact_test.json @@ -131,10 +131,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/paraphrase_ident_qqp_bilstm.tar.gz", diff --git a/tests/test_configs/ranking/paraphrase_ident_qqp_bilstm_test.json b/tests/test_configs/ranking/paraphrase_ident_qqp_bilstm_test.json index 8752cde3dd..78d4f1be54 100644 --- a/tests/test_configs/ranking/paraphrase_ident_qqp_bilstm_test.json +++ b/tests/test_configs/ranking/paraphrase_ident_qqp_bilstm_test.json @@ -123,10 +123,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/quora_question_pairs.zip", diff --git a/tests/test_configs/ranking/paraphrase_ident_qqp_interact_test.json b/tests/test_configs/ranking/paraphrase_ident_qqp_interact_test.json index 7d3f7742c8..c995617f29 100644 --- a/tests/test_configs/ranking/paraphrase_ident_qqp_interact_test.json +++ b/tests/test_configs/ranking/paraphrase_ident_qqp_interact_test.json @@ -131,10 +131,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/paraphrase_ident_qqp.tar.gz", diff --git a/tests/test_configs/ranking/paraphrase_ident_qqp_test.json b/tests/test_configs/ranking/paraphrase_ident_qqp_test.json index a61b273a2e..e274cc2012 100644 --- a/tests/test_configs/ranking/paraphrase_ident_qqp_test.json +++ b/tests/test_configs/ranking/paraphrase_ident_qqp_test.json @@ -123,10 +123,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/quora_question_pairs.zip", diff --git a/tests/test_configs/ranking/ranking_insurance_interact_test.json b/tests/test_configs/ranking/ranking_insurance_interact_test.json index 46b491ca38..0b2515120d 100644 --- a/tests/test_configs/ranking/ranking_insurance_interact_test.json +++ b/tests/test_configs/ranking/ranking_insurance_interact_test.json @@ -134,10 +134,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/insurance_ranking.tar.gz", diff --git a/tests/test_configs/ranking/ranking_insurance_test.json b/tests/test_configs/ranking/ranking_insurance_test.json index 5f5a32a4f7..c325922ca7 100644 --- a/tests/test_configs/ranking/ranking_insurance_test.json +++ b/tests/test_configs/ranking/ranking_insurance_test.json @@ -125,10 +125,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/insuranceQA-master.zip", diff --git a/tests/test_configs/ranking/ranking_ubuntu_v2_mt_interact_test.json b/tests/test_configs/ranking/ranking_ubuntu_v2_mt_interact_test.json index a3fdf230fe..fd75943707 100644 --- a/tests/test_configs/ranking/ranking_ubuntu_v2_mt_interact_test.json +++ b/tests/test_configs/ranking/ranking_ubuntu_v2_mt_interact_test.json @@ -130,10 +130,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/deeppavlov_data/ubuntu_v2_mt_ranking.tar.gz", diff --git a/tests/test_configs/ranking/ranking_ubuntu_v2_mt_test.json b/tests/test_configs/ranking/ranking_ubuntu_v2_mt_test.json index 346decef70..d97841acbe 100644 --- a/tests/test_configs/ranking/ranking_ubuntu_v2_mt_test.json +++ b/tests/test_configs/ranking/ranking_ubuntu_v2_mt_test.json @@ -119,10 +119,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/ubuntu_v2_data.tar.gz", diff --git a/tests/test_configs/ranking/ranking_ubuntu_v2_test.json b/tests/test_configs/ranking/ranking_ubuntu_v2_test.json index 401014ecf1..0021cab2a9 100644 --- a/tests/test_configs/ranking/ranking_ubuntu_v2_test.json +++ b/tests/test_configs/ranking/ranking_ubuntu_v2_test.json @@ -117,10 +117,6 @@ "{DEEPPAVLOV_PATH}/requirements/tf.txt", "{DEEPPAVLOV_PATH}/requirements/fasttext.txt" ], - "labels": { - "telegram_utils": "SiameseModel", - "server_utils": "Ranker" - }, "download": [ { "url": "http://files.deeppavlov.ai/datasets/ubuntu_v2_data.tar.gz", diff --git a/tests/test_quick_start.py b/tests/test_quick_start.py index 746be502c3..4b652da614 100644 --- a/tests/test_quick_start.py +++ b/tests/test_quick_start.py @@ -22,11 +22,10 @@ import deeppavlov from deeppavlov import build_model from deeppavlov.core.commands.utils import parse_config -from deeppavlov.core.common.paths import get_settings_path from deeppavlov.core.data.utils import get_all_elems_from_json from deeppavlov.download import deep_download from deeppavlov.utils.server import get_server_params -from deeppavlov.utils.socket import encode, SOCKET_CONFIG_FILENAME +from deeppavlov.utils.socket import encode tests_dir = Path(__file__).parent test_configs_path = tests_dir / "deeppavlov" / "configs" @@ -430,9 +429,7 @@ def interact_api(config_path): @staticmethod def interact_socket(config_path, socket_type): - socket_config_path = get_settings_path() / SOCKET_CONFIG_FILENAME - - socket_params = get_server_params(config_path, socket_config_path) + socket_params = get_server_params(config_path) model_args_names = socket_params['model_args_names'] host = socket_params['host'] @@ -455,7 +452,7 @@ def interact_socket(config_path, socket_type): p = pexpect.popen_spawn.PopenSpawn(' '.join(args), timeout=None, logfile=logfile) try: - p.expect(socket_params['launch_message']) + p.expect(socket_params['socket_launch_message']) with socket.socket(address_family, socket.SOCK_STREAM) as s: try: s.connect(connect_arg) From f114823beb381fc8457a68aa880c5b87f431d9df Mon Sep 17 00:00:00 2001 From: Alexey Litinsky Date: Thu, 14 Nov 2019 17:21:38 +0300 Subject: [PATCH 22/28] docs: add GPU run instructions (#1064) * Added GPU run instructions * Update README.md Co-Authored-By: Aleksei Lymar * Added GPU usage instructions to the documentation Quick Start chapter * docs: correct monospaced markup for rst --- README.md | 14 ++++++++++++++ docs/intro/quick_start.rst | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/README.md b/README.md index 0f450480bb..7fb4f8804d 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,20 @@ evaluate and infer it: * via [Command line interface (CLI)](#command-line-interface-cli) and * via [Python](#python). +#### GPU requirements + +To run supported DeepPavlov models on GPU you should have [CUDA](https://developer.nvidia.com/cuda-toolkit) 10.0 +installed on your host machine and TensorFlow with GPU support (`tensorflow-gpu`) +installed in your python environment. Current supported TensorFlow version is 1.14.0. +Run + +``` +pip install tensorflow-gpu==1.14.0 +``` + +before installing model's package requirements to install supported `tensorflow-gpu` version. + + Before making choice of an interface, install model's package requirements (CLI): diff --git a/docs/intro/quick_start.rst b/docs/intro/quick_start.rst index e6a7f2c6ee..ea968f24fd 100644 --- a/docs/intro/quick_start.rst +++ b/docs/intro/quick_start.rst @@ -133,6 +133,20 @@ There are also available integrations with various messengers, see Integrations section for more info. +Using GPU +~~~~~~~~~ + +To run or train DeepPavlov models on GPU you should have `CUDA `__ 10.0 +installed on your host machine and TensorFlow with GPU support (``tensorflow-gpu``) +installed in your python environment. Current supported TensorFlow version is 1.14.0. Run + + .. code:: bash + + pip install tensorflow-gpu==1.14.0 + +before installing model's package requirements to install supported ``tensorflow-gpu`` version. + + Pretrained models ~~~~~~~~~~~~~~~~~ From 24260821410d2f183e30e4c85f6a5a1806a96bf1 Mon Sep 17 00:00:00 2001 From: Aleksei Lymar Date: Thu, 14 Nov 2019 17:22:09 +0300 Subject: [PATCH 23/28] docs: remove star footnote for DSTC2 results table (#1068) --- docs/features/overview.rst | 36 ++++++++++++++++----------------- docs/features/skills/go_bot.rst | 36 ++++++++++++++++----------------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/docs/features/overview.rst b/docs/features/overview.rst index a67429e97b..287a9a9dfb 100644 --- a/docs/features/overview.rst +++ b/docs/features/overview.rst @@ -453,25 +453,23 @@ customizable: embeddings, slot filler and intent classifier can be switched on a Available pre-trained models and their comparison with existing benchmarks: -+----------------+------+-------------------------------------------------------------------------------------+---------------+---------+------------+------------------+ -| Dataset | Lang | Model | Metric | Valid | Test | Downloads | -+================+======+=====================================================================================+===============+=========+============+==================+ -| `DSTC 2`_ [*]_ | En | :config:`bot with slot filler ` | Turn Accuracy | 0.544 | 0.542 | 400 Mb | -+ + +-------------------------------------------------------------------------------------+ +---------+------------+------------------+ -| | | :config:`bot with slot filler & intents & attention ` | | 0.548 | **0.553** | 8.5 Gb | -+----------------+ +-------------------------------------------------------------------------------------+ +---------+------------+------------------+ -| `DSTC 2`_ | | Bordes and Weston (2016) | | -- | 0.411 | -- | -+ + +-------------------------------------------------------------------------------------+ +---------+------------+------------------+ -| | | Eric and Manning (2017) | | -- | 0.480 | -- | -+ + +-------------------------------------------------------------------------------------+ +---------+------------+------------------+ -| | | Perez and Liu (2016) | | -- | 0.487 | -- | -+ + +-------------------------------------------------------------------------------------+ +---------+------------+------------------+ -| | | Williams et al. (2017) | | -- | **0.556** | -- | -+----------------+------+-------------------------------------------------------------------------------------+---------------+---------+------------+------------------+ - -.. _`DSTC 2`: http://camdial.org/~mh521/dstc/ - -.. [*] There were a few :ref:`modifications ` to the original dataset. ++-----------------------------------+------+------------------------------------------------------------------------------------+---------------+-----------+---------------+ +| Dataset | Lang | Model | Metric | Test | Downloads | ++===================================+======+====================================================================================+===============+===========+===============+ +| `DSTC 2`_ | En | :config:`basic bot ` | Turn Accuracy | 0.380 | 10 Mb | ++ (:ref:`modified `) + +------------------------------------------------------------------------------------+ +-----------+---------------+ +| | | :config:`bot with slot filler ` | | 0.542 | 400 Mb | ++ + +------------------------------------------------------------------------------------+ +-----------+---------------+ +| | | :config:`bot with slot filler, intents & attention ` | | **0.553** | 8.5 Gb | ++-----------------------------------+ +------------------------------------------------------------------------------------+ +-----------+---------------+ +| `DSTC 2`_ | | Bordes and Weston (2016) | | 0.411 | -- | ++ + +------------------------------------------------------------------------------------+ +-----------+---------------+ +| | | Eric and Manning (2017) | | 0.480 | -- | ++ + +------------------------------------------------------------------------------------+ +-----------+---------------+ +| | | Perez and Liu (2016) | | 0.487 | -- | ++ + +------------------------------------------------------------------------------------+ +-----------+---------------+ +| | | Williams et al. (2017) | | **0.556** | -- | ++-----------------------------------+------+------------------------------------------------------------------------------------+---------------+-----------+---------------+ Seq2seq goal-oriented bot :doc:`[docs] ` diff --git a/docs/features/skills/go_bot.rst b/docs/features/skills/go_bot.rst index b1b3079fef..45409df9e9 100644 --- a/docs/features/skills/go_bot.rst +++ b/docs/features/skills/go_bot.rst @@ -320,28 +320,26 @@ Comparison Scores for different modifications of our bot model and comparison with existing benchmarks: -+----------------+------+-------------------------------------------+----------------------------------------------------------------------+---------------+-----------+---------------+ -| Dataset | Lang | Model | Config | Metric | Test | Downloads | -+================+======+===========================================+======================================================================+===============+===========+===============+ -| `DSTC 2`_ [*]_ | En | basic bot | :config:`gobot_dstc2_minimal.json ` | Turn Accuracy | 0.380 | 10 Mb | -+ + +-------------------------------------------+----------------------------------------------------------------------+ +-----------+---------------+ -| | | bot with slot filler | :config:`gobot_dstc2.json ` | | 0.542 | 400 Mb | -+ + +-------------------------------------------+----------------------------------------------------------------------+ +-----------+---------------+ -| | | bot with slot filler, intents & attention | :config:`gobot_dstc2_best.json ` | | **0.553** | 8.5 Gb | -+----------------+ +-------------------------------------------+----------------------------------------------------------------------+ +-----------+---------------+ -| `DSTC 2`_ | | Bordes and Weston (2016) [3]_ | -- | | 0.411 | -- | -+ + +-------------------------------------------+----------------------------------------------------------------------+ +-----------+---------------+ -| | | Eric and Manning (2017) [4]_ | -- | | 0.480 | -- | -+ + +-------------------------------------------+----------------------------------------------------------------------+ +-----------+---------------+ -| | | Perez and Liu (2016) [5]_ | -- | | 0.487 | -- | -+ + +-------------------------------------------+----------------------------------------------------------------------+ +-----------+---------------+ -| | | Williams et al. (2017) [1]_ | -- | | **0.556** | -- | -+----------------+------+-------------------------------------------+----------------------------------------------------------------------+---------------+-----------+---------------+ ++-----------------------------------+------+------------------------------------------------------------------------------------+---------------+-----------+---------------+ +| Dataset | Lang | Model | Metric | Test | Downloads | ++===================================+======+====================================================================================+===============+===========+===============+ +| `DSTC 2`_ | En | :config:`basic bot ` | Turn Accuracy | 0.380 | 10 Mb | ++ (:ref:`modified `) + +------------------------------------------------------------------------------------+ +-----------+---------------+ +| | | :config:`bot with slot filler ` | | 0.542 | 400 Mb | ++ + +------------------------------------------------------------------------------------+ +-----------+---------------+ +| | | :config:`bot with slot filler, intents & attention ` | | **0.553** | 8.5 Gb | ++-----------------------------------+ +------------------------------------------------------------------------------------+ +-----------+---------------+ +| `DSTC 2`_ | | Bordes and Weston (2016) [3]_ | | 0.411 | -- | ++ + +------------------------------------------------------------------------------------+ +-----------+---------------+ +| | | Eric and Manning (2017) [4]_ | | 0.480 | -- | ++ + +------------------------------------------------------------------------------------+ +-----------+---------------+ +| | | Perez and Liu (2016) [5]_ | | 0.487 | -- | ++ + +------------------------------------------------------------------------------------+ +-----------+---------------+ +| | | Williams et al. (2017) [1]_ | | **0.556** | -- | ++-----------------------------------+------+------------------------------------------------------------------------------------+---------------+-----------+---------------+ .. _`DSTC 2`: http://camdial.org/~mh521/dstc/ -.. [*] There were a few :ref:`modifications ` to the original dataset. It differs from ours in train/valid/test split and intent/action labeling. - References ---------- From c92926a4d53766fe3e25c82b1b54f71284c8f061 Mon Sep 17 00:00:00 2001 From: Aleksei Lymar Date: Thu, 14 Nov 2019 17:22:46 +0300 Subject: [PATCH 24/28] fix: allow to use userids in dstc2 gobot (#1069) * fix: reset go_bot state for user if they have no state * style: add stylistic fixes in the go-bot network code --- deeppavlov/models/go_bot/network.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/deeppavlov/models/go_bot/network.py b/deeppavlov/models/go_bot/network.py index d5774519c3..9cb4aa37e7 100644 --- a/deeppavlov/models/go_bot/network.py +++ b/deeppavlov/models/go_bot/network.py @@ -24,6 +24,7 @@ from tensorflow.contrib.layers import xavier_initializer as xav import deeppavlov.models.go_bot.templates as templ +from deeppavlov import Chainer from deeppavlov.core.commands.utils import expand_path from deeppavlov.core.common.errors import ConfigError from deeppavlov.core.common.registry import register @@ -69,7 +70,8 @@ class GoalOrientedBot(LRScheduledTFModel): dense_size: rnn input size. attention_mechanism: describes attention applied to embeddings of input tokens. - * **type** – type of attention mechanism, possible values are ``'general'``, ``'bahdanau'``, ``'light_general'``, ``'light_bahdanau'``, ``'cs_general'`` and ``'cs_bahdanau'``. + * **type** – type of attention mechanism, possible values are ``'general'``, ``'bahdanau'``, + ``'light_general'``, ``'light_bahdanau'``, ``'cs_general'`` and ``'cs_bahdanau'``. * **hidden_size** – attention hidden state size. * **max_num_tokens** – maximum number of input tokens. * **depth** – number of averages used in constrained attentions @@ -78,7 +80,8 @@ class GoalOrientedBot(LRScheduledTFModel): to attention. * **intent_as_key** – use utterance intents as attention key or not. * **projected_align** – whether to use output projection. - network_parameters: dictionary with network parameters (for compatibility with release 0.1.1, deprecated in the future) + network_parameters: dictionary with network parameters (for compatibility with release 0.1.1, + deprecated in the future) template_path: file with mapping between actions and text templates for response generation. @@ -124,7 +127,7 @@ def __init__(self, l2_reg_coef: float = 0., dense_size: int = None, attention_mechanism: dict = None, - network_parameters: Dict[str, Any] = {}, + network_parameters: Optional[Dict[str, Any]] = None, load_path: str = None, template_type: str = "DefaultTemplate", word_vocab: Component = None, @@ -137,11 +140,12 @@ def __init__(self, use_action_mask: bool = False, debug: bool = False, **kwargs) -> None: + network_parameters = network_parameters or {} if any(p in network_parameters for p in self.DEPRECATED): log.warning(f"parameters {self.DEPRECATED} are deprecated," f" for learning rate schedule documentation see" f" deeppavlov.core.models.lr_scheduled_tf_model" - f" or read gitub tutorial on super convergence.") + f" or read a github tutorial on super convergence.") if 'learning_rate' in network_parameters: kwargs['learning_rate'] = network_parameters.pop('learning_rate') super().__init__(load_path=load_path, save_path=save_path, **kwargs) @@ -169,7 +173,7 @@ def __init__(self, self.api_call_id = self.templates.actions.index(api_call_action) self.intents = [] - if callable(self.intent_classifier): + if isinstance(self.intent_classifier, Chainer): self.intents = self.intent_classifier.get_main_component().classes new_network_parameters = { @@ -492,8 +496,10 @@ def __call__(self, if isinstance(batch[0], str): res = [] if not user_ids: - user_ids = ['finn' for i in range(len(batch))] + user_ids = ['finn'] * len(batch) for user_id, x in zip(user_ids, batch): + if user_id not in self.states: + self.reset(user_id) state = self.states[user_id] state['current_db_result'] = None @@ -535,8 +541,11 @@ def _zero_state(self) -> dict: ) } - def reset(self, user_id: Union[str, int] = 'finn') -> None: - self.states[user_id] = self._zero_state() + def reset(self, user_id: Union[None, str, int] = None) -> None: + if user_id is None: + self.states.clear() + else: + self.states[user_id] = self._zero_state() if self.debug: log.debug("Bot reset.") From 396ca9b1c0d58e96f4ec72d64549ae3d393e468d Mon Sep 17 00:00:00 2001 From: Fedor Ignatov Date: Tue, 19 Nov 2019 13:43:33 +0300 Subject: [PATCH 25/28] fix: fix prepare registry import error (#1075) --- deeppavlov/core/common/registry.json | 2 +- .../deprecated/agents/ecommerce_agent/ecommerce_agent.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/deeppavlov/core/common/registry.json b/deeppavlov/core/common/registry.json index a45f35af92..51deb1c652 100644 --- a/deeppavlov/core/common/registry.json +++ b/deeppavlov/core/common/registry.json @@ -96,12 +96,12 @@ "pymorphy_vectorizer": "deeppavlov.models.vectorizers.word_vectorizer:PymorphyVectorizer", "qqp_reader": "deeppavlov.dataset_readers.quora_question_pairs_reader:QuoraQuestionPairsReader", "random_emb_mat": "deeppavlov.models.preprocessors.random_embeddings_matrix:RandomEmbeddingsMatrix", + "rasa_skill": "deeppavlov.skills.rasa_skill.rasa_skill:RASASkill", "response_base_loader": "deeppavlov.models.preprocessors.response_base_loader:ResponseBaseLoader", "ru_obscenity_classifier": "deeppavlov.models.classifiers.ru_obscenity_classifier:RuObscenityClassifier", "ru_sent_tokenizer": "deeppavlov.models.tokenizers.ru_sent_tokenizer:RuSentTokenizer", "ru_tokenizer": "deeppavlov.models.tokenizers.ru_tokenizer:RussianTokenizer", "russian_words_vocab": "deeppavlov.vocabs.typos:RussianWordsVocab", - "rasa_skill": "deeppavlov.skills.rasa_skill.rasa_skill:RASASkill", "sanitizer": "deeppavlov.models.preprocessors.sanitizer:Sanitizer", "seq2seq_go_bot": "deeppavlov.models.seq2seq_go_bot.bot:Seq2SeqGoalOrientedBot", "seq2seq_go_bot_nn": "deeppavlov.models.seq2seq_go_bot.network:Seq2SeqGoalOrientedBotNetwork", diff --git a/deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py b/deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py index d4b57290c2..d349d64979 100644 --- a/deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py +++ b/deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py @@ -22,7 +22,7 @@ from deeppavlov.deprecated.agent import Agent, RichMessage from deeppavlov.deprecated.agents.rich_content import PlainText, ButtonsFrame, Button from deeppavlov.deprecated.skill import Skill -from deeppavlov.utils.ms_bot_framework.server import run_ms_bot_framework_server +from deeppavlov.utils.ms_bot_framework import start_ms_bf_server parser = argparse.ArgumentParser() parser.add_argument("-i", "--ms-id", help="microsoft bot framework app id", type=str) @@ -190,9 +190,8 @@ def main(): """Parse parameters and run ms bot framework""" args = parser.parse_args() - run_ms_bot_framework_server(agent_generator=make_agent, - app_id=args.ms_id, - app_secret=args.ms_secret) + start_ms_bf_server(app_id=args.ms_id, + app_secret=args.ms_secret) if __name__ == '__main__': From f3d3a99ffd73b9256083bfdc0e6e2cb971a633fe Mon Sep 17 00:00:00 2001 From: AlexeySorokin Date: Wed, 27 Nov 2019 23:28:31 +0300 Subject: [PATCH 26/28] feat: add a syntax tagger (#1077) * refactor: rename BertNerModel to BertSequenceTagger * feat: add last subtoken masking to BertNERPreprocessor * refactor: rename BertNERModel to BertSequenceTagger * fix: check that a token is already in special_tokens when adding it to SimpleVocabulary * refactor: refactor BertSequenceTagger via inheritance * fix: minor BertSequenceTagger refactoring fixes * feat: add reading syntax information to MorphotaggingDatasetReader * fix: fix in reading syntax information in MorphotaggingDatasetReader * refactor: refactor BertSequenceTagger masking * data: add morpho_ru_syntagrus_bert config * feat: initial commit for BertSyntaxTagger * feat: syntax parser * feat: predict dependency label in BertSyntaxParser. * feat: add multitask training to syntax parser. * merge: merge with selected changes from sorokin/sber_challenge_lm branch * docs: documentation for BERT-based models * docs: documentation for BERT-based models * docs: BertSequenceTagger docs (cherry picked from commit 6dfe2fd from other branch) * docs: BertSyntaxParser docs * docs: BertSyntaxParser docs * docs: minor fix in Morphological Tagger docs * config: morpho_ru_syntagrus_bert config * style: minor code and doc changes * config: syntax_ru_syntagrus_bert config * docs: morphotagger docs * docs: morphotagger docs * docs: syntactic parser docs and description * chore: minor fixes * docs: minor fixes * chore: add __init__.py to syntax_parser module * docs: minor fixes * test: create BertSyntaxParser tests * docs, chore: minor fixes to @yurakuratov changes request * chore: fixes to @yurakuratov suggestions on Pull Request #1077. * feat, docs: implement JointParserTagger * refactor: implement JointParserTagger * docs, tests: update tagger and parser documentation and add tests * chore: remove idle imports * chore: remove idle variables in config file * fix: fix join_parsing config json structure * fix: fix download path in morpho_ru_syntagrus_bert config --- .../BERT/morpho_ru_syntagrus_bert.json | 176 ++++++ .../morpho_tagger/UD2.0/morpho_ar.json | 1 + .../morpho_tagger/UD2.0/morpho_cs.json | 1 + .../morpho_tagger/UD2.0/morpho_de.json | 1 + .../morpho_tagger/UD2.0/morpho_en.json | 1 + .../morpho_tagger/UD2.0/morpho_es_ancora.json | 1 + .../morpho_tagger/UD2.0/morpho_fr.json | 1 + .../morpho_tagger/UD2.0/morpho_hi.json | 1 + .../morpho_tagger/UD2.0/morpho_hu.json | 1 + .../morpho_tagger/UD2.0/morpho_it.json | 1 + .../UD2.0/morpho_ru_syntagrus.json | 1 + .../UD2.0/morpho_ru_syntagrus_pymorphy.json | 1 + ...orpho_ru_syntagrus_pymorphy_lemmatize.json | 1 + .../morpho_tagger/UD2.0/morpho_tr.json | 1 + .../configs/ner/ner_conll2003_bert.json | 4 +- .../configs/ner/ner_ontonotes_bert.json | 4 +- .../configs/ner/ner_ontonotes_bert_mult.json | 4 +- deeppavlov/configs/ner/ner_rus_bert.json | 4 +- .../syntax/ru_syntagrus_joint_parsing.json | 38 ++ .../syntax/syntax_ru_syntagrus_bert.json | 193 ++++++ deeppavlov/core/common/metrics_registry.json | 3 + deeppavlov/core/common/registry.json | 6 +- deeppavlov/core/data/simple_vocab.py | 2 + deeppavlov/core/layers/tf_layers.py | 2 +- deeppavlov/core/trainers/fit_trainer.py | 10 +- deeppavlov/core/trainers/utils.py | 15 +- .../morphotagger_iterator.py | 9 +- .../morphotagging_dataset_reader.py | 36 +- deeppavlov/download.py | 1 - deeppavlov/metrics/accuracy.py | 75 ++- deeppavlov/models/bert/bert_ranker.py | 20 +- .../{bert_ner.py => bert_sequence_tagger.py} | 595 ++++++++++-------- deeppavlov/models/morpho_tagger/__main__.py | 6 +- deeppavlov/models/morpho_tagger/common.py | 65 +- .../assemble_embeddings_matrix.py | 2 +- .../models/preprocessors/bert_preprocessor.py | 43 +- deeppavlov/models/preprocessors/mask.py | 3 +- .../models/seq2seq_go_bot/kb_attn_layer.py | 2 +- deeppavlov/models/syntax_parser/__init__.py | 0 deeppavlov/models/syntax_parser/joint.py | 142 +++++ deeppavlov/models/syntax_parser/network.py | 345 ++++++++++ deeppavlov/models/syntax_parser/parser.py | 47 ++ deeppavlov/requirements/syntax_parser.txt | 1 + docs/_static/tree.png | Bin 0 -> 12108 bytes docs/apiref/models/bert.rst | 10 +- docs/apiref/models/syntax_parser.rst | 16 + docs/features/models/bert.rst | 26 +- docs/features/models/morphotagger.rst | 157 +++-- docs/features/models/syntaxparser.rst | 170 +++++ docs/features/overview.rst | 49 +- docs/index.rst | 1 + docs/intro/installation.rst | 2 +- tests/test_quick_start.py | 14 +- 53 files changed, 1907 insertions(+), 404 deletions(-) create mode 100644 deeppavlov/configs/morpho_tagger/BERT/morpho_ru_syntagrus_bert.json create mode 100644 deeppavlov/configs/syntax/ru_syntagrus_joint_parsing.json create mode 100644 deeppavlov/configs/syntax/syntax_ru_syntagrus_bert.json rename deeppavlov/models/bert/{bert_ner.py => bert_sequence_tagger.py} (63%) create mode 100644 deeppavlov/models/syntax_parser/__init__.py create mode 100644 deeppavlov/models/syntax_parser/joint.py create mode 100644 deeppavlov/models/syntax_parser/network.py create mode 100644 deeppavlov/models/syntax_parser/parser.py create mode 100644 deeppavlov/requirements/syntax_parser.txt create mode 100644 docs/_static/tree.png create mode 100644 docs/apiref/models/syntax_parser.rst create mode 100644 docs/features/models/syntaxparser.rst diff --git a/deeppavlov/configs/morpho_tagger/BERT/morpho_ru_syntagrus_bert.json b/deeppavlov/configs/morpho_tagger/BERT/morpho_ru_syntagrus_bert.json new file mode 100644 index 0000000000..a6a6ac792d --- /dev/null +++ b/deeppavlov/configs/morpho_tagger/BERT/morpho_ru_syntagrus_bert.json @@ -0,0 +1,176 @@ +{ + "dataset_reader": { + "class_name": "morphotagger_dataset_reader", + "data_path": "{DOWNLOADS_PATH}/UD2.3_source", + "language": "ru_syntagrus", + "data_types": [ + "train", "dev", "test" + ] + }, + "dataset_iterator": { + "class_name": "morphotagger_dataset" + }, + "chainer": { + "in": ["x"], + "in_y": ["y"], + "pipe": [ + { + "in": [ + "x" + ], + "class_name": "lazy_tokenizer", + "out": [ + "x_words" + ] + }, + { + "class_name": "bert_ner_preprocessor", + "vocab_file": "{BERT_PATH}/vocab.txt", + "do_lower_case": false, + "max_seq_length": 512, + "max_subword_length": 15, + "subword_mask_mode": "last", + "token_masking_prob": 0.0, + "in": ["x_words"], + "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "pred_subword_mask"] + }, + { + "class_name": "mask", + "in": ["x_subword_tokens"], + "out": ["x_subword_mask"] + }, + { + "id": "tag_vocab", + "class_name": "simple_vocab", + "min_freq": 3, + "fit_on": [ + "y" + ], + "in": ["y"], + "out": ["y_ind"], + "special_tokens": [ + "PAD", + "BEGIN", + "END" + ], + "pad_with_zeros": true, + "save_path": "{WORK_PATH}/tag.dict", + "load_path": "{WORK_PATH}/tag.dict" + }, + { + "class_name": "bert_sequence_tagger", + "n_tags": "#tag_vocab.len", + "keep_prob": 0.1, + "bert_config_file": "{BERT_PATH}/bert_config.json", + "pretrained_bert": "{BERT_PATH}/bert_model.ckpt", + "attention_probs_keep_prob": 0.5, + "use_crf": false, + "return_probas": false, + "encoder_layer_ids": [6, 7, 8, 9, 10, 11], + "optimizer": "tf.train:AdamOptimizer", + "learning_rate": 1e-3, + "bert_learning_rate": 2e-5, + "min_learning_rate": 1e-7, + "learning_rate_drop_patience": 30, + "learning_rate_drop_div": 1.5, + "load_before_drop": true, + "clip_norm": null, + "save_path": "{WORK_PATH}/model", + "load_path": "{WORK_PATH}/model", + "in": ["x_subword_tok_ids", "x_subword_mask", "pred_subword_mask"], + "in_y": ["y_ind"], + "out": ["y_predicted_ind"] + }, + { + "ref": "tag_vocab", + "in": ["y_predicted_ind"], + "out": ["y_predicted"] + }, + { + "in": [ + "x_words", + "y_predicted" + ], + "out": [ + "y_lemmas" + ], + "class_name": "UD_pymorphy_lemmatizer", + "end": "\n" + }, + { + "in": [ + "x_words", + "y_predicted", + "y_lemmas" + ], + "out": [ + "y_prettified" + ], + "id": "prettifier", + "class_name": "lemmatized_output_prettifier", + "end": "\n" + } + ], + "out": [ + "y_prettified" + ] + }, + "train": { + "epochs": 1, + "batch_size": 32, + "metrics": [ + { + "name": "per_token_accuracy", + "inputs": [ + "y", + "y_predicted" + ] + }, + { + "name": "accuracy", + "inputs": [ + "y", + "y_predicted" + ] + } + ], + "validation_patience": 10, + "val_every_n_epochs": 1, + "val_every_n_batches": 300, + + "tensorboard_log_dir": "{WORK_PATH}/logs", + "show_examples": false, + "pytest_max_batches": 2, + "pytest_batch_size": 8, + "evaluation_targets": ["valid", "test"], + "class_name": "nn_trainer" + }, + "metadata": { + "variables": { + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "BERT_PATH": "{DOWNLOADS_PATH}/bert_models/rubert_cased_L-12_H-768_A-12_v1", + "WORK_PATH": "{MODELS_PATH}/morpho_ru_syntagrus" + }, + "requirements": [ + "{DEEPPAVLOV_PATH}/requirements/tf.txt", + "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt", + "{DEEPPAVLOV_PATH}/requirements/morpho_tagger.txt" + ], + "download": [ + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/morpho_tagger/BERT/morpho_ru_syntagrus_bert.tar.gz", + "subdir": "{WORK_PATH}" + }, + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/rubert_cased_L-12_H-768_A-12_v1.tar.gz", + "subdir": "{DOWNLOADS_PATH}/bert_models" + }, + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/morpho_tagger/UD2.3/ru_syntagrus.tar.gz", + "subdir": "{DOWNLOADS_PATH}/UD2.3_source/ru_syntagrus" + } + ] + } +} diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ar.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ar.json index 22d2623f8c..80038c4f5c 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ar.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ar.json @@ -115,6 +115,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_cs.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_cs.json index ec1318001d..f83bda3adf 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_cs.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_cs.json @@ -115,6 +115,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_de.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_de.json index 0faeac1b13..4bdeba1c3f 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_de.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_de.json @@ -115,6 +115,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_en.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_en.json index c847f5e5b9..9f47acd7e2 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_en.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_en.json @@ -115,6 +115,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_es_ancora.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_es_ancora.json index a50cb801b7..35cf73673a 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_es_ancora.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_es_ancora.json @@ -115,6 +115,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_fr.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_fr.json index 256977765e..7899039f3f 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_fr.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_fr.json @@ -115,6 +115,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hi.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hi.json index ac79415851..a80c32fea9 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hi.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hi.json @@ -115,6 +115,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hu.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hu.json index 1dc47596a5..8e7cb96692 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hu.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_hu.json @@ -115,6 +115,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_it.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_it.json index d545af336f..acacc011f4 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_it.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_it.json @@ -115,6 +115,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus.json index 3a12f35948..fb040d1190 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus.json @@ -115,6 +115,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy.json index 0f0ec0dd29..d0206cdbe7 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy.json @@ -135,6 +135,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy_lemmatize.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy_lemmatize.json index 664ddfd317..3b55661a5f 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy_lemmatize.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy_lemmatize.json @@ -148,6 +148,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "lemmatized_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_tr.json b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_tr.json index 602c241236..3163670a6c 100644 --- a/deeppavlov/configs/morpho_tagger/UD2.0/morpho_tr.json +++ b/deeppavlov/configs/morpho_tagger/UD2.0/morpho_tr.json @@ -116,6 +116,7 @@ "out": [ "y_prettified" ], + "id": "prettifier", "class_name": "tag_output_prettifier", "end": "\n" } diff --git a/deeppavlov/configs/ner/ner_conll2003_bert.json b/deeppavlov/configs/ner/ner_conll2003_bert.json index 7166cc31f4..ae17c3b34a 100644 --- a/deeppavlov/configs/ner/ner_conll2003_bert.json +++ b/deeppavlov/configs/ner/ner_conll2003_bert.json @@ -18,7 +18,7 @@ "do_lower_case": false, "max_seq_length": 512, "max_subword_length": 15, - "token_maksing_prob": 0.0, + "token_masking_prob": 0.0, "in": ["x"], "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "pred_subword_mask"] }, @@ -39,7 +39,7 @@ "out": ["y_ind"] }, { - "class_name": "bert_ner", + "class_name": "bert_sequence_tagger", "n_tags": "#tag_vocab.len", "keep_prob": 0.1, "bert_config_file": "{BERT_PATH}/bert_config.json", diff --git a/deeppavlov/configs/ner/ner_ontonotes_bert.json b/deeppavlov/configs/ner/ner_ontonotes_bert.json index 8d5dcf8e34..1dd1f1c515 100644 --- a/deeppavlov/configs/ner/ner_ontonotes_bert.json +++ b/deeppavlov/configs/ner/ner_ontonotes_bert.json @@ -18,7 +18,7 @@ "do_lower_case": false, "max_seq_length": 512, "max_subword_length": 15, - "token_maksing_prob": 0.0, + "token_masking_prob": 0.0, "in": ["x"], "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "pred_subword_mask"] }, @@ -39,7 +39,7 @@ "out": ["y_ind"] }, { - "class_name": "bert_ner", + "class_name": "bert_sequence_tagger", "n_tags": "#tag_vocab.len", "keep_prob": 0.1, "bert_config_file": "{BERT_PATH}/bert_config.json", diff --git a/deeppavlov/configs/ner/ner_ontonotes_bert_mult.json b/deeppavlov/configs/ner/ner_ontonotes_bert_mult.json index a9a9066553..bda3dd30df 100644 --- a/deeppavlov/configs/ner/ner_ontonotes_bert_mult.json +++ b/deeppavlov/configs/ner/ner_ontonotes_bert_mult.json @@ -18,7 +18,7 @@ "do_lower_case": false, "max_seq_length": 512, "max_subword_length": 15, - "token_maksing_prob": 0.0, + "token_masking_prob": 0.0, "in": ["x"], "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "pred_subword_mask"] }, @@ -39,7 +39,7 @@ "out": ["y_ind"] }, { - "class_name": "bert_ner", + "class_name": "bert_sequence_tagger", "n_tags": "#tag_vocab.len", "keep_prob": 0.1, "bert_config_file": "{BERT_PATH}/bert_config.json", diff --git a/deeppavlov/configs/ner/ner_rus_bert.json b/deeppavlov/configs/ner/ner_rus_bert.json index 31ba3c96b3..0b8c7e9848 100644 --- a/deeppavlov/configs/ner/ner_rus_bert.json +++ b/deeppavlov/configs/ner/ner_rus_bert.json @@ -18,7 +18,7 @@ "do_lower_case": false, "max_seq_length": 512, "max_subword_length": 15, - "token_maksing_prob": 0.0, + "token_masking_prob": 0.0, "in": ["x"], "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "pred_subword_mask"] }, @@ -39,7 +39,7 @@ "out": ["y_ind"] }, { - "class_name": "bert_ner", + "class_name": "bert_sequence_tagger", "n_tags": "#tag_vocab.len", "keep_prob": 0.1, "bert_config_file": "{BERT_PATH}/bert_config.json", diff --git a/deeppavlov/configs/syntax/ru_syntagrus_joint_parsing.json b/deeppavlov/configs/syntax/ru_syntagrus_joint_parsing.json new file mode 100644 index 0000000000..d807260ef1 --- /dev/null +++ b/deeppavlov/configs/syntax/ru_syntagrus_joint_parsing.json @@ -0,0 +1,38 @@ +{ + "chainer": { + "in": [ + "x_words" + ], + "pipe": [ + { + "id": "main", + "class_name": "joint_tagger_parser", + "tagger": {"config_path": "{CONFIGS_PATH}/morpho_tagger/BERT/morpho_ru_syntagrus_bert.json"}, + "parser": {"config_path": "{CONFIGS_PATH}/syntax/syntax_ru_syntagrus_bert.json"}, + "to_output_string": true, + "in": [ + "x_words" + ], + "out": [ + "y_parsed" + ] + } + ], + "out": [ + "y_parsed" + ] + }, + "metadata": { + "variables": { + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "CONFIGS_PATH": "{DEEPPAVLOV_PATH}/configs", + "MODELS_PATH": "{ROOT_PATH}/models" + }, + "requirements": [ + "{DEEPPAVLOV_PATH}/requirements/tf.txt", + "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt", + "{DEEPPAVLOV_PATH}/requirements/morpho_tagger.txt" + ] + } +} \ No newline at end of file diff --git a/deeppavlov/configs/syntax/syntax_ru_syntagrus_bert.json b/deeppavlov/configs/syntax/syntax_ru_syntagrus_bert.json new file mode 100644 index 0000000000..6ed421b605 --- /dev/null +++ b/deeppavlov/configs/syntax/syntax_ru_syntagrus_bert.json @@ -0,0 +1,193 @@ +{ + "dataset_reader": { + "class_name": "morphotagger_dataset_reader", + "data_path": "{DOWNLOADS_PATH}/UD2.3_source", + "language": "ru_syntagrus", + "data_types": [ + "train", "dev", "test" + ], + "read_syntax": true + }, + "dataset_iterator": { + "class_name": "morphotagger_dataset" + }, + "chainer": { + "in": ["x"], + "in_y": ["y_tags", "y_heads", "y_deps"], + "pipe": [ + { + "in": [ + "x" + ], + "class_name": "lazy_tokenizer", + "out": [ + "x_words" + ] + }, + { + "class_name": "bert_ner_preprocessor", + "vocab_file": "{BERT_PATH}/vocab.txt", + "do_lower_case": false, + "max_seq_length": 512, + "max_subword_length": 15, + "subword_mask_mode": "last", + "token_masking_prob": 0.0, + "in": ["x_words"], + "out": ["x_tokens", "x_subword_tokens", "x_subword_tok_ids", "pred_subword_mask"] + }, + { + "class_name": "mask", + "in": ["x_subword_tokens"], + "out": ["x_subword_mask"] + }, + { + "id": "dep_vocab", + "class_name": "simple_vocab", + "min_freq": 3, + "fit_on": [ + "y_deps" + ], + "in": ["y_deps"], + "out": ["y_deps_indexes"], + "special_tokens": [ + "PAD" + ], + "pad_with_zeros": true, + "save_path": "{WORK_PATH}/deps.dict", + "load_path": "{WORK_PATH}/deps.dict" + }, + { + "class_name": "bert_syntax_parser", + "n_deps": "#dep_vocab.len", + "state_size": 384, + "keep_prob": 0.1, + "bert_config_file": "{BERT_PATH}/bert_config.json", + "pretrained_bert": "{BERT_PATH}/bert_model.ckpt", + "attention_probs_keep_prob": 0.5, + "use_crf": false, + "return_probas": true, + "encoder_layer_ids": [6, 7, 8, 9, 10, 11], + "optimizer": "tf.train:AdamOptimizer", + "learning_rate": 1e-3, + "bert_learning_rate": 2e-5, + "min_learning_rate": 1e-7, + "use_birnn": true, + "learning_rate_drop_patience": 30, + "learning_rate_drop_div": 1.5, + "load_before_drop": true, + "clip_norm": null, + "save_path": "{WORK_PATH}/model_joint", + "load_path": "{WORK_PATH}/model_joint", + "in": ["x_subword_tok_ids", "x_subword_mask", "pred_subword_mask"], + "in_y": ["y_heads", "y_deps_indexes"], + "out": ["y_predicted_heads_probs", "y_predicted_deps_indexes"] + }, + { + "class_name": "chu_liu_edmonds_transformer", + "in": ["y_predicted_heads_probs"], + "out": ["y_predicted_heads"] + }, + { + "ref": "dep_vocab", + "in": ["y_predicted_deps_indexes"], + "out": ["y_predicted_deps"] + }, + { + "in": [ + "x_words", + "y_predicted_heads", + "y_predicted_deps" + ], + "out": [ + "y_prettified" + ], + "id": "dependency_output_prettifier", + "class_name": "dependency_output_prettifier", + "end": "\n" + } + ], + "out": [ + "y_prettified" + ] + }, + "train": { + "epochs": 10, + "batch_size": 32, + "metrics": [ + { + "name": "multitask_token_accuracy", + "alias": "LAS", + "inputs": [ + "y_deps", + "y_heads", + "y_predicted_deps", + "y_predicted_heads" + ] + }, + { + "name": "multitask_sequence_accuracy", + "alias": "sentence_LAS", + "inputs": [ + "y_deps", + "y_heads", + "y_predicted_deps", + "y_predicted_heads" + ] + }, + { + "name": "per_token_accuracy", + "alias": "UAS", + "inputs": [ + "y_heads", + "y_predicted_heads" + ] + }, + { + "name": "accuracy", + "alias": "sentence_UAS", + "inputs": [ + "y_heads", + "y_predicted_heads" + ] + } + ], + "validation_patience": 10, + "val_every_n_epochs": 1, + "val_every_n_batches": 300, + + "tensorboard_log_dir": "{WORK_PATH}/logs", + "show_examples": false, + "pytest_max_batches": 2, + "pytest_batch_size": 8, + "evaluation_targets": ["valid", "test"], + "class_name": "nn_trainer" + }, + "metadata": { + "variables": { + "ROOT_PATH": "~/.deeppavlov", + "DOWNLOADS_PATH": "{ROOT_PATH}/downloads", + "MODELS_PATH": "{ROOT_PATH}/models", + "BERT_PATH": "{DOWNLOADS_PATH}/bert_models/rubert_cased_L-12_H-768_A-12_v1", + "WORK_PATH": "{MODELS_PATH}/syntax_ru_syntagrus" + }, + "requirements": [ + "{DEEPPAVLOV_PATH}/requirements/tf.txt", + "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt", + "{DEEPPAVLOV_PATH}/requirements/syntax_parser.txt" + ], + "download": [ + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/syntax_parser/syntax_ru_syntagrus_bert.tar.gz", + "subdir": "{WORK_PATH}" + }, + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/rubert_cased_L-12_H-768_A-12_v1.tar.gz", + "subdir": "{DOWNLOADS_PATH}/bert_models" + }, + { + "url": "http://files.deeppavlov.ai/deeppavlov_data/morpho_tagger/UD2.3/ru_syntagrus.tar.gz", + "subdir": "{DOWNLOADS_PATH}/UD2.3_source/ru_syntagrus" + } + ] + } +} diff --git a/deeppavlov/core/common/metrics_registry.json b/deeppavlov/core/common/metrics_registry.json index 15053bdb14..02d33cf458 100644 --- a/deeppavlov/core/common/metrics_registry.json +++ b/deeppavlov/core/common/metrics_registry.json @@ -10,6 +10,9 @@ "google_bleu": "deeppavlov.metrics.bleu:google_bleu", "kbqa_accuracy": "deeppavlov.metrics.accuracy:kbqa_accuracy", "log_loss": "deeppavlov.metrics.log_loss:sk_log_loss", + "multitask_accuracy": "deeppavlov.metrics.accuracy:multitask_accuracy", + "multitask_sequence_accuracy": "deeppavlov.metrics.accuracy:multitask_sequence_accuracy", + "multitask_token_accuracy": "deeppavlov.metrics.accuracy:multitask_token_accuracy", "ner_f1": "deeppavlov.metrics.fmeasure:ner_f1", "ner_token_f1": "deeppavlov.metrics.fmeasure:ner_token_f1", "per_item_bleu": "deeppavlov.metrics.bleu:per_item_bleu", diff --git a/deeppavlov/core/common/registry.json b/deeppavlov/core/common/registry.json index 51deb1c652..9f995f778a 100644 --- a/deeppavlov/core/common/registry.json +++ b/deeppavlov/core/common/registry.json @@ -7,7 +7,8 @@ "basic_classification_iterator": "deeppavlov.dataset_iterators.basic_classification_iterator:BasicClassificationDatasetIterator", "basic_classification_reader": "deeppavlov.dataset_readers.basic_classification_reader:BasicClassificationDatasetReader", "bert_classifier": "deeppavlov.models.bert.bert_classifier:BertClassifierModel", - "bert_ner": "deeppavlov.models.bert.bert_ner:BertNerModel", + "bert_sequence_tagger": "deeppavlov.models.bert.bert_sequence_tagger:BertSequenceTagger", + "bert_syntax_parser": "deeppavlov.models.syntax_parser.network:BertSyntaxParser", "bert_ner_preprocessor": "deeppavlov.models.preprocessors.bert_preprocessor:BertNerPreprocessor", "bert_preprocessor": "deeppavlov.models.preprocessors.bert_preprocessor:BertPreprocessor", "bert_ranker": "deeppavlov.models.bert.bert_ranker:BertRankerModel", @@ -22,6 +23,7 @@ "capitalization_featurizer": "deeppavlov.models.preprocessors.capitalization:CapitalizationPreprocessor", "char_splitter": "deeppavlov.models.preprocessors.char_splitter:CharSplitter", "char_splitting_lowercase_preprocessor": "deeppavlov.models.preprocessors.capitalization:CharSplittingLowercasePreprocessor", + "chu_liu_edmonds_transformer": "deeppavlov.models.syntax_parser.parser:ChuLiuEdmonds", "conll2003_reader": "deeppavlov.dataset_readers.conll2003_reader:Conll2003DatasetReader", "cos_sim_classifier": "deeppavlov.models.classifiers.cos_sim_classifier:CosineSimilarityClassifier", "dam_nn": "deeppavlov.models.ranking.deep_attention_matching_network:DAMNetwork", @@ -56,6 +58,7 @@ "go_bot": "deeppavlov.models.go_bot.network:GoalOrientedBot", "hashing_tfidf_vectorizer": "deeppavlov.models.vectorizers.hashing_tfidf_vectorizer:HashingTfIdfVectorizer", "insurance_reader": "deeppavlov.dataset_readers.insurance_reader:InsuranceReader", + "joint_tagger_parser": "deeppavlov.models.syntax_parser.joint:JointTaggerParser", "kb_answer_parser_wikidata": "deeppavlov.models.kbqa.kb_answer_parser_wikidata:KBAnswerParserWikidata", "kbqa_reader": "deeppavlov.dataset_readers.kbqa_reader:KBQAReader", "kenlm_elector": "deeppavlov.models.spelling_correction.electors.kenlm_elector:KenlmElector", @@ -65,6 +68,7 @@ "kvret_dialog_iterator": "deeppavlov.dataset_iterators.kvret_dialog_iterator:KvretDialogDatasetIterator", "kvret_reader": "deeppavlov.dataset_readers.kvret_reader:KvretDatasetReader", "lazy_tokenizer": "deeppavlov.models.tokenizers.lazy_tokenizer:LazyTokenizer", + "dependency_output_prettifier": "deeppavlov.models.morpho_tagger.common:DependencyOutputPrettifier", "lemmatized_output_prettifier": "deeppavlov.models.morpho_tagger.common:LemmatizedOutputPrettifier", "line_reader": "deeppavlov.dataset_readers.line_reader:LineReader", "logit_ranker": "deeppavlov.models.doc_retrieval.logit_ranker:LogitRanker", diff --git a/deeppavlov/core/data/simple_vocab.py b/deeppavlov/core/data/simple_vocab.py index 8332d4d680..73edb5aec9 100644 --- a/deeppavlov/core/data/simple_vocab.py +++ b/deeppavlov/core/data/simple_vocab.py @@ -71,6 +71,8 @@ def fit(self, *args): self._i2t.append(special_token) self.count += 1 for token, freq in self.freqs.most_common()[:self._max_tokens]: + if token in self.special_tokens: + continue if freq >= self._min_freq: self._t2i[token] = self.count self._i2t.append(token) diff --git a/deeppavlov/core/layers/tf_layers.py b/deeppavlov/core/layers/tf_layers.py index ee41a50725..6ac6799b55 100644 --- a/deeppavlov/core/layers/tf_layers.py +++ b/deeppavlov/core/layers/tf_layers.py @@ -125,7 +125,7 @@ def bi_rnn(units: tf.Tensor, Args: units: a tensorflow tensor with dimensionality [None, n_tokens, n_features] - n_hidden: list with number of hidden units at the ouput of each layer + n_hidden: list with number of hidden units in the output of each layer seq_lengths: length of sequences for different length sequences in batch can be None for maximum length as a length for every sample in the batch cell_type: 'lstm' or 'gru' diff --git a/deeppavlov/core/trainers/fit_trainer.py b/deeppavlov/core/trainers/fit_trainer.py index 7a7820f962..f82968029b 100644 --- a/deeppavlov/core/trainers/fit_trainer.py +++ b/deeppavlov/core/trainers/fit_trainer.py @@ -159,12 +159,12 @@ def _load(self) -> None: self._loaded = True def get_chainer(self) -> Chainer: - """Return a :class:`~deeppavlov.core.common.chainer.Chainer` built from ``self.chainer_config`` for inference""" + """Returns a :class:`~deeppavlov.core.common.chainer.Chainer` built from ``self.chainer_config`` for inference""" self._load() return self._chainer def train(self, iterator: Union[DataFittingIterator, DataLearningIterator]) -> None: - """Call :meth:`~fit_chainer` with provided data iterator as an argument""" + """Calls :meth:`~fit_chainer` with provided data iterator as an argument""" self.fit_chainer(iterator) self._saved = True @@ -214,7 +214,11 @@ def test(self, data: Iterable[Tuple[Collection[Any], Collection[Any]]], log.warning('Got empty data iterable for scoring') return {'eval_examples_count': 0, 'metrics': None, 'time_spent': str(datetime.timedelta(seconds=0))} - metrics_values = [(m.name, m.fn(*[outputs[i] for i in m.inputs])) for m in metrics] + # metrics_values = [(m.name, m.fn(*[outputs[i] for i in m.inputs])) for m in metrics] + metrics_values = [] + for metric in metrics: + value = metric.fn(*[outputs[i] for i in metric.inputs]) + metrics_values.append((metric.alias, value)) report = { 'eval_examples_count': examples, diff --git a/deeppavlov/core/trainers/utils.py b/deeppavlov/core/trainers/utils.py index 1e8429aadd..e1daedad54 100644 --- a/deeppavlov/core/trainers/utils.py +++ b/deeppavlov/core/trainers/utils.py @@ -13,28 +13,30 @@ # limitations under the License. from collections import OrderedDict, namedtuple -from typing import List, Tuple, Union, Iterable +from typing import List, Tuple, Union, Iterable, Any from deeppavlov.core.common.metrics_registry import get_metric_by_name -Metric = namedtuple('Metric', ['name', 'fn', 'inputs']) +Metric = namedtuple('Metric', ['name', 'fn', 'inputs', 'alias']) def parse_metrics(metrics: Iterable[Union[str, dict]], in_y: List[str], out_vars: List[str]) -> List[Metric]: metrics_functions = [] for metric in metrics: if isinstance(metric, str): - metric = {'name': metric} + metric = {'name': metric, 'alias': metric} metric_name = metric['name'] + alias = metric.get('alias', metric_name) f = get_metric_by_name(metric_name) inputs = metric.get('inputs', in_y + out_vars) if isinstance(inputs, str): inputs = [inputs] + - metrics_functions.append(Metric(metric_name, f, inputs)) + metrics_functions.append(Metric(metric_name, f, inputs, alias)) return metrics_functions @@ -42,6 +44,9 @@ def prettify_metrics(metrics: List[Tuple[str, float]], precision: int = 4) -> Or """Prettifies the dictionary of metrics.""" prettified_metrics = OrderedDict() for key, value in metrics: - value = round(value, precision) + if key in prettified_metrics: + Warning("Multiple metrics with the same name {}.".format(key)) + if isinstance(value, float): + value = round(value, precision) prettified_metrics[key] = value return prettified_metrics diff --git a/deeppavlov/dataset_iterators/morphotagger_iterator.py b/deeppavlov/dataset_iterators/morphotagger_iterator.py index b94d5b0285..40af273f51 100644 --- a/deeppavlov/dataset_iterators/morphotagger_iterator.py +++ b/deeppavlov/dataset_iterators/morphotagger_iterator.py @@ -76,7 +76,7 @@ def split(self, *args, **kwargs) -> None: """ if len(self.valid) == 0: if self.shuffle: - random.shuffle(self.train) + self.random.shuffle(self.train) L = int(len(self.train) * (1.0 - self.validation_split)) self.train, self.valid = self.train[:L], self.train[L:] elif self.min_train_fraction > 0.0: @@ -103,14 +103,15 @@ def gen_batches(self, batch_size: int, data_type: str = 'train', if shuffle is None: shuffle = self.shuffle data = self.data[data_type] - if shuffle: - random.shuffle(data) lengths = [len(x[0]) for x in data] indexes = np.argsort(lengths) L = len(data) if batch_size < 0: batch_size = L - for start in range(0, L, batch_size): + starts = list(range(0, L, batch_size)) + if shuffle: + self.random.shuffle(starts) + for start in starts: indexes_to_yield = indexes[start:start + batch_size] data_to_yield = tuple(list(x) for x in zip(*([data[i] for i in indexes_to_yield]))) if return_indexes: diff --git a/deeppavlov/dataset_readers/morphotagging_dataset_reader.py b/deeppavlov/dataset_readers/morphotagging_dataset_reader.py index af0af41196..4402022252 100644 --- a/deeppavlov/dataset_readers/morphotagging_dataset_reader.py +++ b/deeppavlov/dataset_readers/morphotagging_dataset_reader.py @@ -22,6 +22,7 @@ from deeppavlov.core.data.utils import download_decompress, mark_done WORD_COLUMN, POS_COLUMN, TAG_COLUMN = 1, 3, 5 +HEAD_COLUMN, DEP_COLUMN = 6, 7 log = getLogger(__name__) @@ -32,10 +33,11 @@ def get_language(filepath: str) -> str: return filepath.split("-")[0] -def read_infile(infile: Union[Path, str], from_words=False, +def read_infile(infile: Union[Path, str], *, from_words=False, word_column: int = WORD_COLUMN, pos_column: int = POS_COLUMN, - tag_column: int = TAG_COLUMN, max_sents: int = -1, - read_only_words: bool = False) -> List[Tuple[List, Union[List, None]]]: + tag_column: int = TAG_COLUMN, head_column: int = HEAD_COLUMN, + dep_column: int = DEP_COLUMN, max_sents: int = -1, + read_only_words: bool = False, read_syntax: bool = False) -> List[Tuple[List, Union[List, None]]]: """Reads input file in CONLL-U format Args: @@ -43,14 +45,22 @@ def read_infile(infile: Union[Path, str], from_words=False, word_column: column containing words (default=1) pos_column: column containing part-of-speech labels (default=3) tag_column: column containing fine-grained tags (default=5) - max_sents: maximal number of sents to read + head_column: column containing syntactic head position (default=6) + dep_column: column containing syntactic dependency label (default=7) + max_sents: maximal number of sentences to read read_only_words: whether to read only words + read_syntax: whether to return ``heads`` and ``deps`` alongside ``tags``. Ignored if read_only_words is ``True`` Returns: - a list of sentences. Each item contains a word sequence and a tag sequence, which is ``None`` - in case ``read_only_words = True`` + a list of sentences. Each item contains a word sequence and an output sequence. + The output sentence is ``None``, if ``read_only_words`` is ``True``, + a single list of word tags if ``read_syntax`` is False, + and a list of the form [``tags``, ``heads``, ``deps``] in case ``read_syntax`` is ``True``. + """ answer, curr_word_sent, curr_tag_sent = [], [], [] + curr_head_sent, curr_dep_sent = [], [] + # read_syntax = read_syntax and read_only_words if from_words: word_column, read_only_words = 0, True if infile is not sys.stdin: @@ -65,8 +75,11 @@ def read_infile(infile: Union[Path, str], from_words=False, if len(curr_word_sent) > 0: if read_only_words: curr_tag_sent = None + elif read_syntax: + curr_tag_sent = [curr_tag_sent, curr_head_sent, curr_dep_sent] answer.append((curr_word_sent, curr_tag_sent)) curr_tag_sent, curr_word_sent = [], [] + curr_head_sent, curr_dep_sent = [], [] if len(answer) == max_sents: break continue @@ -79,9 +92,14 @@ def read_infile(infile: Union[Path, str], from_words=False, pos, tag = splitted[pos_column], splitted[tag_column] tag = pos if tag == "_" else "{},{}".format(pos, tag) curr_tag_sent.append(tag) + if read_syntax: + curr_head_sent.append(int(splitted[head_column])) + curr_dep_sent.append(splitted[dep_column]) if len(curr_word_sent) > 0: if read_only_words: curr_tag_sent = None + elif read_syntax: + curr_tag_sent = [curr_tag_sent, curr_head_sent, curr_dep_sent] answer.append((curr_word_sent, curr_tag_sent)) if infile is not sys.stdin: fin.close() @@ -95,7 +113,7 @@ class MorphotaggerDatasetReader(DatasetReader): URL = 'http://files.deeppavlov.ai/datasets/UD2.0_source/' def read(self, data_path: Union[List, str], - language: Optional[None] = None, + language: Optional[str] = None, data_types: Optional[List[str]] = None, **kwargs) -> Dict[str, List]: """Reads UD dataset from data_path. @@ -164,7 +182,7 @@ def read(self, data_path: Union[List, str], for mode, filepath in zip(data_types, data_path): if mode == "dev": mode = "valid" - # if mode == "test": - # kwargs["read_only_words"] = True +# if mode == "test": +# kwargs["read_only_words"] = True data[mode] = read_infile(filepath, **kwargs) return data diff --git a/deeppavlov/download.py b/deeppavlov/download.py index e3d0d6e490..917b6184ec 100644 --- a/deeppavlov/download.py +++ b/deeppavlov/download.py @@ -63,7 +63,6 @@ def get_config_downloads(config: Union[str, Path, dict]) -> Set[Tuple[str, Path] def get_configs_downloads(config: Optional[Union[str, Path, dict]] = None) -> Dict[str, Set[Path]]: all_downloads = defaultdict(set) - if config: configs = [config] else: diff --git a/deeppavlov/metrics/accuracy.py b/deeppavlov/metrics/accuracy.py index ff5e45a6aa..fdf0788cce 100644 --- a/deeppavlov/metrics/accuracy.py +++ b/deeppavlov/metrics/accuracy.py @@ -30,13 +30,84 @@ def accuracy(y_true: [list, np.ndarray], y_predicted: [list, np.ndarray]) -> flo y_predicted: array of predicted values Returns: - portion of absolutely coincidental samples + fraction of absolutely coincidental samples """ examples_len = len(y_true) - correct = sum([y1 == y2 for y1, y2 in zip(y_true, y_predicted)]) + # if y1 and y2 are both arrays, == can be erroneously interpreted as element-wise equality + def _are_equal(y1, y2): + answer = (y1 == y2) + if isinstance(answer, np.ndarray): + answer = answer.all() + return answer + equalities = [_are_equal(y1, y2) for y1, y2 in zip(y_true, y_predicted)] + correct = sum(equalities) return correct / examples_len if examples_len else 0 +@register_metric('multitask_accuracy') +def multitask_accuracy(*args) -> float: + """ + Accuracy for multiple simultaneous tasks. + + Args: + *args: a list of `2n` inputs. The first `n` inputs are the correct answers for `n` tasks + and the last `n` are the predicted ones. + + Returns: + The percentage of inputs where the answers for all `n` tasks are correct. + """ + n = len(args) + y_true_by_tasks, y_predicted_by_tasks = args[:n // 2], args[n // 2:] + y_true, y_predicted = list(zip(*y_true_by_tasks)), list(zip(*y_predicted_by_tasks)) + return accuracy(y_true, y_predicted) + + +@register_metric('multitask_sequence_accuracy') +def multitask_sequence_accuracy(*args) -> float: + """ + Accuracy for multiple simultaneous sequence labeling (tagging) tasks. + For each sequence the model checks whether all its elements + are labeled correctly for all the individual taggers. + + Args: + *args: a list of `2n` inputs. The first `n` inputs are the correct answers for `n` tasks + and the last `n` are the predicted ones. For each task an + + Returns: + The percentage of sequences where all the items has correct answers for all `n` tasks. + + """ + n = len(args) + y_true_by_tasks, y_predicted_by_tasks = args[:n // 2], args[n // 2:] + y_true_by_sents = list(zip(*y_true_by_tasks)) + y_predicted_by_sents = list(zip(*y_predicted_by_tasks)) + y_true = list(list(zip(*elem)) for elem in y_true_by_sents) + y_predicted = list(list(zip(*elem)) for elem in y_predicted_by_sents) + return accuracy(y_true, y_predicted) + + +@register_metric('multitask_token_accuracy') +def multitask_token_accuracy(*args) -> float: + """ + Per-item accuracy for multiple simultaneous sequence labeling (tagging) tasks. + + Args: + *args: a list of `2n` inputs. The first `n` inputs are the correct answers for `n` tasks + and the last `n` are the predicted ones. For each task an + + Returns: + The percentage of sequence elements for which the answers for all `n` tasks are correct. + + """ + n = len(args) + y_true_by_tasks, y_predicted_by_tasks = args[:n // 2], args[n // 2:] + y_true_by_sents = list(zip(*y_true_by_tasks)) + y_predicted_by_sents = list(zip(*y_predicted_by_tasks)) + y_true = list(list(zip(*elem)) for elem in y_true_by_sents) + y_predicted = list(list(zip(*elem)) for elem in y_predicted_by_sents) + return per_token_accuracy(y_true, y_predicted) + + @register_metric('sets_accuracy') def sets_accuracy(y_true: [list, np.ndarray], y_predicted: [list, np.ndarray]) -> float: """ diff --git a/deeppavlov/models/bert/bert_ranker.py b/deeppavlov/models/bert/bert_ranker.py index d01923e417..dbf2686d55 100644 --- a/deeppavlov/models/bert/bert_ranker.py +++ b/deeppavlov/models/bert/bert_ranker.py @@ -43,12 +43,12 @@ class BertRankerModel(BertClassifierModel): bert_config_file: path to Bert configuration file n_classes: number of classes keep_prob: dropout keep_prob for non-Bert layers - return_probas: set True if return class probabilites instead of most probable label needed + return_probas: set True if class probabilities are returned instead of the most probable label """ def __init__(self, bert_config_file, n_classes=2, keep_prob=0.9, return_probas=True, **kwargs) -> None: - super().__init__(bert_config_file=bert_config_file, n_classes=n_classes, keep_prob=keep_prob, - return_probas=return_probas, **kwargs) + super().__init__(bert_config_file=bert_config_file, n_classes=n_classes, + keep_prob=keep_prob, return_probas=return_probas, **kwargs) def train_on_batch(self, features_li: List[List[InputFeatures]], y: Union[List[int], List[List[int]]]) -> Dict: """Train the model on the given batch. @@ -118,8 +118,8 @@ class BertSepRankerModel(LRScheduledTFModel): keep_prob: dropout keep_prob for non-Bert layers attention_probs_keep_prob: keep_prob for Bert self-attention layers hidden_keep_prob: keep_prob for Bert hidden layers - optimizer: name of tf.train.* optimizer or None for `AdamWeightDecayOptimizer` - weight_decay_rate: L2 weight decay for `AdamWeightDecayOptimizer` + optimizer: name of tf.train.* optimizer or None for ``AdamWeightDecayOptimizer`` + weight_decay_rate: L2 weight decay for ``AdamWeightDecayOptimizer`` pretrained_bert: pretrained Bert checkpoint min_learning_rate: min value of learning rate if learning rate decay is used """ @@ -353,11 +353,11 @@ class BertSepRankerPredictor(BertSepRankerModel): batch_size: batch size for building response (and context) vectors over the base keep_prob: dropout keep_prob for non-Bert layers resps: list of strings containing the base of text responses - resp_vecs: BERT vector respresentations of `resps`, if is `None` it will be build - resp_features: features of `resps` to build their BERT vector representations + resp_vecs: BERT vector respresentations of ``resps``, if is ``None`` it will be build + resp_features: features of ``resps`` to build their BERT vector representations conts: list of strings containing the base of text contexts - cont_vecs: BERT vector respresentations of `conts`, if is `None` it will be build - cont_features: features of `conts` to build their BERT vector representations + cont_vecs: BERT vector respresentations of ``conts``, if is ``None`` it will be build + cont_features: features of ``conts`` to build their BERT vector representations """ def __init__(self, bert_config_file, interact_mode=0, batch_size=32, @@ -428,7 +428,7 @@ def _get_predictions(self, features_li): return np.vstack(pred) def _retrieve_db_response(self, ctx_vec): - """Retrieve a text response from the base based on the policy determined by `interact_mode`. + """Retrieve a text response from the base based on the policy determined by ``interact_mode``. Uses cosine similarity scores over vectors of responses (and corresponding contexts) from the base. """ diff --git a/deeppavlov/models/bert/bert_ner.py b/deeppavlov/models/bert/bert_sequence_tagger.py similarity index 63% rename from deeppavlov/models/bert/bert_ner.py rename to deeppavlov/models/bert/bert_sequence_tagger.py index 55171a92aa..ae40ac315b 100644 --- a/deeppavlov/models/bert/bert_ner.py +++ b/deeppavlov/models/bert/bert_sequence_tagger.py @@ -13,7 +13,7 @@ # limitations under the License. from logging import getLogger -from typing import List, Union, Dict +from typing import List, Union, Dict, Optional import numpy as np import tensorflow as tf @@ -28,36 +28,159 @@ log = getLogger(__name__) -@register('bert_ner') -class BertNerModel(LRScheduledTFModel): - """BERT-based model for text tagging. +def token_from_subtoken(units: tf.Tensor, mask: tf.Tensor) -> tf.Tensor: + """ Assemble token level units from subtoken level units - For each token a tag is predicted. Can be used for any tagging. + Args: + units: tf.Tensor of shape [batch_size, SUBTOKEN_seq_length, n_features] + mask: mask of token beginnings. For example: for tokens + + [[``[CLS]`` ``My``, ``capybara``, ``[SEP]``], + [``[CLS]`` ``Your``, ``aar``, ``##dvark``, ``is``, ``awesome``, ``[SEP]``]] + + the mask will be + + [[0, 1, 1, 0, 0, 0, 0], + [0, 1, 1, 0, 1, 1, 0]] + + Returns: + word_level_units: Units assembled from ones in the mask. For the + example above this units will correspond to the following + + [[``My``, ``capybara``], + [``Your`, ``aar``, ``is``, ``awesome``,]] + + the shape of this tensor will be [batch_size, TOKEN_seq_length, n_features] + """ + shape = tf.cast(tf.shape(units), tf.int64) + batch_size = shape[0] + nf = shape[2] + nf_int = units.get_shape().as_list()[-1] + + # number of TOKENS in each sentence + token_seq_lengths = tf.cast(tf.reduce_sum(mask, 1), tf.int64) + # for a matrix m = + # [[1, 1, 1], + # [0, 1, 1], + # [1, 0, 0]] + # it will be + # [3, 2, 1] + + n_words = tf.reduce_sum(token_seq_lengths) + # n_words -> 6 + + max_token_seq_len = tf.cast(tf.reduce_max(token_seq_lengths), tf.int64) + # max_token_seq_len -> 3 + + idxs = tf.where(mask) + # for the matrix mentioned above + # tf.where(mask) -> + # [[0, 0], + # [0, 1] + # [0, 2], + # [1, 1], + # [1, 2] + # [2, 0]] + + sample_ids_in_batch = tf.pad(idxs[:, 0], [[1, 0]]) + # for indices + # [[0, 0], + # [0, 1] + # [0, 2], + # [1, 1], + # [1, 2], + # [2, 0]] + # it is + # [0, 0, 0, 0, 1, 1, 2] + # padding is for computing change from one sample to another in the batch + + a = tf.cast(tf.not_equal(sample_ids_in_batch[1:], sample_ids_in_batch[:-1]), tf.int64) + # for the example above the result of this statement equals + # [0, 0, 0, 1, 0, 1] + # so data samples begin in 3rd and 5th positions (the indexes of ones) + + # transforming sample start masks to the sample starts themselves + q = a * tf.cast(tf.range(n_words), tf.int64) + # [0, 0, 0, 3, 0, 5] + count_to_substract = tf.pad(tf.boolean_mask(q, q), [(1, 0)]) + # [0, 3, 5] + + new_word_indices = tf.cast(tf.range(n_words), tf.int64) - tf.gather(count_to_substract, tf.cumsum(a)) + # tf.range(n_words) -> [0, 1, 2, 3, 4, 5] + # tf.cumsum(a) -> [0, 0, 0, 1, 1, 2] + # tf.gather(count_to_substract, tf.cumsum(a)) -> [0, 0, 0, 3, 3, 5] + # new_word_indices -> [0, 1, 2, 3, 4, 5] - [0, 0, 0, 3, 3, 5] = [0, 1, 2, 0, 1, 0] + # new_word_indices is the concatenation of range(word_len(sentence)) + # for all sentences in units + + n_total_word_elements = tf.cast(batch_size * max_token_seq_len, tf.int32) + word_indices_flat = tf.cast(idxs[:, 0] * max_token_seq_len + new_word_indices, tf.int32) + x_mask = tf.reduce_sum(tf.one_hot(word_indices_flat, n_total_word_elements), 0) + x_mask = tf.cast(x_mask, tf.bool) + # to get absolute indices we add max_token_seq_len: + # idxs[:, 0] * max_token_seq_len -> [0, 0, 0, 1, 1, 2] * 2 = [0, 0, 0, 3, 3, 6] + # word_indices_flat -> [0, 0, 0, 3, 3, 6] + [0, 1, 2, 0, 1, 0] = [0, 1, 2, 3, 4, 6] + # total number of words in the batch (including paddings) + # batch_size * max_token_seq_len -> 3 * 3 = 9 + # tf.one_hot(...) -> + # [[1. 0. 0. 0. 0. 0. 0. 0. 0.] + # [0. 1. 0. 0. 0. 0. 0. 0. 0.] + # [0. 0. 1. 0. 0. 0. 0. 0. 0.] + # [0. 0. 0. 1. 0. 0. 0. 0. 0.] + # [0. 0. 0. 0. 1. 0. 0. 0. 0.] + # [0. 0. 0. 0. 0. 0. 1. 0. 0.]] + # x_mask -> [1, 1, 1, 1, 1, 0, 1, 0, 0] + + full_range = tf.cast(tf.range(batch_size * max_token_seq_len), tf.int32) + # full_range -> [0, 1, 2, 3, 4, 5, 6, 7, 8] + nonword_indices_flat = tf.boolean_mask(full_range, tf.math.logical_not(x_mask)) + # # y_idxs -> [5, 7, 8] + + # get a sequence of units corresponding to the start subtokens of the words + # size: [n_words, n_features] + elements = tf.gather_nd(units, idxs) + + # prepare zeros for paddings + # size: [batch_size * TOKEN_seq_length - n_words, n_features] + paddings = tf.zeros(tf.stack([tf.reduce_sum(max_token_seq_len - token_seq_lengths), + nf], 0), tf.float32) + + tensor_flat = tf.dynamic_stitch([word_indices_flat, nonword_indices_flat], + [elements, paddings]) + # tensor_flat -> [x, x, x, x, x, 0, x, 0, 0] + + tensor = tf.reshape(tensor_flat, tf.stack([batch_size, max_token_seq_len, nf_int], 0)) + # tensor -> [[x, x, x], + # [x, x, 0], + # [x, 0, 0]] + + return tensor + + +@register('bert_sequence_network') +class BertSequenceNetwork(LRScheduledTFModel): + """ + Basic class for BERT-based sequential architectures. Args: - n_tags: number of distinct tags keep_prob: dropout keep_prob for non-Bert layers bert_config_file: path to Bert configuration file pretrained_bert: pretrained Bert checkpoint attention_probs_keep_prob: keep_prob for Bert self-attention layers hidden_keep_prob: keep_prob for Bert hidden layers - use_crf: whether to use CRF on top or not encoder_layer_ids: list of averaged layers from Bert encoder (layer ids) optimizer: name of tf.train.* optimizer or None for `AdamWeightDecayOptimizer` weight_decay_rate: L2 weight decay for `AdamWeightDecayOptimizer` - use_birnn: whether to add bi rnn on top of the representations produced by BERT - birnn_cell_type: type of cell to use. Either `lstm` or `gru` - birnn_hidden_size: number of hidden units in the lstm + encoder_dropout: dropout probability of encoder output layer ema_decay: what exponential moving averaging to use for network parameters, value from 0.0 to 1.0. Values closer to 1.0 put weight on the parameters history and values closer to 0.0 corresponds put weight on the current parameters. ema_variables_on_cpu: whether to put EMA variables to CPU. It may save a lot of GPU memory - return_probas: set True if return class probabilites instead of most probable label needed freeze_embeddings: set True to not train input embeddings set True to not train input embeddings set True to not train input embeddings - learning_rate: learning rate of the NER head - bert_learning_rate: learning rate of the BERT body - min_learning_rate: min value of learning rate if learning rate decay is used + learning_rate: learning rate of BERT head + bert_learning_rate: learning rate of BERT body + min_learning_rate: min value of learning rate if learning rate decay is used learning_rate_drop_patience: how many validations with no improvements to wait learning_rate_drop_div: the divider of the learning rate after `learning_rate_drop_patience` unsuccessful validations @@ -66,22 +189,17 @@ class BertNerModel(LRScheduledTFModel): """ def __init__(self, - n_tags: List[str], keep_prob: float, bert_config_file: str, pretrained_bert: str = None, attention_probs_keep_prob: float = None, hidden_keep_prob: float = None, - use_crf=False, encoder_layer_ids: List[int] = (-1,), + encoder_dropout: float = 0.0, optimizer: str = None, weight_decay_rate: float = 1e-6, - use_birnn: bool = False, - birnn_cell_type: str = 'lstm', - birnn_hidden_size: int = 128, ema_decay: float = None, ema_variables_on_cpu: bool = True, - return_probas: bool = False, freeze_embeddings: bool = False, learning_rate: float = 1e-3, bert_learning_rate: float = 2e-5, @@ -97,19 +215,13 @@ def __init__(self, load_before_drop=load_before_drop, clip_norm=clip_norm, **kwargs) - - self.n_tags = n_tags self.keep_prob = keep_prob - self.use_crf = use_crf self.encoder_layer_ids = encoder_layer_ids + self.encoder_dropout = encoder_dropout self.optimizer = optimizer self.weight_decay_rate = weight_decay_rate - self.use_birnn = use_birnn - self.birnn_cell_type = birnn_cell_type - self.birnn_hidden_size = birnn_hidden_size self.ema_decay = ema_decay self.ema_variables_on_cpu = ema_variables_on_cpu - self.return_probas = return_probas self.freeze_embeddings = freeze_embeddings self.bert_learning_rate_multiplier = bert_learning_rate / learning_rate self.min_learning_rate = min_learning_rate @@ -150,8 +262,6 @@ def __init__(self, self.sess.run(self.ema.init_op) def _init_graph(self) -> None: - self._init_placeholders() - self.seq_lengths = tf.reduce_sum(self.y_masks_ph, axis=1) self.bert = BertModel(config=self.bert_config, @@ -161,58 +271,37 @@ def _init_graph(self) -> None: token_type_ids=self.token_types_ph, use_one_hot_embeddings=False) - encoder_layers = [self.bert.all_encoder_layers[i] - for i in self.encoder_layer_ids] - with tf.variable_scope('ner'): layer_weights = tf.get_variable('layer_weights_', - shape=len(encoder_layers), + shape=len(self.encoder_layer_ids), initializer=tf.ones_initializer(), trainable=True) - layer_weights = tf.unstack(layer_weights / len(encoder_layers)) + layer_mask = tf.ones_like(layer_weights) + layer_mask = tf.nn.dropout(layer_mask, self.encoder_keep_prob_ph) + layer_weights *= layer_mask + # to prevent zero division + mask_sum = tf.maximum(tf.reduce_sum(layer_mask), 1.0) + layer_weights = tf.unstack(layer_weights / mask_sum) # TODO: may be stack and reduce_sum is faster - units = sum(w * l for w, l in zip(layer_weights, encoder_layers)) + units = sum(w * l for w, l in zip(layer_weights, self.encoder_layers())) units = tf.nn.dropout(units, keep_prob=self.keep_prob_ph) - if self.use_birnn: - units, _ = bi_rnn(units, - self.birnn_hidden_size, - cell_type=self.birnn_cell_type, - seq_lengths=self.seq_lengths, - name='birnn') - units = tf.concat(units, -1) - # TODO: maybe add one more layer? - logits = tf.layers.dense(units, units=self.n_tags, name="output_dense") - - self.logits = self.token_from_subtoken(logits, self.y_masks_ph) - - max_length = tf.reduce_max(self.seq_lengths) - one_hot_max_len = tf.one_hot(self.seq_lengths - 1, max_length) - tag_mask = tf.cumsum(one_hot_max_len[:, ::-1], axis=1)[:, ::-1] + return units - # CRF - if self.use_crf: - transition_params = tf.get_variable('Transition_Params', - shape=[self.n_tags, self.n_tags], - initializer=tf.zeros_initializer()) - log_likelihood, transition_params = \ - tf.contrib.crf.crf_log_likelihood(self.logits, - self.y_ph, - self.seq_lengths, - transition_params) - loss_tensor = -log_likelihood - self._transition_params = transition_params - - self.y_predictions = tf.argmax(self.logits, -1) - self.y_probas = tf.nn.softmax(self.logits, axis=2) + def _get_tag_mask(self) -> tf.Tensor: + """ + Returns: tag_mask, + a mask that selects positions corresponding to word tokens (not padding and `CLS`) + """ + max_length = tf.reduce_max(self.seq_lengths) + one_hot_max_len = tf.one_hot(self.seq_lengths - 1, max_length) + tag_mask = tf.cumsum(one_hot_max_len[:, ::-1], axis=1)[:, ::-1] + return tag_mask - with tf.variable_scope("loss"): - y_mask = tf.cast(tag_mask, tf.float32) - if self.use_crf: - self.loss = tf.reduce_mean(loss_tensor) - else: - self.loss = tf.losses.sparse_softmax_cross_entropy(labels=self.y_ph, - logits=self.logits, - weights=y_mask) + def encoder_layers(self): + """ + Returns: the output of BERT layers specfied in ``self.encoder_layers_ids`` + """ + return [self.bert.all_encoder_layers[i] for i in self.encoder_layer_ids] def _init_placeholders(self) -> None: self.input_ids_ph = tf.placeholder(shape=(None, None), @@ -225,14 +314,9 @@ def _init_placeholders(self) -> None: tf.placeholder_with_default(tf.zeros_like(self.input_ids_ph, dtype=tf.int32), shape=self.input_ids_ph.shape, name='token_types_ph') - - self.y_ph = tf.placeholder(shape=(None, None), dtype=tf.int32, name='y_ph') - self.y_masks_ph = tf.placeholder(shape=(None, None), - dtype=tf.int32, - name='y_mask_ph') - self.learning_rate_ph = tf.placeholder_with_default(0.0, shape=[], name='learning_rate_ph') self.keep_prob_ph = tf.placeholder_with_default(1.0, shape=[], name='keep_prob_ph') + self.encoder_keep_prob_ph = tf.placeholder_with_default(1.0, shape=[], name='encoder_keep_prob_ph') self.is_train_ph = tf.placeholder_with_default(False, shape=[], name='is_train_ph') def _init_optimizer(self) -> None: @@ -299,197 +383,223 @@ def get_train_op(self, loss: tf.Tensor, learning_rate: Union[tf.Tensor, float], **kwargs) return tf.group(bert_train_op, head_train_op) - @staticmethod - def token_from_subtoken(units: tf.Tensor, mask: tf.Tensor) -> tf.Tensor: - """ Assemble token level units from subtoken level units - - Args: - units: tf.Tensor of shape [batch_size, SUBTOKEN_seq_length, n_features] - mask: mask of startings of new tokens. Example: for tokens - - [[`[CLS]` `My`, `capybara`, `[SEP]`], - [`[CLS]` `Your`, `aar`, `##dvark`, `is`, `awesome`, `[SEP]`]] - - the mask will be - - [[0, 1, 1, 0, 0, 0, 0], - [0, 1, 1, 0, 1, 1, 0]] - - Returns: - word_level_units: Units assembled from ones in the mask. For the - example above this units will correspond to the following - - [[`My`, `capybara`], - [`Your`, `aar`, `is`, `awesome`,]] - - the shape of this thesor will be [batch_size, TOKEN_seq_length, n_features] + def _build_basic_feed_dict(self, input_ids: tf.Tensor, input_masks: tf.Tensor, + token_types: Optional[tf.Tensor]=None, train: bool=False) -> dict: + """Fills the feed_dict with the tensors defined in the basic class. + You need to update this dict by the values of output placeholders + and class-specific network inputs in your derived class. """ - shape = tf.cast(tf.shape(units), tf.int64) - bs = shape[0] - nf = shape[2] - nf_int = units.get_shape().as_list()[-1] - - # numer of TOKENS in each sentence - token_seq_lenghs = tf.cast(tf.reduce_sum(mask, 1), tf.int64) - # for a matrix m = - # [[1, 1, 1], - # [0, 1, 1], - # [1, 0, 0]] - # it will be - # [3, 2, 1] - - n_words = tf.reduce_sum(token_seq_lenghs) - # n_words -> 6 - - max_token_seq_len = tf.reduce_max(token_seq_lenghs) - max_token_seq_len = tf.cast(max_token_seq_len, tf.int64) - # max_token_seq_len -> 3 - - idxs = tf.where(mask) - # for the matrix mentioned above - # tf.where(mask) -> - # [[0, 0], - # [0, 1] - # [0, 2], - # [1, 1], - # [1, 2] - # [2, 0]] - - sample_id_in_batch = tf.pad(idxs[:, 0], [[1, 0]]) - # for indices - # [[0, 0], - # [0, 1] - # [0, 2], - # [1, 1], - # [1, 2], - # [2, 0]] - # it will be - # [0, 0, 0, 0, 1, 1, 2] - # padding is for computing change from one sample to another in the batch - - a = tf.cast(tf.not_equal(sample_id_in_batch[1:], sample_id_in_batch[:-1]), tf.int64) - # for the example above the result of this line will be - # [0, 0, 0, 1, 0, 1] - # so the number of the sample in batch changes only in the last word element - - q = a * tf.cast(tf.range(n_words), tf.int64) - # [0, 0, 0, 3, 0, 5] - - count_to_substract = tf.pad(tf.boolean_mask(q, q), [(1, 0)]) - # [0, 3, 5] - - new_word_indices = tf.cast(tf.range(n_words), tf.int64) - tf.gather(count_to_substract, tf.cumsum(a)) - # tf.range(n_words) -> [0, 1, 2, 3, 4, 5] - # tf.cumsum(a) -> [0, 0, 0, 1, 1, 2] - # tf.gather(count_to_substract, tf.cumsum(a)) -> [0, 0, 0, 3, 3, 5] - # new_word_indices -> [0, 1, 2, 3, 4, 5] - [0, 0, 0, 3, 3, 5] = [0, 1, 2, 0, 1, 0] - # this is new indices token dimension - - n_total_word_elements = tf.cast(bs * max_token_seq_len, tf.int32) - x_mask = tf.reduce_sum(tf.one_hot(idxs[:, 0] * max_token_seq_len + new_word_indices, n_total_word_elements), 0) - x_mask = tf.cast(x_mask, tf.bool) - # to get absolute indices we add max_token_seq_len: - # idxs[:, 0] * max_token_seq_len -> [0, 0, 0, 1, 1, 2] * 2 = [0, 0, 0, 3, 3, 6] - # idxs[:, 0] * max_token_seq_len + new_word_indices -> - # [0, 0, 0, 3, 3, 6] + [0, 1, 2, 0, 1, 0] = [0, 1, 2, 3, 4, 6] - # total number of words in the batch (including paddings) - # bs * max_token_seq_len -> 3 * 2 = 6 - # tf.one_hot(...) -> - # [[1. 0. 0. 0. 0. 0. 0. 0. 0.] - # [0. 1. 0. 0. 0. 0. 0. 0. 0.] - # [0. 0. 1. 0. 0. 0. 0. 0. 0.] - # [0. 0. 0. 1. 0. 0. 0. 0. 0.] - # [0. 0. 0. 0. 1. 0. 0. 0. 0.] - # [0. 0. 0. 0. 0. 0. 1. 0. 0.]] - # x_mask -> [1, 1, 1, 1, 1, 0, 1, 0, 0] - - # full_range -> [0, 1, 2, 3, 4, 5, 6, 7, 8] - full_range = tf.cast(tf.range(bs * max_token_seq_len), tf.int32) - - x_idxs = tf.boolean_mask(full_range, x_mask) - # x_idxs -> [0, 1, 2, 3, 4, 6] - - y_mask = tf.math.logical_not(x_mask) - y_idxs = tf.boolean_mask(full_range, y_mask) - # y_idxs -> [5, 7, 8] - - # get a sequence of units corresponding to the start subtokens of the words - # size: [n_words, n_features] - els = tf.gather_nd(units, idxs) - - # prepare zeros for paddings - # size: [batch_size * TOKEN_seq_length - n_words, n_features] - paddings = tf.zeros(tf.stack([tf.reduce_sum(max_token_seq_len - token_seq_lenghs), - nf], 0), tf.float32) - - tensor_flat = tf.dynamic_stitch([x_idxs, y_idxs], [els, paddings]) - # tensor_flat -> [x, x, x, x, x, 0, x, 0, 0] - - tensor = tf.reshape(tensor_flat, tf.stack([bs, max_token_seq_len, nf_int], 0)) - # tensor_flat -> [[x, x, x], - # [x, x, 0], - # [x, 0, 0]] - - return tensor - - def _decode_crf(self, feed_dict: Dict[tf.Tensor, np.ndarray]) -> List[np.ndarray]: - logits, trans_params, mask, seq_lengths = self.sess.run([self.logits, - self._transition_params, - self.y_masks_ph, - self.seq_lengths], - feed_dict=feed_dict) - # iterate over the sentences because no batching in viterbi_decode - y_pred = [] - for logit, sequence_length in zip(logits, seq_lengths): - logit = logit[:int(sequence_length)] # keep only the valid steps - viterbi_seq, viterbi_score = tf.contrib.crf.viterbi_decode(logit, trans_params) - y_pred += [viterbi_seq] - return y_pred - - def _build_feed_dict(self, input_ids, input_masks, y_masks, token_types=None, y=None): feed_dict = { self.input_ids_ph: input_ids, self.input_masks_ph: input_masks, - self.y_masks_ph: y_masks } if token_types is not None: feed_dict[self.token_types_ph] = token_types - if y is not None: + if train: feed_dict.update({ - self.y_ph: y, self.learning_rate_ph: max(self.get_learning_rate(), self.min_learning_rate), self.keep_prob_ph: self.keep_prob, + self.encoder_keep_prob_ph: 1.0 - self.encoder_dropout, self.is_train_ph: True, }) return feed_dict + def _build_feed_dict(self, input_ids, input_masks, token_types=None, *args, **kwargs): + raise NotImplementedError("You must implement _build_feed_dict in your derived class.") + def train_on_batch(self, input_ids: Union[List[List[int]], np.ndarray], input_masks: Union[List[List[int]], np.ndarray], - y_masks: Union[List[List[int]], np.ndarray], - y: Union[List[List[int]], np.ndarray]) -> Dict[str, float]: + *args, **kwargs) -> Dict[str, float]: """ Args: input_ids: batch of indices of subwords input_masks: batch of masks which determine what should be attended - y_masks: batch of masks of the first subtokens in the token - y: batch of indices of tags + args: arguments passed to _build_feed_dict + and corresponding to additional input + and output tensors of the derived class. + kwargs: keyword arguments passed to _build_feed_dict + and corresponding to additional input + and output tensors of the derived class. Returns: dict with fields 'loss', 'head_learning_rate', and 'bert_learning_rate' """ - feed_dict = self._build_feed_dict(input_ids, input_masks, y_masks, y=y) + feed_dict = self._build_feed_dict(input_ids, input_masks, *args, **kwargs) if self.ema: self.sess.run(self.ema.switch_to_train_op) _, loss, lr = self.sess.run([self.train_op, self.loss, self.learning_rate_ph], - feed_dict=feed_dict) + feed_dict=feed_dict) return {'loss': loss, 'head_learning_rate': float(lr), 'bert_learning_rate': float(lr) * self.bert_learning_rate_multiplier} + def __call__(self, + input_ids: Union[List[List[int]], np.ndarray], + input_masks: Union[List[List[int]], np.ndarray], + **kwargs) -> Union[List[List[int]], List[np.ndarray]]: + raise NotImplementedError("You must implement method __call__ in your derived class.") + + def save(self, exclude_scopes=('Optimizer', 'EMA/BackupVariables')) -> None: + if self.ema: + self.sess.run(self.ema.switch_to_train_op) + return super().save(exclude_scopes=exclude_scopes) + + def load(self, + exclude_scopes=('Optimizer', + 'learning_rate', + 'momentum', + 'EMA/BackupVariables'), + **kwargs) -> None: + return super().load(exclude_scopes=exclude_scopes, **kwargs) + + +@register('bert_sequence_tagger') +class BertSequenceTagger(BertSequenceNetwork): + """BERT-based model for text tagging. It predicts a label for every token (not subtoken) in the text. + You can use it for sequence labeling tasks, such as morphological tagging or named entity recognition. + See :class:`deeppavlov.models.bert.bert_sequence_tagger.BertSequenceNetwork` + for the description of inherited parameters. + + Args: + n_tags: number of distinct tags + use_crf: whether to use CRF on top or not + use_birnn: whether to use bidirection rnn after BERT layers. + For NER and morphological tagging we usually set it to `False` as otherwise the model overfits + birnn_cell_type: the type of Bidirectional RNN. Either `lstm` or `gru` + birnn_hidden_size: number of hidden units in the BiRNN layer in each direction + return_probas: set this to `True` if you need the probabilities instead of raw answers + """ + + def __init__(self, + n_tags: List[str], + keep_prob: float, + bert_config_file: str, + pretrained_bert: str = None, + attention_probs_keep_prob: float = None, + hidden_keep_prob: float = None, + use_crf=False, + encoder_layer_ids: List[int] = (-1,), + encoder_dropout: float = 0.0, + optimizer: str = None, + weight_decay_rate: float = 1e-6, + use_birnn: bool = False, + birnn_cell_type: str = 'lstm', + birnn_hidden_size: int = 128, + ema_decay: float = None, + ema_variables_on_cpu: bool = True, + return_probas: bool = False, + freeze_embeddings: bool = False, + learning_rate: float = 1e-3, + bert_learning_rate: float = 2e-5, + min_learning_rate: float = 1e-07, + learning_rate_drop_patience: int = 20, + learning_rate_drop_div: float = 2.0, + load_before_drop: bool = True, + clip_norm: float = 1.0, + **kwargs) -> None: + self.n_tags = n_tags + self.use_crf = use_crf + self.use_birnn = use_birnn + self.birnn_cell_type = birnn_cell_type + self.birnn_hidden_size = birnn_hidden_size + self.return_probas = return_probas + super().__init__(keep_prob=keep_prob, + bert_config_file=bert_config_file, + pretrained_bert=pretrained_bert, + attention_probs_keep_prob=attention_probs_keep_prob, + hidden_keep_prob=hidden_keep_prob, + encoder_layer_ids=encoder_layer_ids, + encoder_dropout=encoder_dropout, + optimizer=optimizer, + weight_decay_rate=weight_decay_rate, + ema_decay=ema_decay, + ema_variables_on_cpu=ema_variables_on_cpu, + freeze_embeddings=freeze_embeddings, + learning_rate=learning_rate, + bert_learning_rate=bert_learning_rate, + min_learning_rate=min_learning_rate, + learning_rate_drop_div=learning_rate_drop_div, + learning_rate_drop_patience=learning_rate_drop_patience, + load_before_drop=load_before_drop, + clip_norm=clip_norm, + **kwargs) + + def _init_graph(self) -> None: + self._init_placeholders() + + units = super()._init_graph() + + with tf.variable_scope('ner'): + if self.use_birnn: + units, _ = bi_rnn(units, + self.birnn_hidden_size, + cell_type=self.birnn_cell_type, + seq_lengths=self.seq_lengths, + name='birnn') + units = tf.concat(units, -1) + # TODO: maybe add one more layer? + logits = tf.layers.dense(units, units=self.n_tags, name="output_dense") + + self.logits = token_from_subtoken(logits, self.y_masks_ph) + + # CRF + if self.use_crf: + transition_params = tf.get_variable('Transition_Params', + shape=[self.n_tags, self.n_tags], + initializer=tf.zeros_initializer()) + log_likelihood, transition_params = \ + tf.contrib.crf.crf_log_likelihood(self.logits, + self.y_ph, + self.seq_lengths, + transition_params) + loss_tensor = -log_likelihood + self._transition_params = transition_params + + self.y_predictions = tf.argmax(self.logits, -1) + self.y_probas = tf.nn.softmax(self.logits, axis=2) + + with tf.variable_scope("loss"): + tag_mask = self._get_tag_mask() + y_mask = tf.cast(tag_mask, tf.float32) + if self.use_crf: + self.loss = tf.reduce_mean(loss_tensor) + else: + self.loss = tf.losses.sparse_softmax_cross_entropy(labels=self.y_ph, + logits=self.logits, + weights=y_mask) + + def _init_placeholders(self) -> None: + super()._init_placeholders() + self.y_ph = tf.placeholder(shape=(None, None), dtype=tf.int32, name='y_ph') + self.y_masks_ph = tf.placeholder(shape=(None, None), + dtype=tf.int32, + name='y_mask_ph') + + def _decode_crf(self, feed_dict: Dict[tf.Tensor, np.ndarray]) -> List[np.ndarray]: + logits, trans_params, mask, seq_lengths = self.sess.run([self.logits, + self._transition_params, + self.y_masks_ph, + self.seq_lengths], + feed_dict=feed_dict) + # iterate over the sentences because no batching in viterbi_decode + y_pred = [] + for logit, sequence_length in zip(logits, seq_lengths): + logit = logit[:int(sequence_length)] # keep only the valid steps + viterbi_seq, viterbi_score = tf.contrib.crf.viterbi_decode(logit, trans_params) + y_pred += [viterbi_seq] + return y_pred + + def _build_feed_dict(self, input_ids, input_masks, y_masks, y=None): + feed_dict = self._build_basic_feed_dict(input_ids, input_masks, train=(y is not None)) + feed_dict[self.y_masks_ph] = y_masks + if y is not None: + feed_dict[self.y_ph] = y + return feed_dict + def __call__(self, input_ids: Union[List[List[int]], np.ndarray], input_masks: Union[List[List[int]], np.ndarray], @@ -502,7 +612,7 @@ def __call__(self, y_masks: mask which determines the first subword units in the the word Returns: - Predictions indices or predicted probabilities fro each token (not subtoken) + Label indices or class probabilities for each token (not subtoken) """ feed_dict = self._build_feed_dict(input_ids, input_masks, y_masks) @@ -518,19 +628,6 @@ def __call__(self, pred = self.sess.run(self.y_probas, feed_dict=feed_dict) return pred - def save(self, exclude_scopes=('Optimizer', 'EMA/BackupVariables')) -> None: - if self.ema: - self.sess.run(self.ema.switch_to_train_op) - return super().save(exclude_scopes=exclude_scopes) - - def load(self, - exclude_scopes=('Optimizer', - 'learning_rate', - 'momentum', - 'EMA/BackupVariables'), - **kwargs) -> None: - return super().load(exclude_scopes=exclude_scopes, **kwargs) - class ExponentialMovingAverage: def __init__(self, diff --git a/deeppavlov/models/morpho_tagger/__main__.py b/deeppavlov/models/morpho_tagger/__main__.py index 0ea0b112bb..9649755157 100644 --- a/deeppavlov/models/morpho_tagger/__main__.py +++ b/deeppavlov/models/morpho_tagger/__main__.py @@ -19,5 +19,7 @@ config_path = find_config(args.config_path) if args.download: deep_download(config_path) - predict_with_model(config_path, infile=args.file_path, input_format=args.input_format, - batch_size=args.batch_size, output_format=args.output_format) + answer = predict_with_model(config_path, infile=args.file_path, input_format=args.input_format, + batch_size=args.batch_size, output_format=args.output_format) + for elem in answer: + print(elem) diff --git a/deeppavlov/models/morpho_tagger/common.py b/deeppavlov/models/morpho_tagger/common.py index 4f6371536a..a6288093bf 100644 --- a/deeppavlov/models/morpho_tagger/common.py +++ b/deeppavlov/models/morpho_tagger/common.py @@ -56,10 +56,10 @@ def predict_with_model(config_path: [Path, str], infile: Optional[Union[Path, st else: data = sys.stdin.readlines() model = build_model(config, load_trained=True) - model.pipe[-1][-1].set_format_mode(output_format) + for elem in model.pipe: + if isinstance(elem[-1], TagOutputPrettifier): + elem[-1].set_format_mode(output_format) answers = model.batched_call(data, batch_size=batch_size) - for elem in answers: - print(elem) return answers @@ -117,7 +117,7 @@ def _make_format_string(self) -> None: "it must be 'basic', 'conllu' or 'ud'.".format(self.mode)) def __call__(self, X: List[List[str]], Y: List[List[str]]) -> List[Union[List[str], str]]: - """Calls the ``prettify`` function for each input sentence. + """Calls the :meth:`~prettify` function for each input sentence. Args: X: a list of input sentences @@ -177,7 +177,7 @@ class LemmatizedOutputPrettifier(Component): in `conllu` or `ud` mode it contains 10 columns: id, word, lemma, pos, xpos, feats, head, deprel, deps, misc (see http://universaldependencies.org/format.html for details) - Only id, word, tag and pos values are a in current version, + Only id, word, lemma, tag and pos columns are predicted in current version, other columns are filled by `_` value. return_string: whether to return a list of strings or a single string begin: a string to append in the beginning @@ -194,7 +194,7 @@ def __init__(self, return_string: bool = True, self.format_string = "{0}\t{1}\t{4}\t{2}\t_\t{3}\t_\t_\t_\t_" def __call__(self, X: List[List[str]], Y: List[List[str]], Z: List[List[str]]) -> List[Union[List[str], str]]: - """Calls the ``prettify`` function for each input sentence. + """Calls the :meth:`~prettify` function for each input sentence. Args: X: a list of input sentences @@ -238,3 +238,56 @@ def prettify(self, tokens: List[str], tags: List[str], lemmas: List[str]) -> Uni if self.return_string: answer = self.begin + self.sep.join(answer) + self.end return answer + + +@register('dependency_output_prettifier') +class DependencyOutputPrettifier(Component): + """Class which prettifies dependency parser output + to 10-column (Universal Dependencies) format. + + Args: + return_string: whether to return a list of strings or a single string + begin: a string to append in the beginning + end: a string to append in the end + sep: separator between word analyses + """ + + def __init__(self, return_string: bool = True, begin: str = "", + end: str = "", sep: str = "\n", **kwargs) -> None: + self.return_string = return_string + self.begin = begin + self.end = end + self.sep = sep + self.format_string = "{}\t{}\t_\t_\t_\t_\t{}\t{}\t_\t_" + + def __call__(self, X: List[List[str]], Y: List[List[int]], Z: List[List[str]]) -> List[Union[List[str], str]]: + """Calls the :meth:`~prettify` function for each input sentence. + + Args: + X: a list of input sentences + Y: a list of lists of head positions for sentence words + Z: a list of lists of dependency labels for sentence words + + Returns: + a list of prettified UD outputs + """ + return [self.prettify(x, y, z) for x, y, z in zip(X, Y, Z)] + + def prettify(self, tokens: List[str], heads: List[int], deps: List[str]) -> Union[List[str], str]: + """Prettifies output of dependency parser. + + Args: + tokens: tokenized source sentence + heads: list of head positions, the output of the parser + deps: list of head positions, the output of the parser + + Returns: + the prettified output of the parser + + """ + answer = [] + for i, (word, head, dep) in enumerate(zip(tokens, heads, deps)): + answer.append(self.format_string.format(i + 1, word, head, dep)) + if self.return_string: + answer = self.begin + self.sep.join(answer) + self.end + return answer diff --git a/deeppavlov/models/preprocessors/assemble_embeddings_matrix.py b/deeppavlov/models/preprocessors/assemble_embeddings_matrix.py index c8a2af9653..35b0266d30 100644 --- a/deeppavlov/models/preprocessors/assemble_embeddings_matrix.py +++ b/deeppavlov/models/preprocessors/assemble_embeddings_matrix.py @@ -35,7 +35,7 @@ class also can assemble embeddins of characters using character_level: whether to perform assembling on character level. This procedure will assemble matrix with embeddings for every character using averaged embeddings of words, that contain this character. - emb_dim: dimensionality of the resulting embeddings. If not `None` it should be less + emb_dim: dimensionality of the resulting embeddings. If not ``None`` it should be less or equal to the dimensionality of the embeddings provided by `Embedder`. The reduction of dimensionality is performed by taking main components of PCA. estimate_by_n: how much samples to use to estimate covariance matrix for PCA. diff --git a/deeppavlov/models/preprocessors/bert_preprocessor.py b/deeppavlov/models/preprocessors/bert_preprocessor.py index 5275800e5c..ed6241fd7f 100644 --- a/deeppavlov/models/preprocessors/bert_preprocessor.py +++ b/deeppavlov/models/preprocessors/bert_preprocessor.py @@ -31,7 +31,7 @@ class BertPreprocessor(Component): """Tokenize text on subtokens, encode subtokens with their indices, create tokens and segment masks. - Check details in convert_examples_to_features function. + Check details in :func:`bert_dp.preprocessing.convert_examples_to_features` function. Args: vocab_file: path to vocabulary @@ -54,7 +54,7 @@ def __init__(self, do_lower_case=do_lower_case) def __call__(self, texts_a: List[str], texts_b: Optional[List[str]] = None) -> List[InputFeatures]: - """Call Bert convert_examples_to_features function to tokenize and create masks. + """Call Bert :func:`bert_dp.preprocessing.convert_examples_to_features` function to tokenize and create masks. texts_a and texts_b are separated by [SEP] token @@ -63,7 +63,7 @@ def __call__(self, texts_a: List[str], texts_b: Optional[List[str]] = None) -> L texts_b: list of texts, it could be None, e.g. single sentence classification task Returns: - batch of InputFeatures with subtokens, subtoken ids, subtoken mask, segment mask. + batch of :class:`bert_dp.preprocessing.InputFeatures` with subtokens, subtoken ids, subtoken mask, segment mask. """ @@ -77,10 +77,10 @@ def __call__(self, texts_a: List[str], texts_b: Optional[List[str]] = None) -> L @register('bert_ner_preprocessor') class BertNerPreprocessor(Component): - """Takes tokens and splits them into bert subtokens, encode subtokens with their indices. - Creates mask of subtokens (one for first subtoken, zero for later subtokens). + """Takes tokens and splits them into bert subtokens, encodes subtokens with their indices. + Creates a mask of subtokens (one for the first subtoken, zero for the others). - If tags are provided, calculate tags for subtokens. + If tags are provided, calculates tags for subtokens. Args: vocab_file: path to vocabulary @@ -88,8 +88,10 @@ class BertNerPreprocessor(Component): max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens max_subword_length: replace token to if it's length is larger than this (defaults to None, which is equal to +infinity) - token_mask_prob: probability of masking token while training + token_masking_prob: probability of masking token while training provide_subword_tags: output tags for subwords or for words + subword_mask_mode: subword to select inside word tokens, can be "first" or "last" + (default="first") Attributes: max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens @@ -102,18 +104,20 @@ def __init__(self, do_lower_case: bool = False, max_seq_length: int = 512, max_subword_length: int = None, - token_maksing_prob: float = 0.0, + token_masking_prob: float = 0.0, provide_subword_tags: bool = False, + subword_mask_mode: str = "first", **kwargs): self._re_tokenizer = re.compile(r"[\w']+|[^\w ]") self.provide_subword_tags = provide_subword_tags self.mode = kwargs.get('mode') self.max_seq_length = max_seq_length self.max_subword_length = max_subword_length + self.subword_mask_mode = subword_mask_mode vocab_file = str(expand_path(vocab_file)) self.tokenizer = FullTokenizer(vocab_file=vocab_file, do_lower_case=do_lower_case) - self.token_maksing_prob = token_maksing_prob + self.token_masking_prob = token_masking_prob def __call__(self, tokens: Union[List[List[str]], List[str]], @@ -135,7 +139,8 @@ def __call__(self, self.tokenizer, self.max_subword_length, mode=self.mode, - token_maksing_prob=self.token_maksing_prob) + subword_mask_mode=self.subword_mask_mode, + token_masking_prob=self.token_masking_prob) if self.max_seq_length is not None: if len(sw_toks) > self.max_seq_length: raise RuntimeError(f"input sequence after bert tokenization" @@ -175,7 +180,8 @@ def _ner_bert_tokenize(tokens: List[str], tokenizer: FullTokenizer, max_subword_len: int = None, mode: str = None, - token_maksing_prob: float = 0.0) -> Tuple[List[str], List[int], List[str]]: + subword_mask_mode: str = "first", + token_masking_prob: float = 0.0) -> Tuple[List[str], List[int], List[str]]: tokens_subword = ['[CLS]'] mask_subword = [0] tags_subword = ['X'] @@ -187,11 +193,14 @@ def _ner_bert_tokenize(tokens: List[str], mask_subword.append(flag) tags_subword.append(tag) else: - if mode == 'train' and token_maksing_prob > 0.0 and np.random.rand() < token_maksing_prob: + if mode == 'train' and token_masking_prob > 0.0 and np.random.rand() < token_masking_prob: tokens_subword.extend(['[MASK]'] * len(subwords)) else: tokens_subword.extend(subwords) - mask_subword.extend([flag] + [0] * (len(subwords) - 1)) + if subword_mask_mode == "last": + mask_subword.extend([0] * (len(subwords) - 1) + [flag]) + else: + mask_subword.extend([flag] + [0] * (len(subwords) - 1)) tags_subword.extend([tag] + ['X'] * (len(subwords) - 1)) tokens_subword.append('[SEP]') @@ -208,7 +217,7 @@ class BertRankerPreprocessor(BertPreprocessor): """ def __call__(self, batch: List[List[str]]) -> List[List[InputFeatures]]: - """Call BERT convert_examples_to_features function to tokenize and create masks. + """Call BERT :func:`bert_dp.preprocessing.convert_examples_to_features` function to tokenize and create masks. Args: batch: list of elemenents where the first element represents the batch with contexts @@ -250,7 +259,7 @@ class BertSepRankerPreprocessor(BertPreprocessor): """ def __call__(self, batch: List[List[str]]) -> List[List[InputFeatures]]: - """Call BERT convert_examples_to_features function to tokenize and create masks. + """Call BERT :func:`bert_dp.preprocessing.convert_examples_to_features` function to tokenize and create masks. Args: batch: list of elemenents where the first element represents the batch with contexts @@ -291,9 +300,9 @@ class BertSepRankerPredictorPreprocessor(BertSepRankerPreprocessor): Args: resps: list of strings containing the base of text responses - resp_vecs: BERT vector respresentations of `resps`, if is `None` features for the response base will be build + resp_vecs: BERT vector respresentations of ``resps``, if is ``None`` features for the response base will be build conts: list of strings containing the base of text contexts - cont_vecs: BERT vector respresentations of `conts`, if is `None` features for the response base will be build + cont_vecs: BERT vector respresentations of ``conts``, if is ``None`` features for the response base will be build """ def __init__(self, diff --git a/deeppavlov/models/preprocessors/mask.py b/deeppavlov/models/preprocessors/mask.py index ca4e3149a2..6a7cc246a1 100644 --- a/deeppavlov/models/preprocessors/mask.py +++ b/deeppavlov/models/preprocessors/mask.py @@ -20,8 +20,7 @@ @register('mask') class Mask(Component): - """Takes batch of tokens and returns the masks of corresponding length""" - + """Takes a batch of tokens and returns the masks of corresponding length""" def __init__(self, *args, **kwargs): pass diff --git a/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py b/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py index 4b362c24f7..66dbff0d66 100644 --- a/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py +++ b/deeppavlov/models/seq2seq_go_bot/kb_attn_layer.py @@ -27,7 +27,7 @@ class KBAttention(base.Layer): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: Initializer function for the weight matrix. - If `None` (default), weights are initialized using the default + If ``None`` (default), weights are initialized using the default initializer used by `tf.get_variable`. bias_initializer: Initializer function for the bias. kernel_regularizer: Regularizer function for the weight matrix. diff --git a/deeppavlov/models/syntax_parser/__init__.py b/deeppavlov/models/syntax_parser/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deeppavlov/models/syntax_parser/joint.py b/deeppavlov/models/syntax_parser/joint.py new file mode 100644 index 0000000000..78cf1244e9 --- /dev/null +++ b/deeppavlov/models/syntax_parser/joint.py @@ -0,0 +1,142 @@ +from typing import Union, List + +from deeppavlov.core.common.registry import register +from deeppavlov.core.models.component import Component +from deeppavlov.core.common.chainer import Chainer + +from deeppavlov.models.morpho_tagger.common import TagOutputPrettifier,\ + LemmatizedOutputPrettifier, DependencyOutputPrettifier + + +UD_COLUMN_FEAT_MAPPING = {"id": 0, "word": 1, "lemma": 2, "upos": 3, "feats": 5, "head": 6, "deprel": 7} + + +@register("joint_tagger_parser") +class JointTaggerParser(Component): + """ + A class to perform joint morphological and syntactic parsing. + It is just a wrapper that calls the models for tagging and parsing + and comprises their results in a single output. + + Args: + tagger: the morphological tagger model (a :class:`~deeppavlov.core.common.chainer.Chainer` instance) + parser_path: the syntactic parser model (a :class:`~deeppavlov.core.common.chainer.Chainer` instance) + output_format: the output format, it may be either `ud` (alias: `conllu`) or `json`. + to_output_string: whether to convert the output to a list of strings + + Attributes: + tagger: a morphological tagger model (a :class:`~deeppavlov.core.common.chainer.Chainer` instance) + parser: a syntactic parser model (a :class:`~deeppavlov.core.common.chainer.Chainer` instance) + + """ + + def __init__(self, tagger: Chainer, parser: Chainer, + output_format: str = "ud", to_output_string: bool = False, + *args, **kwargs): + if output_format not in ["ud", "conllu", "json", "dict"]: + UserWarning("JointTaggerParser output_format can be only `ud`, `conllu` or `json`. "\ + "Unknown format: {}, setting the output_format to `ud`.".format(output_format)) + output_format = "ud" + self.output_format = output_format + self.to_output_string = to_output_string + self.tagger = tagger + self.parser = parser + self._check_models() + + def _check_models(self): + tagger_prettifier = self.tagger[-1] + if not isinstance(tagger_prettifier, (TagOutputPrettifier, LemmatizedOutputPrettifier)): + raise ValueError("The tagger should output prettified data: last component of the config " + "should be either a TagOutputPrettifier or a LemmatizedOutputPrettifier " + "instance.") + if isinstance(tagger_prettifier, TagOutputPrettifier): + tagger_prettifier.set_format_mode("ud") + tagger_prettifier.return_string = False + parser_prettifier = self.parser[-1] + if not isinstance(parser_prettifier, DependencyOutputPrettifier): + raise ValueError("The tagger should output prettified data: last component of the config " + "should be either a DependencyOutputPrettifier instance.") + parser_prettifier.return_string = False + + def __call__(self, data: Union[List[str], List[List[str]]])\ + -> Union[List[List[dict]], List[str], List[List[str]]]: + r"""Parses a batch of sentences. + + Args: + data: either a batch of tokenized sentences, or a batch of raw sentences + + Returns: + `answer`, a batch of parsed sentences. A sentence parse is a list of single word parses. + Each word parse is either a CoNLL-U-formatted string or a dictionary. + A sentence parse is returned either as is if ``self.to_output_string`` is ``False``, + or as a single string, where each word parse begins with a new string. + + .. code-block:: python + + >>> from deeppavlov.core.commands.infer import build_model + >>> model = build_model("ru_syntagrus_joint_parsing") + >>> batch = ["Девушка пела в церковном хоре.", "У этой задачи есть сложное решение."] + >>> print(*model(batch), sep="\\n\\n") + 1 Девушка девушка NOUN _ Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing 2 nsubj _ _ + 2 пела петь VERB _ Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 0 root _ _ + 3 в в ADP _ _ 5 case _ _ + 4 церковном церковный ADJ _ Case=Loc|Degree=Pos|Gender=Masc|Number=Sing 5 amod _ _ + 5 хоре хор NOUN _ Animacy=Inan|Case=Loc|Gender=Masc|Number=Sing 2 obl _ _ + 6 . . PUNCT _ _ 2 punct _ _ + + 1 У у ADP _ _ 3 case _ _ + 2 этой этот DET _ Case=Gen|Gender=Fem|Number=Sing 3 det _ _ + 3 задачи задача NOUN _ Animacy=Inan|Case=Gen|Gender=Fem|Number=Sing 4 obl _ _ + 4 есть быть VERB _ Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act 0 root _ _ + 5 сложное сложный ADJ _ Case=Nom|Degree=Pos|Gender=Neut|Number=Sing 6 amod _ _ + 6 решение решение NOUN _ Animacy=Inan|Case=Nom|Gender=Neut|Number=Sing 4 nsubj _ _ + 7 . . PUNCT _ _ 4 punct _ _ + + >>> # Dirty hacks to change model parameters in the code, you should do it in the configuration file. + >>> model["main"].to_output_string = False + >>> model["main"].output_format = "json" + >>> for sent_parse in model(batch): + >>> for word_parse in sent_parse: + >>> print(word_parse) + >>> print("") + {'id': '1', 'word': 'Девушка', 'lemma': 'девушка', 'upos': 'NOUN', 'feats': 'Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing', 'head': '2', 'deprel': 'nsubj'} + {'id': '2', 'word': 'пела', 'lemma': 'петь', 'upos': 'VERB', 'feats': 'Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act', 'head': '0', 'deprel': 'root'} + {'id': '3', 'word': 'в', 'lemma': 'в', 'upos': 'ADP', 'feats': '_', 'head': '5', 'deprel': 'case'} + {'id': '4', 'word': 'церковном', 'lemma': 'церковный', 'upos': 'ADJ', 'feats': 'Case=Loc|Degree=Pos|Gender=Masc|Number=Sing', 'head': '5', 'deprel': 'amod'} + {'id': '5', 'word': 'хоре', 'lemma': 'хор', 'upos': 'NOUN', 'feats': 'Animacy=Inan|Case=Loc|Gender=Masc|Number=Sing', 'head': '2', 'deprel': 'obl'} + {'id': '6', 'word': '.', 'lemma': '.', 'upos': 'PUNCT', 'feats': '_', 'head': '2', 'deprel': 'punct'} + + {'id': '1', 'word': 'У', 'lemma': 'у', 'upos': 'ADP', 'feats': '_', 'head': '3', 'deprel': 'case'} + {'id': '2', 'word': 'этой', 'lemma': 'этот', 'upos': 'DET', 'feats': 'Case=Gen|Gender=Fem|Number=Sing', 'head': '3', 'deprel': 'det'} + {'id': '3', 'word': 'задачи', 'lemma': 'задача', 'upos': 'NOUN', 'feats': 'Animacy=Inan|Case=Gen|Gender=Fem|Number=Sing', 'head': '4', 'deprel': 'obl'} + {'id': '4', 'word': 'есть', 'lemma': 'быть', 'upos': 'VERB', 'feats': 'Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act', 'head': '0', 'deprel': 'root'} + {'id': '5', 'word': 'сложное', 'lemma': 'сложный', 'upos': 'ADJ', 'feats': 'Case=Nom|Degree=Pos|Gender=Neut|Number=Sing', 'head': '6', 'deprel': 'amod'} + {'id': '6', 'word': 'решение', 'lemma': 'решение', 'upos': 'NOUN', 'feats': 'Animacy=Inan|Case=Nom|Gender=Neut|Number=Sing', 'head': '4', 'deprel': 'nsubj'} + {'id': '7', 'word': '.', 'lemma': '.', 'upos': 'PUNCT', 'feats': '_', 'head': '4', 'deprel': 'punct'} + + """ + tagger_output = self.tagger(data) + parser_output = self.parser(data) + answer = [] + for i, (tagger_sent, parser_sent) in enumerate(zip(tagger_output, parser_output)): + curr_sent_answer = [] + for j, curr_word_tagger_output in enumerate(tagger_sent): + curr_word_tagger_output = curr_word_tagger_output.split("\t") + curr_word_parser_output = parser_sent[j].split("\t") + curr_word_answer = curr_word_tagger_output[:] + # setting parser output + curr_word_answer[6:8] = curr_word_parser_output[6:8] + if self.output_format in ["json", "dict"]: + curr_word_answer = {key: curr_word_answer[index] + for key, index in UD_COLUMN_FEAT_MAPPING.items()} + if self.to_output_string: + curr_word_answer = str(curr_word_answer) + elif self.to_output_string: + curr_word_answer = "\t".join(curr_word_answer) + curr_sent_answer.append(curr_word_answer) + if self.to_output_string: + curr_sent_answer = "\n".join(str(x) for x in curr_sent_answer) + answer.append(curr_sent_answer) + return answer + + diff --git a/deeppavlov/models/syntax_parser/network.py b/deeppavlov/models/syntax_parser/network.py new file mode 100644 index 0000000000..094f00f11e --- /dev/null +++ b/deeppavlov/models/syntax_parser/network.py @@ -0,0 +1,345 @@ +# Copyright 2019 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from logging import getLogger +from typing import List, Union, Tuple + +import numpy as np +import tensorflow as tf +import tensorflow.keras.backend as kb +from tensorflow.contrib.layers import xavier_initializer + +from deeppavlov.core.common.registry import register +from deeppavlov.core.data.utils import zero_pad +from deeppavlov.core.layers.tf_layers import bi_rnn +from deeppavlov.models.bert.bert_sequence_tagger import BertSequenceNetwork, token_from_subtoken + +log = getLogger(__name__) + + +def gather_indexes(A: tf.Tensor, B: tf.Tensor) -> tf.Tensor: + """ + Args: + A: a tensor with data + B: an integer tensor with indexes + + Returns: + `answer` a tensor such that ``answer[i, j] = A[i, B[i, j]]``. + In case `B` is one-dimensional, the output is ``answer[i] = A[i, B[i]]`` + + """ + are_indexes_one_dim = (kb.ndim(B) == 1) + if are_indexes_one_dim: + B = tf.expand_dims(B, -1) + first_dim_indexes = tf.expand_dims(tf.range(tf.shape(B)[0]), -1) + first_dim_indexes = tf.tile(first_dim_indexes, [1, tf.shape(B)[1]]) + indexes = tf.stack([first_dim_indexes, B], axis=-1) + answer = tf.gather_nd(A, indexes) + if are_indexes_one_dim: + answer = answer[:,0] + return answer + + +def biaffine_layer(deps: tf.Tensor, heads: tf.Tensor, deps_dim: int, + heads_dim: int, output_dim: int, name: str = "biaffine_layer") -> tf.Tensor: + """Implements a biaffine layer from [Dozat, Manning, 2016]. + + Args: + deps: the 3D-tensor of dependency states, + heads: the 3D-tensor of head states, + deps_dim: the dimension of dependency states, + heads_dim: the dimension of head_states, + output_dim: the output dimension + name: the name of a layer + + Returns: + `answer` the output 3D-tensor + + """ + input_shape = [kb.shape(deps)[i] for i in range(tf.keras.backend.ndim(deps))] + first_input = tf.reshape(deps, [-1, deps_dim]) # first_input.shape = (B*L, D1) + second_input = tf.reshape(heads, [-1, heads_dim]) # second_input.shape = (B*L, D2) + with tf.variable_scope(name): + kernel_shape = (deps_dim, heads_dim * output_dim) + kernel = tf.get_variable('kernel', shape=kernel_shape, initializer=xavier_initializer()) + first = tf.matmul(first_input, kernel) # (B*L, D2*H) + first = tf.reshape(first, [-1, heads_dim, output_dim]) # (B*L, D2, H) + answer = kb.batch_dot(first, second_input, axes=[1, 1]) # (B*L, H) + first_bias = tf.get_variable('first_bias', shape=(deps_dim, output_dim), + initializer=xavier_initializer()) + answer += tf.matmul(first_input, first_bias) + second_bias = tf.get_variable('second_bias', shape=(heads_dim, output_dim), + initializer=xavier_initializer()) + answer += tf.matmul(second_input, second_bias) + label_bias = tf.get_variable('label_bias', shape=(output_dim,), + initializer=xavier_initializer()) + answer = kb.bias_add(answer, label_bias) + answer = tf.reshape(answer, input_shape[:-1] + [output_dim]) # (B, L, H) + return answer + + +def biaffine_attention(deps: tf.Tensor, heads: tf.Tensor, name="biaffine_attention") -> tf.Tensor: + """Implements a trainable matching layer between two families of embeddings. + + Args: + deps: the 3D-tensor of dependency states, + heads: the 3D-tensor of head states, + name: the name of a layer + + Returns: + `answer` a 3D-tensor of pairwise scores between deps and heads + + """ + deps_dim_int = deps.get_shape().as_list()[-1] + heads_dim_int = heads.get_shape().as_list()[-1] + assert deps_dim_int == heads_dim_int + with tf.variable_scope(name): + kernel_shape = (deps_dim_int, heads_dim_int) + kernel = tf.get_variable('kernel', shape=kernel_shape, initializer=tf.initializers.identity()) + first_bias = tf.get_variable('first_bias', shape=(kernel_shape[0], 1), + initializer=xavier_initializer()) + second_bias = tf.get_variable('second_bias', shape=(kernel_shape[1], 1), + initializer=xavier_initializer()) + # deps.shape = (B, L, D) + # first.shape = (B, L, D), first_rie = sum_d deps_{rid} kernel_{de} + first = tf.tensordot(deps, kernel, axes=[-1, -2]) + answer = tf.matmul(first, heads, transpose_b=True) # answer.shape = (B, L, L) + # add bias over x axis + first_bias_term = tf.tensordot(deps, first_bias, axes=[-1, -2]) + answer += first_bias_term + # add bias over y axis + second_bias_term = tf.tensordot(heads, second_bias, axes=[-1, -2]) # (B, L, 1) + second_bias_term = tf.transpose(second_bias_term, [0, 2, 1]) # (B, 1, L) + answer += second_bias_term + return answer + + +@register('bert_syntax_parser') +class BertSyntaxParser(BertSequenceNetwork): + """BERT-based model for syntax parsing. + For each word the model predicts the index of its syntactic head + and the label of the dependency between this head and the current word. + See :class:`deeppavlov.models.bert.bert_sequence_tagger.BertSequenceNetwork` + for the description of inherited parameters. + + Args: + n_deps: number of distinct syntactic dependencies + embeddings_dropout: dropout for embeddings in biaffine layer + state_size: the size of hidden state in biaffine layer + dep_state_size: the size of hidden state in biaffine layer + use_birnn: whether to use bidirection rnn after BERT layers. + Set it to `True` as it leads to much higher performance at least on large datasets + birnn_cell_type: the type of Bidirectional RNN. Either `lstm` or `gru` + birnn_hidden_size: number of hidden units in the BiRNN layer in each direction + return_probas: set this to `True` if you need the probabilities instead of raw answers + predict tags: whether to predict morphological tags together with syntactic information + n_tags: the number of morphological tags + tag_weight: the weight of tag model loss in multitask training + """ + + def __init__(self, + n_deps: int, + keep_prob: float, + bert_config_file: str, + pretrained_bert: str = None, + attention_probs_keep_prob: float = None, + hidden_keep_prob: float = None, + embeddings_dropout: float = 0.0, + encoder_layer_ids: List[int] = (-1,), + encoder_dropout: float = 0.0, + optimizer: str = None, + weight_decay_rate: float = 1e-6, + state_size: int = 256, + use_birnn: bool = True, + birnn_cell_type: str = 'lstm', + birnn_hidden_size: int = 256, + ema_decay: float = None, + ema_variables_on_cpu: bool = True, + predict_tags = False, + n_tags = None, + tag_weight = 1.0, + return_probas: bool = False, + freeze_embeddings: bool = False, + learning_rate: float = 1e-3, + bert_learning_rate: float = 2e-5, + min_learning_rate: float = 1e-07, + learning_rate_drop_patience: int = 20, + learning_rate_drop_div: float = 2.0, + load_before_drop: bool = True, + clip_norm: float = 1.0, + **kwargs) -> None: + self.n_deps = n_deps + self.embeddings_dropout = embeddings_dropout + self.state_size = state_size + self.use_birnn = use_birnn + self.birnn_cell_type = birnn_cell_type + self.birnn_hidden_size = birnn_hidden_size + self.return_probas = return_probas + self.predict_tags = predict_tags + self.n_tags = n_tags + self.tag_weight = tag_weight + if self.predict_tags and self.n_tags is None: + raise ValueError("n_tags should be given if `predict_tags`=True.") + super().__init__(keep_prob=keep_prob, + bert_config_file=bert_config_file, + pretrained_bert=pretrained_bert, + attention_probs_keep_prob=attention_probs_keep_prob, + hidden_keep_prob=hidden_keep_prob, + encoder_layer_ids=encoder_layer_ids, + encoder_dropout=encoder_dropout, + optimizer=optimizer, + weight_decay_rate=weight_decay_rate, + ema_decay=ema_decay, + ema_variables_on_cpu=ema_variables_on_cpu, + freeze_embeddings=freeze_embeddings, + learning_rate=learning_rate, + bert_learning_rate=bert_learning_rate, + min_learning_rate=min_learning_rate, + learning_rate_drop_div=learning_rate_drop_div, + learning_rate_drop_patience=learning_rate_drop_patience, + load_before_drop=load_before_drop, + clip_norm=clip_norm, + **kwargs) + + def _init_graph(self) -> None: + self._init_placeholders() + + units = super()._init_graph() + + with tf.variable_scope('ner'): + units = token_from_subtoken(units, self.y_masks_ph) + if self.use_birnn: + units, _ = bi_rnn(units, + self.birnn_hidden_size, + cell_type=self.birnn_cell_type, + seq_lengths=self.seq_lengths, + name='birnn') + units = tf.concat(units, -1) + # for heads + head_embeddings = tf.layers.dense(units, units=self.state_size, activation="relu") + head_embeddings = tf.nn.dropout(head_embeddings, self.embeddings_keep_prob_ph) + dep_embeddings = tf.layers.dense(units, units=self.state_size, activation="relu") + dep_embeddings = tf.nn.dropout(dep_embeddings, self.embeddings_keep_prob_ph) + self.dep_head_similarities = biaffine_attention(dep_embeddings, head_embeddings) + self.dep_heads = tf.argmax(self.dep_head_similarities, -1) + self.dep_head_probs = tf.nn.softmax(self.dep_head_similarities) + # for dependency types + head_embeddings = tf.layers.dense(units, units=self.state_size, activation="relu") + head_embeddings = tf.nn.dropout(head_embeddings, self.embeddings_keep_prob_ph) + dep_embeddings = tf.layers.dense(units, units=self.state_size, activation="relu") + dep_embeddings = tf.nn.dropout(dep_embeddings, self.embeddings_keep_prob_ph) + # matching each word with its head + head_embeddings = gather_indexes(head_embeddings, self.y_head_ph) + self.dep_logits = biaffine_layer(dep_embeddings, head_embeddings, + deps_dim=self.state_size, heads_dim=self.state_size, + output_dim=self.n_deps) + self.deps = tf.argmax(self.dep_logits, -1) + self.dep_probs = tf.nn.softmax(self.dep_logits) + if self.predict_tags: + tag_embeddings = tf.layers.dense(units, units=self.state_size, activation="relu") + tag_embeddings = tf.nn.dropout(tag_embeddings, self.embeddings_keep_prob_ph) + self.tag_logits = tf.layers.dense(tag_embeddings, units=self.n_tags) + self.tags = tf.argmax(self.tag_logits, -1) + self.tag_probs = tf.nn.softmax(self.tag_logits) + with tf.variable_scope("loss"): + tag_mask = self._get_tag_mask() + y_mask = tf.cast(tag_mask, tf.float32) + self.loss = tf.losses.sparse_softmax_cross_entropy(labels=self.y_head_ph, + logits=self.dep_head_similarities, + weights=y_mask) + self.loss += tf.losses.sparse_softmax_cross_entropy(labels=self.y_dep_ph, + logits=self.dep_logits, + weights=y_mask) + if self.predict_tags: + tag_loss = tf.losses.sparse_softmax_cross_entropy(labels=self.y_tag_ph, + logits=self.tag_logits, + weights=y_mask) + self.loss += self.tag_weight_ph * tag_loss + + def _init_placeholders(self) -> None: + super()._init_placeholders() + self.y_head_ph = tf.placeholder(shape=(None, None), dtype=tf.int32, name='y_head_ph') + self.y_dep_ph = tf.placeholder(shape=(None, None), dtype=tf.int32, name='y_dep_ph') + if self.predict_tags: + self.y_tag_ph = tf.placeholder(shape=(None, None), dtype=tf.int32, name='y_tag_ph') + self.y_masks_ph = tf.placeholder(shape=(None, None), dtype=tf.int32, name='y_mask_ph') + self.embeddings_keep_prob_ph = tf.placeholder_with_default( + 1.0, shape=[], name="embeddings_keep_prob_ph") + if self.predict_tags: + self.tag_weight_ph = tf.placeholder_with_default(1.0, shape=[], name="tag_weight_ph") + + def _build_feed_dict(self, input_ids, input_masks, y_masks, + y_head=None, y_dep=None, y_tag=None) -> dict: + y_masks = np.concatenate([np.ones_like(y_masks[:,:1]), y_masks[:, 1:]], axis=1) + feed_dict = self._build_basic_feed_dict(input_ids, input_masks, train=(y_head is not None)) + feed_dict[self.y_masks_ph] = y_masks + if y_head is not None: + y_head = zero_pad(y_head) + y_head = np.concatenate([np.zeros_like(y_head[:,:1]), y_head], axis=1) + y_dep = zero_pad(y_dep) + y_dep = np.concatenate([np.zeros_like(y_dep[:,:1]), y_dep], axis=1) + feed_dict.update({self.embeddings_keep_prob_ph: 1.0 - self.embeddings_dropout, + self.y_head_ph: y_head, + self.y_dep_ph: y_dep}) + if self.predict_tags: + y_tag = np.concatenate([np.zeros_like(y_tag[:,:1]), y_tag], axis=1) + feed_dict.update({self.y_tag_ph: y_tag, self.tag_weight_ph: self.tag_weight}) + return feed_dict + + def __call__(self, + input_ids: Union[List[List[int]], np.ndarray], + input_masks: Union[List[List[int]], np.ndarray], + y_masks: Union[List[List[int]], np.ndarray]) \ + -> Union[Tuple[List[Union[List[int], np.ndarray]], List[List[int]]], + Tuple[List[Union[List[int], np.ndarray]], List[List[int]], List[List[int]]]]: + + """ Predicts the outputs for a batch of inputs. + By default (``return_probas`` = `False` and ``predict_tags`` = `False`) it returns two output batches. + The first is the batch of head indexes: `i` stands for `i`-th word in the sequence, + where numeration starts with 1. `0` is predicted for the syntactic root of the sentence. + The second is the batch of indexes for syntactic dependencies. + In case ``return_probas`` = `True` we return the probability distribution over possible heads + instead of the position of the most probable head. For a sentence of length `k` the output + is an array of shape `k * (k+1)`. + In case ``predict_tags`` = `True` the model additionally returns the index of the most probable + morphological tag for each word. The batch of such indexes becomes the third output of the function. + + Returns: + `pred_heads_to_return`, either a batch of most probable head positions for each token + (in case ``return_probas`` = `False`) + or a batch of probability distribution over token head positions + + `pred_deps`, the indexes of token dependency relations + + `pred_tags`: the indexes of token morphological tags (only if ``predict_tags`` = `True`) + + """ + feed_dict = self._build_feed_dict(input_ids, input_masks, y_masks) + if self.ema: + self.sess.run(self.ema.switch_to_test_op) + if self.return_probas: + pred_head_probs, pred_heads, seq_lengths =\ + self.sess.run([self.dep_head_probs, self.dep_heads, self.seq_lengths], feed_dict=feed_dict) + pred_heads_to_return = [np.array(p[1:l,:l]) for l, p in zip(seq_lengths, pred_head_probs)] + else: + pred_heads, seq_lengths = self.sess.run([self.dep_heads, self.seq_lengths], feed_dict=feed_dict) + pred_heads_to_return = [p[1:l] for l, p in zip(seq_lengths, pred_heads)] + feed_dict[self.y_head_ph] = pred_heads + pred_deps = self.sess.run(self.deps, feed_dict=feed_dict) + pred_deps = [p[1:l] for l, p in zip(seq_lengths, pred_deps)] + answer = [pred_heads_to_return, pred_deps] + if self.predict_tags: + pred_tags = self.sess.run(self.tags, feed_dict=feed_dict) + pred_tags = [p[1:l] for l, p in zip(seq_lengths, pred_tags)] + answer.append(pred_tags) + return tuple(answer) diff --git a/deeppavlov/models/syntax_parser/parser.py b/deeppavlov/models/syntax_parser/parser.py new file mode 100644 index 0000000000..eb18f97d88 --- /dev/null +++ b/deeppavlov/models/syntax_parser/parser.py @@ -0,0 +1,47 @@ +# Copyright 2019 Neural Networks and Deep Learning lab, MIPT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List +import numpy as np + +from dependency_decoding import chu_liu_edmonds + +from deeppavlov.core.common.registry import register +from deeppavlov.core.models.component import Component + + +@register('chu_liu_edmonds_transformer') +class ChuLiuEdmonds(Component): + """ + A wrapper for Chu-Liu-Edmonds algorithm for maximum spanning tree + """ + def __init__(self, min_edge_prob=1e-6, **kwargs): + self.min_edge_prob = min_edge_prob + + def __call__(self, probs: List[np.ndarray]) -> List[List[int]]: + """Applies Chu-Liu-Edmonds algorithm to the matrix of head probabilities. + + probs: a 3D-array of probabilities of shape B*L*(L+1) + """ + answer = [] + for elem in probs: + m, n = elem.shape + assert n == m+1 + elem = np.log10(np.maximum(self.min_edge_prob, elem)) - np.log10(self.min_edge_prob) + elem = np.concatenate([np.zeros_like(elem[:1,:]), elem], axis=0) + # it makes impossible to create multiple edges 0->i + elem[1:, 0] += np.log10(self.min_edge_prob) * len(elem) + chl_data = chu_liu_edmonds(elem.astype("float64")) + answer.append(chl_data[0][1:]) + return answer diff --git a/deeppavlov/requirements/syntax_parser.txt b/deeppavlov/requirements/syntax_parser.txt new file mode 100644 index 0000000000..053781647f --- /dev/null +++ b/deeppavlov/requirements/syntax_parser.txt @@ -0,0 +1 @@ +git+https://github.com/andersjo/dependency_decoding.git@79510908223b93bd4c1fb0409a2a66dd75577c2c \ No newline at end of file diff --git a/docs/_static/tree.png b/docs/_static/tree.png new file mode 100644 index 0000000000000000000000000000000000000000..9fb57fb311a82290789c4c542d5be479771cc839 GIT binary patch literal 12108 zcmaKS1yodR*ESMT0z*qT2uP2V2$BL4D&6TYgn)FHNDYl33@I&L(kU%DbR*q4bbg2D zec%84*1x|0U#!K<+~=J8K6mWBuWRos^qtBZJRC|K6ciLZ1$mhdz~=x8$|GW|$H2Gv z`+xux6nYc|8A)}wl)cpVPHK}$2kxV@DK#m1TkeWOef=rdyw+Pk&JCRh^b@pUed7bl z1YvPg4yKroxIzS-!Pr|TBamTU@&fD*jT8-z^+-pIQw|Q2q9ATYYAI4IfiP(zJUrRg zROr|VHa4K&Q!Ur_%sLex^%6cRB%IlC6@Rp*OGT!7pWW{^wE%yU^dPK+4!8#uJ*Y@9 z5Q-ka!YIQ8dQQkd>2Dm0g}_38D-$9|ugxe0Mr|V^%p}B02mwXBE|W=Mf&cu;E+;3K z2!|IWyh4?1S1X75cpft^x{6LC=MP3I-31Qr)`iBa-qJ(0Ifw?nR$5%X3I0;PZ#z{L z3j7zt$0Q$0Ev)ti9xXRV=+`-X6>{12Q%+v@!mriWZ}MTF{qie$ zt>^i^;uJdw{Df2lmFP%#_NC~>`JjqieZwX2kwPVOd>l!mq@toZE9al_BY$emYj5Vo z-SmpdP+@7sLAS*ES8~JGlg>9jzN7=hSd>u(16cAoA8xKDTh=d*R_1CyXX)mVVo^pG zsARbx{(aPI9;?pLne`^Bz}?C}!zBM|t}-+fJGPhE%*;$lRaNWy1XXhWXFW7GFOPzX ziYZBnUtHX9Zxk2Z*+Lc~BH?ioqRt`9pS0Tc1+!0BEli$XrcpNt8(kx#AXA}JDlpR$ zg;v}v@nBJXKL~&2-EN?t6Ig~fp#WZtDw(4$3AS7|p;IXg%bY%v9R)dOS*aM_Oia4( zQr)+9XVqMs+&ddcCG3+NZW$3+G~>3DP0)~=p__8?`|}8H z5;LfXz7M1v25tRfp!Ng1~ z)>~D91Tgb4@M9f+T@sRkpqC)ou~z}Qz|r{NgB!cuO2_ry#>`f|SMu2>^a6vsUf-@0 zAqgh;`{tN5eB-$17evvAI=nh)0s)-z$(-;xXf-$~J zA5AXzBwa%;cEBx@)AZ--GIOiuK(*7y;urx}@4h<^@h62RPO)qM8v9h*oOOgu0)MiNXw2f4qyy*d8gLL!ifQ5aOu{q}n8yV1z}d_>6(wTMG@S}qp) zN44``DH4~f#DcbSJzWu0mxt{HJD(z7mpyYP$7_vYdNb@y6WreR-N;AB&JjHT!x9Uj zq^x`tb$=1HSsyvOM^*xrlpY0TK4Mbx=u6<*tRJMZYkdE3tGne>1j@}4f$!fWi!{ke z=if_ZQ-7p~qru;xD8H9@>q|zD#?*E<_sR6!0b2ZY#ZpsK(Z5LYCn+^4lT~5ial$2lc~DFEq(q5dJ-eR=7NfT}j!nep zM);uRUVLLPbp^2dhwH8j?!nfRC87TB+QLT5GJE&Lc^e)lrKI!+zm1BDLLM$jn@RpQ z=Wo&T&v!=WRA>)ltwFy2X}|f~`SJcjQ{Tx6I7x~1K6&oklvBD*aX30NDJ^ZVl!2En ziuRSMlM~peVHN(7fL5%>H@z^TLliy0Y6Q3^C52gJb@`~frB8}eXn-wcYs+$E%c)sV zku^?9N$L6AVp8RKhOiw~`y+Bi4?T*YVd2cy!qV6f zyUV4hNd{?jJXz^5h>cD7YBcURvl+otDFt+!o;rBO>KS!Io!dEgud5*PSGTJ@-(*y1 z?K^d$_kR(c-!HDr?o)juSQHj+{L*js%o3aLyrqr#gbEbeXn9rOhk7ynhFJ>Lt@{`O z6In|#ajvM1HE0@w)jKHkG##u-M_n6>X)Ad`c@z>&@e;msot||xQ#dklhdR+Vl3i?U z6m)bO>#f8$(<|JAk&&^N7uWd}U+ni&-V0f&WJ`eqP$^ZwadB}$eHXU}nFS>Q;>zx` zj!O4~os+cp+|qT$xQmMk=_IFwY5_4TE(`IpWwHJM*pO$vpgXK(3=-!0Ly?fmr zx(>K>z+LRiHf!Ci4^4n!wnH(+I)f2ZFG;d}ZmBC~q)9?N2jtI;n}ZaCcjXI`ag&~y z(e3}`x$_b@SnYD-J_W+DB48W6L+G`K-s*7yOy=@Tq*Y8CR%u_WH$pu5-U}aJ@Evhdpm!l^P!&O+zihL%GNwVkXk9Ci|Rn27HxFFTJwqO-E?#EQu=_ zGU6yaB0^(aux&f*hYYfpEq!$(LL=EpRGI(mu`hB;=prxb!|SjzHCdNH^0^>;E2}Ud zfUn6;E-Vc4_#ENxZwStlR#D@zz>RsK!jJan^e+t!0Gs$WYMj4Ryyx!0$xIf zS``=ZF7tDpJ~9YRJr3}yWXh{WflEs!bGELAVm!?kjO09qMHM*#a@i&_alPhKwV%z# zQ&EsdhfD35#?^{~@$j^?b=Q__n%Nq=A=t*amJtgX=bA1EjElzvR~5fRH<&EkMVBYW z)=za+yx)B`tmtayd9q2^a=T|YU1h0eC`KCp=aI7%xteL(u6(?F-)c1?Txh(o#%53cu~KvTtL*{n}>JM~dHiaUpzr zdwWR(nwgooxE%83?s9_%1`$T|tw~Kfw_}57q~fpa(JNC=reyn%UrRoF4br)8efmxR z85dzEBbWZ@Y9bGglSp`Y`0(Up@VxDtNoT&UQD$%;)Ir4$@XK8njDmq2cuOT)9^eIC>CNh1;yLjyNkS1CazE!BK2r} z0-Q6#L5TOXWHp`$KZGxBTtzyBbpstMxc%WK>BibcqH_ZJWFCn0BQp872ERG*)WJEv z9v-4CSSB3g7`n0@PlvX17-ZrcQE#?XefgUcx!^jq{NrNs^qMoCnkg=r>A%Po_Di9P z_kd$xsWYx71^qLQ{JgAys3_bO6>K`|Q*AGPcdv_4RXQrpm_6F$5Yjn53Y#*&kHmSM$ z`V!(NkF6b)frS?y(*WnBqSPFIawqC5uk*BGm3k$w zrve&EX{&ajv2GT*9t{$}`6e$^Y$rYI{7v4rrY*os$y&%*_nJA(TEc&44WqRQIr#Dt zb#8IdDE{UdCSRAD?=}bgG6f}cSS6e5jfL_SUE78Sku{WA@2~6MR37y@^{Wfg##f(u zu7e`jg({quFvP@QkPqbcoo9lNZCM|`2G z1g3_I5!Xl@JC44AWT9yQ6D_caiHg=Y5_1y+%WSaC3O|OR&i$_Wh?dL67B)YGj2yM$ zO~FKlaEOFH%w`bKW# z#kZITMPKp*EC%11aKH*1GoK8=D2>N|INg`@8(fMPSl0;6#NOs{XX+>ePcf#dR9a7R z17@>iZ-05b=8>!(_;iImy*>o;>noZ%4Djg{7DKd71{>2B=?3M`hS%u3+{l2ZEiu?y zy~Kd10UWVc_ku>fz@@`Za-+XTZh+lkTXDlR9Z>tbPfnZ+g@1*h8sL{}hd)d&{D|b8 zBccjUY%C6tI`^I88Y@b`iE$e?CZ3~KUp-YFkR1(E@%#GhNqUI}2{Z>l)O=uY$3(H7 zo=13IZssN6+%B(M?ynP^P@l_~T$}(n9EGVg69JX3_hDmW6LCK__lcbM0&LuipzjAH zbIbGA*>HQ?bg^=%!hJ>V?0!^Ye0!uY;PpJqCS7TmHH*5g<2 zZm(~a$y+j-n!E>}s_wfBUAUmcfY<;fI@g}>W!JKt}Z<~=Q^tn$g*$;~20*ZYTK23e_=;)|04L&R6<0@2(TrbVD zB)6~zEDE%W-S-O=ik9jwxleWwMyupa=kMGUG^Ii<*jlzX|8C`_73e+j&f*snOG!um zf+Gv2(_NLjEqyNgj2y~pz5_&k;<%zPAX&(61c8V#daO9<+JaS8Um=@v<-BGv1UHL*Twj#ZZngu`|#9#02&o{^>%YWpLFAZbT772uI z_b)VgB4Ku`qP~S z>xm+h0ZM^|3r5WDIcOMc{9ro0ui>B>Rr3;2_-rcB%cu`)pDH+v=JrFj&XQ!9(6vL-1x94s{Dq^gJHLco5|kq%V+{;+_| zXU`xn@1PUubdFg+@f&G5;fzeZl!NbFFj@hY@3`zdTc+EeALh<<_x>yX5~SbfRL z`SnINm5N`o*G=hOA}NKA=J2jh_;Uqf&LpJ7pzY6NnE&)u2xZV1FnOj!2zuRyq z&MSd%-?c$-=Z$!&UfJKM1Vk+n^^=06M?fq~qqPA$rAv@@ox)}!=ZGNj@OcU~J zq7T4WWqW>Rib`dpEQ>#{pin)NB{|oLF21V}Dk196+0RtCc%E3~pOj-f`p700!w6OS zDS-94xwhp7UQCiHTEAvxV?&m#0bZv(`=!VRdhNtD9l!*6)cuij#oC56v0YmKfh23A_ZAplDHb#pSR%2Qhp z8N{})JZK@=P~eb!9>Ypn#%O8lV9@N;s{5RBT8^H!D~Dt1OIN9y4`(cwcD(F7#^V=- zmW(PGz1vI)D#Yir(T*5mhI4tL*Tp>$yJpqO8Cl{`nd2X=FlE>fz&sksD{b1y-=blo z3TCMipC?(sb$>oHEDUF#jN-IM&t~FQpH|_OtX4JaBo{J|3-kC~8}MRtlsN?biwwLB zsot=z!x7$HKwqr$C0qMQ#^C^AdXQx00tz8@8`3O?avL@GkE%)lIg1qG&!&hK3ka%` zIZ{R{)*Re0ly5-9kOB`cf#+ z5vGI%RL0QC$Abh=HAAE(9Yp#xaz3#05gW(T%T%JyHMsH|=-wfseAFjClmIZSA_*k` z&LpGu@!aioY^3n~D?*!)O=NYLV22f*RjKQK3vXz-UWht$VomG$&ex!+6s0Kzbs5|GMgfDDv zU5@_qTDDy4InlEUChyr^f0=hRi)JON)g2WH9$=$nRmicbIT}uV>3UqU2GDd(r^vZ* zf7w(NJg3Eoc0B4opB0rwb=j?$;G3WV7o5_UA>$)a1t%*{l%HB|&0g#VzxyUmQb#aA z_Tup|W8QllT=}K@hv7j{JG=!WFy`}>KdGfbc5@H(>f8o^3|buf9W7@w)|>Tdx3|pY zt@Mzbp&nx7=Nm-PQ_9t5auyFt4VsQU>Q+b7?D{6(m2gEc-DEMQ}P5u zW>NB!+_SZ3ScX2NC`Ay0HkNaPA~9rrtx5TlOgn3YRJ#23X)(6pD)A*M0#^4w6oAAi1Jj9vu4 zt4vD7PnXgRaFceGCmH0>R{|f?*L^47S~QEhiV%*YBu#5>ZWcMZjel30%J}-X>(jUw znjjodJ1dPKYnfoH_jg&R2_$!wBOjt)P36N~yL|O;uO#rDEd&9r6Be@ z)t3PykC5$xC-{-&5ha`G(Yxv%f>^@z9^|W9a1_=GI}X>ol#~myCLQx!5nS z!mF)r7$NK-gk-U-yJ8_f^HtS`A^5a!2Aife%MJlN#dV;|O}c1=EyC7%eb8d(MaZ*1 z+Hs54vkt@XhR-aX$LjP9eofnv;+ebv2%w2qxq@+kAJ=T2!UJal=4GXqIc#?% z%|&)d%f0;wU+_xyxMRt_3hX56*n9OgfqcMH{Szq48bx_eS`&X|(1lj3;1G!Zn0cdh z#$LWG0C)4f8cnZQB=HfE|ATYHIG2 z$5Oyn+uz+>GP1Qgl;IyVXFDJ`TJLp z@=ok*Zez;Donqn&I;aRI5V{T^9^n=zn}>tLJBdxNjW;F-P7;`!C@~ula!kjQ-+TEv zl`e_RnR#3c?B(wMvUFDE-58V>MRD`bZClA;+AL}(sy-r0iMWV<(kvP zknJ^vU_w0k8d<+=sm3GNtKfp^1O|r;s-NDjf^_`ZCB7jwd3e3frkb^_7PY%Hwc^Dm zxsd!gD2naU-i+U#4J7SEpEZq_)sZ#Q)KKF5X{~>rHQKsa&7X$BtNDw_Q~L81xXv}_ zHT}M->c;m-14ZCm08CN4L(Z&OgTnp2OPl_xZhYC4V}8OXro6JE#jtCSBljMChNqNi zzpC}rLe?Lrm9Ui$vB8gq4(rooUfE;#oopE-)<-5h5e#c;A{`wR$nL8H216GRI&^Y2 zmPq%un#TNWVVarwu)UMmWg!ukfBIo8B>i7YGzftia!eS+Tu>ei;uw=a%CQM%pEN}grQ!X%u0;gWsqq)0gI5$MTbotNfr(C4z55 z@u?AWsTNptn{rk>Cr=F+{!WLDIS=<_u!OB6vaTCz7JJRED&>98li|fhEx@SLU%!SQ zp_aRnxJQr;;PMBaJJUBrf@phDD3&(>I&1W<=TE&R#=}ZN^n&D#gjlb+8xeZhJ<*C@ zA$fUtbsOpe9z$fRQ3SiMIjn-hL9|m5)8YbxCPyiW6n#1Y|0&L0cJr#<4>>2g1X;7f zF)UE~*`K#w1PiBWTR0sejksgUQIhn~@pomse{Zt=F=(()Z_;Q3>@hR*XYQvJnaI7) zk%b`3y~o*X)uuZyzaQWZmCeYE8Us~8$Mb!I|5Fm2Jh?JuZQaggkrm+Mi!u7qGQp|4 z(CkfRFuAq8U7)KJnNeMp0(EGJ0N@Ky`_o45wA|la1I*dFv`G_bjawl7HgtX7by0Tm zwBW&GjsJdx?ws{e#t|UDH$`h!KwAJ7ruJ=BAQZ=yr4Ry~IxM!K(j>;Ss3M+IXlv%qo5x$Tcv6}a_lqmAkua+a?- z%LmpRu-RS{4}&v5+s$_QbLj~q5N)dU^_%_NGz}ksu4c|=)Krf~YFZ5J;?Vmt*kx80 zsv{inF>PkP-KRJUPt8p0*p06L+9Um52yiM6^Ksbv%_Wk0bDx{zTiwu3vXJA^vaCsR zMnRDWB1OzSmY9Zy1$VtSGO}K9h;mv`at&zLueJ9VGIaZ4-X`{LqsE`gNM>j z%VqcPZjnzrMer9oulws$!QvmEh}GrPLLUDm{SKQ-Ot z&F-Bs6qNi3j)0Hj2+QlbABot zvaTOf-^(X&9ejE6S(TaLV7?X_<@fJMC^~nUL8VbFK<*PbARQg(pl-W3zT%i@4XV2f zRF~YWi(SgF%#0o!_`n;m!Nv8js#`W}J$uL&(y1r1$eq&c%M95Mcc3<67Sr#xf@1j( zuPT@>ND}Gl?smW0%0oDNL!|S8BJa0icCs=7nlc8R+v(Fn;y`^tu^1=B!HS_kELZeA zAH8x9Lza7>xs?JkWZ1)lQgFv-vLOknmlGga6IlOn!%H%d}5 zP_haE<>#ASw(p~vDJt7l!LP2*vAxK!3ngSlcV+lIzkbQ+0`~C3BhJ`$K^%%`d~IuP z_#D|bz~gyb1jtFQD)y9Qk90rwFI=R+IRFZ(y-uFM3l9SS!es6MU47*ubs zIRw(W4^K6DSp124F`KR1GnhCTHcs4o*8U-oCJbT3e$f;|(#p}>lWD0*4wnNnO9{WA z&_N6raR(4e2Dis1r0+RE_HM0ok zRG^-bxv-99IIbmT-1OGQk3BO$%l|j|wJbV3-pF;HIOW<~(e3AXTQa!2S zz~CJ_7dM+J-KqyPa!EVr%GH1tMZihQ`vp#@yGTak6}0}Vs1FF5m@HQ z4_y`FaJ2B{np0EVN;)09R>uO%kbKdba{qqzfB?nTo#EUy`JUf_MVBiawbYL)wPIx49bG##W#=&aZ?=Vl{8;p~C{KNz~@b|I`3^W#Z-^%3AG-o}Aw&z%keNPn;c zhC(A((FysD^7~8AbU@rOom@p1WZfmlR6B>>AKXVY(8trFe^-LO)te-NCaIM0Ht1D3 z19*+3P6YS;xg^O(#IA46=R(ZSZz8S*ba3~1XaKs=^qgL2fzyiVvoCcC{$uR-b|&*SdTE>KCqvWagxf&arYc=+3GXXbm=4OwC&wUqPP z+(ZymA3=29twDXvAXfhMz@UsD{1r;n?87{JV6jmB<2M|VAzZu}93Ki90_JuAO~83V zzXsQTEy`PFj##yVdwtSTAMN)&2#5#fD!mF;Y&MXPY+ZO+s4C#cV+)S?fPe|>Pvg66 z6KPr+L(c3eR+sPhTwXFUF~Ml*&QFKH1Zp)$x}JPVJe0Qk{IWom1;`jYR9Ps!;a~R+ z${4}HPM2~&$dnU7u00G^3jTYl^yUX9eX6UH0ODVFK*2fJC~mNs!_P^;$8?y zXExF?mn@HY#KcQOCq~hkzL)hjLHBw7kPkcu@(8rdpyZ=8E?{@IN0~E!;(a~3H~piu z1GrXnhkw3W0~DVXXhr02@>TImfj{!w-Lzz^fil^@{Jyxi7uE;F49;>Pxle%GVJb<4 zRp&f!<dMVj<5gmGwM5Z-Es%SkAc&b;uLd$u^vt-MOOsz z++{YRcgS~wVNc8~B%gW~2Fm;?q|60}E^20{sda^~16c1R;-3QfLQ{j-Y}b(SuY)wp z$VcWupQl`xhbH}Sr_&f*&d}R->K4=5G%FqxjfJg~D6f;nm^HkguE)VqO@;5B=xaG&?b@z#y$Stpc74H{ zmWH2gS0Y_;(1G0g!{g1Fn7$5Dj<53sR1D^^fbL=p2>)n|G6JyPUIhT`{Zv;}>L>Xo z69OUADm7foiV|mx0qwI(S^}`Fr}KW&s_#kLwoFK_pZ<+6S9L$A$S@FE5pv-xS%-x% z@9n%ln6AuQ$i8-K>DBqLPJBl*^Ll#+ppXB zGsQ1W&kI#R09uc9!G=^LS0~Wk?y8&TOBVwRQ`1ZUYH2>=VtbJBrKG0L`7U8hRk7bE zZSF&mjzfGeUOdQc@KpLj*Kf}kyJzPOm=otLjI_vGuICT7KV@dLS&BSl^p6YA&SEn! zH9TK_`SRsd@cr;#0IUWD2XDFUTf5JR!doii z{L^oX{`C^pvCBkeCJX(1!f2ipQZ|~C4kv<5x@Ab zERI_-ZdZg!GKV*ng zKz`~dh2~7rk4^tkziCObcxf|p`jbTCZwxU&ZFRN49fweyL9Fa{m2di^{r{c#CG+}B zzb>byj{%EfJJew=x>HAppbseH!XBH_;*FCs?5dYtETd%KXfo^BPjKAV(e;zzF$2E} zqcYU^;I09=c{#Gu{aOH)I zHE)tPNy!{FfoOR?P8efWzNXlc4ME;F=mQ?%4b9c*8(z-*`FP6)sq#E=fddQiJkF^f zCR?t?tIq^=2!tj`cH}Sum4uA^kGk0ZQ$q^j%aqRF1|c--N{rtJLdm2<$O{pl<;Vv#n@@?!sQT+r>$ z8jA6lkJI}Z03$Ai?>1K%CrQDS#q977mggUQUC@1F%PI+`KaVGh5(6TJFHiBy!3=Nc za&49(Jg+Sx&0?dofQ%c7sdRqsms-ZSR%Xd-AEg-(0vvt!>TK?pDLG#s+T}E6=s$u7 zEEes*^#NU(FJ#f8x%2r@A7Xp9U*G|6ESI3b`E#+56wbyYAZ^*U+rJ590aKm=+6A@C zB-H;a>N_SEVQi>$mnj4vv5?i3Wc+}>M}!kOVe(lA4_8S;|J*e~P1B@%A`1nA$xgZU z|01p`1hQ0rB=S(IS(r*gx3~Veo?kN8ocd3!-&p6szXYXCnF2E>;$nxZ8>6sKdZCW^ z`v<^t>3YwulZ1ToFCx;3Wu~TsJHH#4mnl8O`I}Y$OgLm|2Zu5^fkb(3GU^*=Q z;-*B0Uvdv&-;NIm`+9RVA>Mwuc*Yt8L;d00YG65}ZSH>u^v)7a=u_Pf;QuW=`+*s> zBr3*WnY3MNuQZskvO>>%d6TjP1hV=_|7YO#GLXy~hCir`{r^A4-6`w%kei69$0v&l zHXk(BYlwWYsNW6$Ma2uq7$%34s8Sz{Qm9*y((&_y9^Y)}G9agHwXA++kf>2PK@7RT4Qo zk{L=p74`xwwSB@p=G`u!rVW9yzV$NhR^sCKc|3bn+}X_xGt{zTAhv%^U<51UKdA((HSU+`ywOAgOf(kpGLyI!MTG-daexU$Xa3r9;*I)rRLrF5BK1;oLlc)s-az65}^1>3z zBF>z%cBxo_1m3R$MZ9qGv1H&JENc@SmpE2l(E@KI_`KUT{v=H{a!*dDh4g%*5^gjZ zl`-vER&~(o*d%y{yZsh7tm|oVd1~6uvSRvtMC01Cqu-z5b|vjq)p+B-=oa&G=1c(@ s%Cw(C_(0VYu4jP0-2H#(%kSw}SiTXKU7W@ODtQzISrwTQDdT|u0v>kT*Z=?k literal 0 HcmV?d00001 diff --git a/docs/apiref/models/bert.rst b/docs/apiref/models/bert.rst index a9222ef495..b9510ea17f 100644 --- a/docs/apiref/models/bert.rst +++ b/docs/apiref/models/bert.rst @@ -29,11 +29,15 @@ deeppavlov.models.bert .. automethod:: __call__ .. automethod:: train_on_batch -.. autoclass:: deeppavlov.models.bert.bert_ner.BertNerModel +.. autofunction:: deeppavlov.models.bert.bert_sequence_tagger.token_from_subtoken + +.. autoclass:: deeppavlov.models.bert.bert_sequence_tagger.BertSequenceNetwork - .. automethod:: __call__ .. automethod:: train_on_batch - .. automethod:: token_from_subtoken + +.. autoclass:: deeppavlov.models.bert.bert_sequence_tagger.BertSequenceTagger + + .. automethod:: __call__ .. autoclass:: deeppavlov.models.bert.bert_squad.BertSQuADModel diff --git a/docs/apiref/models/syntax_parser.rst b/docs/apiref/models/syntax_parser.rst new file mode 100644 index 0000000000..3884daf8d2 --- /dev/null +++ b/docs/apiref/models/syntax_parser.rst @@ -0,0 +1,16 @@ +deeppavlov.models.syntax_parser +=============================== + +.. autoclass:: deeppavlov.models.syntax_parser.network.BertSyntaxParser + + .. automethod:: __call__ + +.. autofunction:: deeppavlov.models.syntax_parser.network.gather_indexes + +.. autofunction:: deeppavlov.models.syntax_parser.network.biaffine_layer + +.. autofunction:: deeppavlov.models.syntax_parser.network.biaffine_attention + +.. autoclass:: deeppavlov.models.syntax_parser.joint.JointTaggerParser + + .. automethod:: __call__ \ No newline at end of file diff --git a/docs/features/models/bert.rst b/docs/features/models/bert.rst index 50e85ac8d0..2f5a6a3feb 100644 --- a/docs/features/models/bert.rst +++ b/docs/features/models/bert.rst @@ -59,15 +59,31 @@ it is followed by ``softmax`` activation (``sigmoid`` if ``multilabel`` paramete BERT for Named Entity Recognition (Sequence Tagging) ---------------------------------------------------- -Pre-trained BERT model can be used for sequence tagging. Examples of usage of BERT for sequence tagging can be -found :doc:`here `. The module used for tagging is :class:`~deeppavlov.models.bert.bert_ner.BertNerModel`. -To tag each word representations of the first sub-word elements are extracted. So for each word there is only one vector produced. -These representations are passed to a dense layer or Bi-RNN layer to produce distribution over tags. There is -also an optional CRF layer on the top. +Pre-trained BERT model can be used for sequence tagging. Examples of BERT application to sequence tagging +can be found :doc:`here `. The module used for tagging +is :class:`~deeppavlov.models.bert.bert_sequence_tagger.BertSequenceTagger`. +The tags are obtained by applying a dense layer to the representation of +the first subtoken of each word. There is also an optional CRF layer on the top. Multilingual BERT model allows to perform zero-shot transfer across languages. To use our 19 tags NER for over a hundred languages see :ref:`ner_multi_bert`. +BERT for Morphological Tagging +------------------------------ + +Since morphological tagging is also a sequence labeling task, it can be solved in a similar fashion. +The only difference is that we may use the last subtoken of each word in case word morphology +is mostly defined by its suffixes, not prefixes (that is the case for most Indo-European languages, +such as Russian, Spanish, German etc.). See :doc:`also `. + +BERT for Syntactic Parsing +-------------------------- + +You can use BERT for syntactic parsing also. As most modern parsers, we use the biaffine model +over the embedding layer, which is the output of BERT. The model outputs the index of syntactic +head and the dependency type for each word. See :doc:`the parser documentation ` +for more information about model performance and algorithm. + BERT for Context Question Answering (SQuAD) ------------------------------------------- diff --git a/docs/features/models/morphotagger.rst b/docs/features/models/morphotagger.rst index dabe43b0b3..e1f962d00a 100644 --- a/docs/features/models/morphotagger.rst +++ b/docs/features/models/morphotagger.rst @@ -1,40 +1,61 @@ Neural Morphological Tagging ============================ -It is an implementation of neural morphological tagger from +It is an implementation of neural morphological tagger. +As for now (November, 2019) we have two types of models: +the BERT-based ones (available only for Russian) and +the character-based bidirectional LSTM. The BERT model +includes only a dense layer on the top of BERT embedder. +See the `BERT paper `__ +for a more complete description, as well as the +`BERT section `__ of the documentation. +We plan to release more BERT-based models in near future. + +Most of our models follow `Heigold et al., 2017. An extensive empirical evaluation of character-based morphological tagging for 14 languages `__. -We distribute the models for 11 languages trained on `Universal -Dependencies corpora `__. -Our model achieves the state-of-the-art performance among open source +They also achieve the state-of-the-art performance among open source systems. -+----------------+--------------+-----------------+------------------+----------------+ -| Language | Code | UDPipe accuracy | Our top accuracy | Model size (MB)| -+================+==============+=================+==================+================+ -| Arabic | ar | 88.31 | 90.85 | 23.7 | -+----------------+--------------+-----------------+------------------+----------------+ -| Czech | cs | 91.86 | 94.35 | 41.8 | -+----------------+--------------+-----------------+------------------+----------------+ -| English | en | 92.53 | 93.00 | 16.9 | -+----------------+--------------+-----------------+------------------+----------------+ -| French | fr | 95.25 | 95.45 | 19.0 | -+----------------+--------------+-----------------+------------------+----------------+ -| German | de | 76.65 | 83.83 | 18.6 | -+----------------+--------------+-----------------+------------------+----------------+ -| Hindi | hi | 87.74 | 90.01 | 21.9 | -+----------------+--------------+-----------------+------------------+----------------+ -| Hungarian | hu | 69.52 | 75.34 | 15.4 | -+----------------+--------------+-----------------+------------------+----------------+ -| Italian | it | 96.33 | 96.47 | 32.0 | -+----------------+--------------+-----------------+------------------+----------------+ -| Russian | ru_syntagrus | 93.57 | 96.23 | 48.7 | -+----------------+--------------+-----------------+------------------+----------------+ -| Spanish | es_ancora | 96.88 | 97.00 | 20.8 | -+----------------+--------------+-----------------+------------------+----------------+ -| Turkish | tr | 86.98 | 88.03 | 16.1 | -+----------------+--------------+-----------------+------------------+----------------+ +The BERT-based model is trained on `Universal +Dependencies corpora `__ +(version 2.3), while all the other models were trained +on Universal Dependencies 2.0 corpora. + ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| Language | Code | UDPipe accuracy | UDPipe Future accuracy [#f1]_ | Our top accuracy | Model size (MB)| ++================+==============+=================+===============================+==================+================+ +| Arabic | ar | 88.31 | | 90.85 | 23.7 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| Czech | cs | 91.86 | | 94.35 | 41.8 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| English | en | 92.53 | | 93.00 | 16.9 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| French | fr | 95.25 | | 95.45 | 19.0 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| German | de | 76.65 | | 83.83 | 18.6 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| Hindi | hi | 87.74 | | 90.01 | 21.9 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| Hungarian | hu | 69.52 | | 75.34 | 15.4 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| Italian | it | 96.33 | | 96.47 | 32.0 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| Russian | ru_syntagrus | 93.57 | | 96.23 | 48.7 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| Russian (UD2.3)| ru_syntagrus | 93.5 | 96.90 | 97.83 | 661 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| Spanish | es_ancora | 96.88 | | 97.00 | 20.8 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ +| Turkish | tr | 86.98 | | 88.03 | 16.1 | ++----------------+--------------+-----------------+-------------------------------+------------------+----------------+ + +.. rubric:: Footnotes + +.. [#f1] No models available, only the source code. The scores are taken from + `Straka. UDPipe 2.0 Prototype at CoNLL 2018 UD Shared Task. `__. + =========================== Usage examples. @@ -121,40 +142,50 @@ If you want the output in UD format, try setting ``"data_format": ud`` in the `` of :config:`configuration file ` you import. -Exclusively for Russian language you can obtain lemmatized UD output by using -:config:`augmented version ` -of Pymorphy model. - -.. code:: python - - from deeppavlov import build_model, configs - model = build_model(configs.morpho_tagger.UD2_0.morpho_ru_syntagrus_pymorphy_lemmatize, download=True) - sentences = ["Я шёл домой по незнакомой улице.", "Девушка пела в церковном хоре о всех уставших в чужом краю."] - for parse in model(sentences): - print(parse) - -:: - - 1 Я я PRON _ Case=Nom|Number=Sing|Person=1 _ _ _ _ - 2 шёл идти VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act _ _ _ _ - 3 домой домой ADV _ Degree=Pos _ _ _ _ - 4 по по ADP _ _ _ _ _ _ - 5 незнакомой незнакомый ADJ _ Case=Dat|Degree=Pos|Gender=Fem|Number=Sing _ _ _ _ - 6 улице улица NOUN _ Animacy=Inan|Case=Dat|Gender=Fem|Number=Sing _ _ _ _ - 7 . . PUNCT _ _ _ _ _ _ - - 1 Девушка девушка NOUN _ Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing _ _ _ _ - 2 пела петь VERB _ Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act _ _ _ _ - 3 в в ADP _ _ _ _ _ _ - 4 церковном церковный ADJ _ Case=Loc|Degree=Pos|Gender=Masc|Number=Sing _ _ _ _ - 5 хоре хор NOUN _ Animacy=Inan|Case=Loc|Gender=Masc|Number=Sing _ _ _ _ - 6 о о ADP _ _ _ _ _ _ - 7 всех весь PRON _ Animacy=Anim|Case=Loc|Number=Plur _ _ _ _ - 8 уставших устать VERB _ Aspect=Perf|Case=Loc|Number=Plur|Tense=Past|VerbForm=Part|Voice=Act _ _ _ _ - 9 в в ADP _ _ _ _ _ _ - 10 чужом чужой ADJ _ Case=Loc|Degree=Pos|Gender=Masc|Number=Sing _ _ _ _ - 11 краю край NOUN _ Animacy=Inan|Case=Loc|Gender=Masc|Number=Sing _ _ _ _ - 12 . . PUNCT _ _ _ _ _ _ +Advanced models (BERT and lemmatized models). +--------------------------------------------- + +#. For Russian you can use the BERT-based model. It has much higher performance (97.8% instead of 96.2), + however, you need a more powerful GPU (ideally, 16 GB) to train it. However, the speed + of inference and training on such GPU is comparable with character-based model. + +#. Exclusively for Russian language you can obtain lemmatized UD output by using either the + :config:`BERT model ` + :config:`augmented version ` + of Pymorphy model. Both models select the Pymorphy lemma whose tag correspond to the tag + predicted by the tagger. + + .. code:: python + + from deeppavlov import build_model, configs + model = build_model(configs.morpho_tagger.BERT.morpho_ru_syntagrus_bert, download=True) + # model = build_model(configs.morpho_tagger.UD2_0.morpho_ru_syntagrus_pymorphy_lemmatize, download=True) + sentences = ["Я шёл домой по незнакомой улице.", "Девушка пела в церковном хоре о всех уставших в чужом краю."] + for parse in model(sentences): + print(parse) + + :: + + 1 Я я PRON _ Case=Nom|Number=Sing|Person=1 _ _ _ _ + 2 шёл идти VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act _ _ _ _ + 3 домой домой ADV _ Degree=Pos _ _ _ _ + 4 по по ADP _ _ _ _ _ _ + 5 незнакомой незнакомый ADJ _ Case=Dat|Degree=Pos|Gender=Fem|Number=Sing _ _ _ _ + 6 улице улица NOUN _ Animacy=Inan|Case=Dat|Gender=Fem|Number=Sing _ _ _ _ + 7 . . PUNCT _ _ _ _ _ _ + + 1 Девушка девушка NOUN _ Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing _ _ _ _ + 2 пела петь VERB _ Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act _ _ _ _ + 3 в в ADP _ _ _ _ _ _ + 4 церковном церковный ADJ _ Case=Loc|Degree=Pos|Gender=Masc|Number=Sing _ _ _ _ + 5 хоре хор NOUN _ Animacy=Inan|Case=Loc|Gender=Masc|Number=Sing _ _ _ _ + 6 о о ADP _ _ _ _ _ _ + 7 всех весь PRON _ Animacy=Anim|Case=Loc|Number=Plur _ _ _ _ + 8 уставших устать VERB _ Aspect=Perf|Case=Loc|Number=Plur|Tense=Past|VerbForm=Part|Voice=Act _ _ _ _ + 9 в в ADP _ _ _ _ _ _ + 10 чужом чужой ADJ _ Case=Loc|Degree=Pos|Gender=Masc|Number=Sing _ _ _ _ + 11 краю край NOUN _ Animacy=Inan|Case=Loc|Gender=Masc|Number=Sing _ _ _ _ + 12 . . PUNCT _ _ _ _ _ _ Command line: ---------------- diff --git a/docs/features/models/syntaxparser.rst b/docs/features/models/syntaxparser.rst new file mode 100644 index 0000000000..60659cfbb2 --- /dev/null +++ b/docs/features/models/syntaxparser.rst @@ -0,0 +1,170 @@ +Syntactic parsing +============================ + +Syntactic parsing is the task of prediction of the syntactic tree given the tokenized (or raw) sentence. +The typical output of the parser looks looks like + +.. image:: /_static/tree.png + +To define a tree, for each word one should know its syntactic head and the dependency label for the edge between them. +For example, the tree above can be restored from the data + +:: + + 1 John 2 nsubj + 2 bought 0 root + 3 a 6 det + 4 very 5 advmod + 5 tasty 6 amod + 6 cake 2 obj + 7 . . 2 punct + +Here the third column contains the positions of syntactic heads and the last one -- the dependency labels. +The words are enumerated from 1 since 0 is the index of the artificial root of the tree, whose only +dependent is the actual syntactic head of the sentence (usually a verb). + +Syntactic trees can be used in many information extraction tasks. For example, to detect who is the winner +and who is the loser in the sentence *Manchester defeated Liverpool* one relies on the word order. However, +many languages, such as Russian, Spanish and German, have relatively free word order, which means we need +other cues. Note also that syntactic relations (`nsubj`, `obj` and so one) have clear semantic counterparts, +which makes syntactic parsing an appealing preprocessing step for the semantic-oriented tasks. + +Model usage +=========== + +Before using the model make sure that all required packages are installed using the command: + +.. code:: bash + + python -m deeppavlov install syntax_ru_syntagrus_bert + +Our model produces the output in `CONLL-U format `__ +and is trained on Universal Dependency corpora, available on http://universaldependencies.org/format.html . +The example usage for inference is + +.. code:: python + + from deeppavlov import build_model, configs + model = build_model(configs.syntax_parser.syntax_ru_syntagrus_bert, download=True) + sentences = ["Я шёл домой по незнакомой улице.", "Девушка пела в церковном хоре."] + for parse in model(sentences): + print(parse, end="\n\n") + + +:: + + 1 Я _ _ _ _ 2 nsubj _ _ + 2 шёл _ _ _ _ 0 root _ _ + 3 домой _ _ _ _ 2 advmod _ _ + 4 по _ _ _ _ 6 case _ _ + 5 незнакомой _ _ _ _ 6 amod _ _ + 6 улице _ _ _ _ 2 obl _ _ + 7 . _ _ _ _ 2 punct _ _ + + 1 Девушка _ _ _ _ 2 nsubj _ _ + 2 пела _ _ _ _ 0 root _ _ + 3 в _ _ _ _ 5 case _ _ + 4 церковном _ _ _ _ 5 amod _ _ + 5 хоре _ _ _ _ 2 obl _ _ + 6 . _ _ _ _ 2 punct _ _ + +As prescribed by UD standards, our model writes the head information to the 7th column and the dependency +information -- to the 8th. Our parser does not return morphological tags and even does not use them in +training. + +Model training is done via configuration files, see the +:config:`configuration file ` for reference. Note that as any BERT +model, it requires 16GB of GPU and the training speed is 1-5 sentences per second. However, you can +try less powerful GPU at your own risk (the author himself was able to run the model on 11GB). +The inference speed is several hundreds sentences per second, depending on their length, on GPU +and one magnitude lower on CPU. + +For other usage options see the :doc:`morphological tagger documentation `, +the training and prediction procedure is analogous, only the model name is changed. + +Joint model usage +================== + +Our model in principle supports joint prediction of morphological tags and syntactic information, +however, the quality of the joint model is slightly inferior to the separate ones. Therefore we +release a special component that can combine the outputs of tagger and parser: +:class:`~deeppavlov.models.syntax_parser.joint.JointTaggerParser`. Its sample output for the +Russian language with default settings +(see the :config:`configuration file ` for exact options) +looks like + +.. code:: python + + from deeppavlov import build_model, configs + model = build_model("ru_syntagrus_joint_parsing", download=True) + sentences = ["Я шёл домой по незнакомой улице.", "Девушка пела в церковном хоре."] + for parse in model(sentences): + print(parse, end="\n\n") + +:: + + 1 Я я PRON _ Case=Nom|Number=Sing|Person=1 2 nsubj _ _ + 2 шёл идти VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 0 root _ _ + 3 домой домой ADV _ Degree=Pos 2 advmod _ _ + 4 по по ADP _ _ 6 case _ _ + 5 незнакомой незнакомый ADJ _ Case=Dat|Degree=Pos|Gender=Fem|Number=Sing 6 amod _ _ + 6 улице улица NOUN _ Animacy=Inan|Case=Dat|Gender=Fem|Number=Sing 2 obl _ _ + 7 . . PUNCT _ _ 2 punct _ _ + + 1 Девушка девушка NOUN _ Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing 2 nsubj _ _ + 2 пела петь VERB _ Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 0 root _ _ + 3 в в ADP _ _ 5 case _ _ + 4 церковном церковный ADJ _ Case=Loc|Degree=Pos|Gender=Masc|Number=Sing 5 amod _ _ + 5 хоре хор NOUN _ Animacy=Inan|Case=Loc|Gender=Masc|Number=Sing 2 obl _ _ + 6 . . PUNCT _ _ 2 punct _ _ + +In the basic case the model outputs a human-readable string with parse data for each information. If you need +to use the output in Python, consult the +:class:`class documentation ` and source code. + +Model architecture +================== + +We use BERT as the lowest layer of our model (the embedder). To extract syntactic information we apply +the biaffine network of `[Dozat, Manning, 2017] `__. +For each sentence of length `K` this network produces two outputs: the first is an array of shape ``K*(K+1)``, +where `i`-th row is the probability distribution of the head of `i`-th word over the sentence elements. +The 0-th element of this distribution is the probability of the word to be a root of the sentence. +The second output of the network is of shape `K*D`, where `D` is the number of possible dependency labels. + +The easiest way to obtain a tree is simply to return the head with the highest probability +for each word in the sentence. However, the graph obtained in such a way may fail to be a valid tree: +it may either contain a cycle or have multiple nodes with head at position 0. +Therefore we apply the well-known Chu-Liu-Edmonds algorithm for minimal spanning tree +to return the optimal tree, using the open-source modification from +`dependency_decoding package `. + +Model quality +============= + +Syntactic parsers are evaluated using two metrics: UAS (unlabeled attachment score), which is +the percentage of correctly predicted head positions. The second metric is LAS (labeled attachment +score) which treats as positive only the words with correctly predicted dependency label +and dependency head. + +.. table:: + :widths: auto + + +-------------------------+-------------------------------------------------------------------------------------------+---------+----------+ + | Dataset | Model | UAS | LAS | + +=========================+===========================================================================================+=========+==========+ + | `UD2.3`_ (Russian) | `UD Pipe 2.3`_ (Straka et al., 2017) | 90.3 | 89.0 | + | +-------------------------------------------------------------------------------------------+---------+----------+ + | | `UD Pipe Future`_ (Straka, 2018) | 93.0 | 91.5 | + | +-------------------------------------------------------------------------------------------+---------+----------+ + | | `UDify (multilingual BERT)`_ (Kondratyuk, 2018) | 94.8 | 93.1 | + | +-------------------------------------------------------------------------------------------+---------+----------+ + | |:config:`our BERT model ` | 95.2 | 93.7 | + +-------------------------+-------------------------------------------------------------------------------------------+---------+----------+ + +.. _`UD2.3`: http://hdl.handle.net/11234/1-2895 +.. _`UD Pipe 2.3`: http://ufal.mff.cuni.cz/udpipe +.. _`UD Pipe Future`: https://github.com/CoNLL-UD-2018/UDPipe-Future +.. _`UDify (multilingual BERT)`: https://github.com/hyperparticle/udify + +So our model is by a valuable margin the state-of-the-art system for Russian syntactic parsing. \ No newline at end of file diff --git a/docs/features/overview.rst b/docs/features/overview.rst index 287a9a9dfb..ae21c2cf6a 100644 --- a/docs/features/overview.rst +++ b/docs/features/overview.rst @@ -394,11 +394,14 @@ model. This model outputs empty string in case if there is no answer in context. Morphological tagging model :doc:`[docs] ` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Based on character-based approach to morphological tagging `Heigold et al., 2017. An extensive empirical evaluation of -character-based morphological tagging for 14 languages `__. A state-of-the-art -model for Russian and several other languages. Model takes as input tokenized sentences and outputs the corresponding -sequence of morphological labels in `UD format `__. The table below -contains word and sentence accuracy on UD2.0 datasets. For more scores see :doc:`full table `. +We have a BERT-based model for Russian and character-based models for 11 languages. +The character model is based on `Heigold et al., 2017. An extensive empirical evaluation of +character-based morphological tagging for 14 languages `__. +It is a state-of-the-art model for Russian and near state of the art for several other languages. +Model takes as input tokenized sentences and outputs the corresponding +sequence of morphological labels in `UD format `__. +The table below contains word and sentence accuracy on UD2.0 datasets. +For more scores see :doc:`full table `. .. table:: :widths: auto @@ -406,6 +409,12 @@ contains word and sentence accuracy on UD2.0 datasets. For more scores see :doc: +----------------------+--------------------------------------------------------------------------------------------------------------+---------------+----------------+--------------------+ | Dataset | Model | Word accuracy | Sent. accuracy | Download size (MB) | +======================+==============================================================================================================+===============+================+====================+ + |`UD2.3`_ (Russian) |`UD Pipe 2.3`_ (Straka et al., 2017) | 93.5 | | | + | +--------------------------------------------------------------------------------------------------------------+---------------+----------------+--------------------+ + | | `UD Pipe Future`_ (Straka et al., 2018) | 96.90 | | | + | +--------------------------------------------------------------------------------------------------------------+---------------+----------------+--------------------+ + | |:config:`BERT-based model ` | 97.83 | 72.02 | 661 | + +----------------------+--------------------------------------------------------------------------------------------------------------+---------------+----------------+--------------------+ | |`Pymorphy`_ + `russian_tagsets`_ (first tag) | 60.93 | 0.00 | | + +--------------------------------------------------------------------------------------------------------------+---------------+----------------+--------------------+ |`UD2.0`_ (Russian) |`UD Pipe 1.2`_ (Straka et al., 2017) | 93.57 | 43.04 | | @@ -430,7 +439,37 @@ contains word and sentence accuracy on UD2.0 datasets. For more scores see :doc: .. _`Pymorphy`: https://pymorphy2.readthedocs.io/en/latest/ .. _`russian_tagsets`: https://github.com/kmike/russian-tagsets .. _`UD2.0`: https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1983 +.. _`UD2.3`: http://hdl.handle.net/11234/1-2895 .. _`UD Pipe 1.2`: http://ufal.mff.cuni.cz/udpipe +.. _`UD Pipe 2.3`: http://ufal.mff.cuni.cz/udpipe +.. _`UD Pipe Future`: https://github.com/CoNLL-UD-2018/UDPipe-Future + +Syntactic parsing model :doc:`[docs] ` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We have a biaffine model for syntactic parsing based on RuBERT. +It achieves the highest known labeled attachments score of 93.7% +on ``ru_syntagrus`` Russian corpus (version UD 2.3). + +.. table:: + :widths: auto + + +-------------------------+-------------------------------------------------------------------------------------------+---------+----------+ + | Dataset | Model | UAS | LAS | + +=========================+===========================================================================================+=========+==========+ + | `UD2.3`_ (Russian) | `UD Pipe 2.3`_ (Straka et al., 2017) | 90.3 | 89.0 | + | +-------------------------------------------------------------------------------------------+---------+----------+ + | | `UD Pipe Future`_ (Straka, 2018) | 93.0 | 91.5 | + | +-------------------------------------------------------------------------------------------+---------+----------+ + | | `UDify (multilingual BERT)`_ (Kondratyuk, 2018) | 94.8 | 93.1 | + | +-------------------------------------------------------------------------------------------+---------+----------+ + | |:config:`our BERT model ` | 95.2 | 93.7 | + +-------------------------+-------------------------------------------------------------------------------------------+---------+----------+ + +.. _`UD2.3`: http://hdl.handle.net/11234/1-2895 +.. _`UD Pipe 2.3`: http://ufal.mff.cuni.cz/udpipe +.. _`UD Pipe Future`: https://github.com/CoNLL-UD-2018/UDPipe-Future +.. _`UDify (multilingual BERT)`: https://github.com/hyperparticle/udify Frequently Asked Questions (FAQ) model :doc:`[docs] ` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/index.rst b/docs/index.rst index fd1ae647b5..690454ec2e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,6 +34,7 @@ Welcome to DeepPavlov's documentation! Neural Ranking Slot filling Spelling Correction + Syntactic Parser TF-IDF Ranking Popularity Ranking Knowledge Base Question answering diff --git a/docs/intro/installation.rst b/docs/intro/installation.rst index 8506bee5b4..ea4634d4db 100644 --- a/docs/intro/installation.rst +++ b/docs/intro/installation.rst @@ -23,7 +23,7 @@ We support ``Linux`` and ``Windows`` platforms, ``Python 3.6`` and ``Python 3.7` * Linux - .. code:: + .. code:: bash source ./env/bin/activate diff --git a/tests/test_quick_start.py b/tests/test_quick_start.py index 4b652da614..9e914fda9e 100644 --- a/tests/test_quick_start.py +++ b/tests/test_quick_start.py @@ -237,12 +237,14 @@ }, "morpho_tagger": { ("morpho_tagger/UD2.0/morpho_en.json", "morpho_en", ('IP', 'TI')): [ONE_ARGUMENT_INFER_CHECK], - ("morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy.json", "morpho_tagger_pymorphy", ('IP', 'TI')): - [ONE_ARGUMENT_INFER_CHECK], ("morpho_tagger/UD2.0/morpho_ru_syntagrus_pymorphy_lemmatize.json", "morpho_tagger_pymorphy", ('IP', 'TI')): [ONE_ARGUMENT_INFER_CHECK], - ("morpho_tagger/UD2.0/morpho_ru_syntagrus.json", "morpho_tagger_pymorphy", ('IP', 'TI')): + ("morpho_tagger/BERT/morpho_ru_syntagrus_bert.json", "morpho_tagger_bert", ('IP', 'TI')): [ONE_ARGUMENT_INFER_CHECK] + }, + "syntax_tagger": { + ("syntax/syntax_ru_syntagrus_bert.json", "syntax_ru_bert", ('IP', 'TI')): [ONE_ARGUMENT_INFER_CHECK], + ("syntax/ru_syntagrus_joint_parsing.json", "syntax_ru_bert", ('IP',)): [ONE_ARGUMENT_INFER_CHECK] } } @@ -285,8 +287,10 @@ def download_config(config_path): # Download referenced config files config_references = get_all_elems_from_json(parse_config(config), 'config_path') for config_ref in config_references: - m_name = config_ref.split('/')[-2] - config_ref = '/'.join(config_ref.split('/')[-2:]) + splitted = config_ref.split("/") + first_subdir_index = splitted.index("configs") + 1 + m_name = config_ref.split('/')[first_subdir_index] + config_ref = '/'.join(config_ref.split('/')[first_subdir_index:]) test_configs_path.joinpath(m_name).mkdir(exist_ok=True) if not test_configs_path.joinpath(config_ref).exists(): From ba2f438263bf0f6eeef1f244a1c852af79a82734 Mon Sep 17 00:00:00 2001 From: Aleksei Lymar Date: Thu, 28 Nov 2019 13:06:05 +0300 Subject: [PATCH 27/28] fix: correct train epochs count for morpho_ru_syntagrus_bert --- .../configs/morpho_tagger/BERT/morpho_ru_syntagrus_bert.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/configs/morpho_tagger/BERT/morpho_ru_syntagrus_bert.json b/deeppavlov/configs/morpho_tagger/BERT/morpho_ru_syntagrus_bert.json index a6a6ac792d..58c9db617c 100644 --- a/deeppavlov/configs/morpho_tagger/BERT/morpho_ru_syntagrus_bert.json +++ b/deeppavlov/configs/morpho_tagger/BERT/morpho_ru_syntagrus_bert.json @@ -116,7 +116,7 @@ ] }, "train": { - "epochs": 1, + "epochs": 10, "batch_size": 32, "metrics": [ { From aeafdcf602e64276b3f32e1e2457c144cb7ace51 Mon Sep 17 00:00:00 2001 From: Aleksei Lymar Date: Thu, 28 Nov 2019 16:32:05 +0300 Subject: [PATCH 28/28] docs: minor changes --- deeppavlov/core/trainers/nn_trainer.py | 8 ++++---- deeppavlov/dataset_readers/file_paths_reader.py | 2 +- docs/index.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deeppavlov/core/trainers/nn_trainer.py b/deeppavlov/core/trainers/nn_trainer.py index 7dcdeaf881..23fb921c24 100644 --- a/deeppavlov/core/trainers/nn_trainer.py +++ b/deeppavlov/core/trainers/nn_trainer.py @@ -52,7 +52,7 @@ class NNTrainer(FitTrainer): train_metrics: metrics calculated for train logs (if omitted, ``metrics`` argument is used) metric_optimization: one of ``'maximize'`` or ``'minimize'`` — strategy for metric optimization used in early stopping (default is ``'maximize'``) - evaluation_targets: data types on which to evaluate trained pipeline (default is ``('valid', 'test')``) + evaluation_targets: data types on which to evaluate a trained pipeline (default is ``('valid', 'test')``) show_examples: a flag used to print inputs, expected outputs and predicted outputs for the last batch in evaluation logs (default is ``False``) tensorboard_log_dir: path to a directory where tensorboard logs can be stored, ignored if None @@ -75,7 +75,7 @@ class NNTrainer(FitTrainer): **kwargs: additional parameters whose names will be logged but otherwise ignored - Trainer saves the model if it sees a progress in scores. The full rules look like following: + Trainer saves the model if it sees progress in scores. The full rules look like following: - For the validation savepoint: * 0-th validation (optional). Don't save model, establish a baseline. @@ -143,7 +143,7 @@ def _improved(op): self.patience = 0 self.last_result = {} self.losses = [] - self.start_time = None # type:Optional[float] + self.start_time: Optional[float] = None if self.tensorboard_log_dir is not None: self.tb_train_writer = self._tf.summary.FileWriter(str(self.tensorboard_log_dir / 'train_log')) @@ -208,7 +208,7 @@ def _validate(self, iterator: DataLearningIterator, log.info('Saving model') self.save() else: - log.info('Did not improved on the {} of {}'.format(m_name, self.score_best)) + log.info('Did not improve on the {} of {}'.format(m_name, self.score_best)) report['impatience'] = self.patience if self.validation_patience > 0: diff --git a/deeppavlov/dataset_readers/file_paths_reader.py b/deeppavlov/dataset_readers/file_paths_reader.py index adddc08470..4e6cbae299 100644 --- a/deeppavlov/dataset_readers/file_paths_reader.py +++ b/deeppavlov/dataset_readers/file_paths_reader.py @@ -57,7 +57,7 @@ def _get_files(self, data_path, tgt): paths = Path(data_path).resolve().glob(tgt) files = [file for file in paths if Path(file).is_file()] paths_info = Path(data_path, tgt).absolute().as_posix() - if not (files): + if not files: raise Exception(f"Not find files. Data path '{paths_info}' does not exist or does not hold files!") else: log.info(f"Found {len(files)} files located '{paths_info}'.") diff --git a/docs/index.rst b/docs/index.rst index 690454ec2e..5c9b2d89bf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -66,7 +66,7 @@ Welcome to DeepPavlov's documentation! Amazon Alexa integration Microsoft Bot Framework integration Amazon AWS deployment - Deeppavlov settings + DeepPavlov settings .. toctree::