From 0e7d44e4f2ef0893df27c398db03334fc6927f97 Mon Sep 17 00:00:00 2001 From: Ashish Vaswani Date: Wed, 21 Jun 2017 17:36:21 -0700 Subject: [PATCH 1/5] fixed a small bug in transformer_parsing_base PiperOrigi-RevId: 159769630 --- .../data_generators/generator_utils.py | 18 +----------------- tensor2tensor/models/README.md | 2 +- tensor2tensor/models/transformer.py | 1 - 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/tensor2tensor/data_generators/generator_utils.py b/tensor2tensor/data_generators/generator_utils.py index 673eefd25..487546e16 100644 --- a/tensor2tensor/data_generators/generator_utils.py +++ b/tensor2tensor/data_generators/generator_utils.py @@ -126,18 +126,6 @@ def generate_files(generator, return output_files -def download_report_hook(count, block_size, total_size): - """Report hook for download progress - - Args: - count: current block number - block_size: block size - total_size: total size - """ - percent = int(count*block_size*100/total_size) - print("\r%d%%" % percent + ' completed', end='\r') - - def maybe_download(directory, filename, url): """Download filename from url unless it's already in directory. @@ -155,11 +143,7 @@ def maybe_download(directory, filename, url): filepath = os.path.join(directory, filename) if not tf.gfile.Exists(filepath): tf.logging.info("Downloading %s to %s" % (url, filepath)) - filepath, _ = urllib.urlretrieve(url, filepath, - reporthook=download_report_hook) - - # Print newline to clear the carriage return from the download progress - print() + filepath, _ = urllib.urlretrieve(url, filepath) statinfo = os.stat(filepath) tf.logging.info("Succesfully downloaded %s, %s bytes." % (filename, statinfo.st_size)) diff --git a/tensor2tensor/models/README.md b/tensor2tensor/models/README.md index 69050b9b2..29b88484f 100644 --- a/tensor2tensor/models/README.md +++ b/tensor2tensor/models/README.md @@ -12,5 +12,5 @@ To add a model to the built-in set, create a new file (see, e.g., `neural_gpu.py`) and write your model class inheriting from `T2TModel` there and decorate it with `registry.register_model`. Import it in `models.py`. -It is now available to use with the trainer binary (`t2t-trainer`) using the +It is now avaialable to use with the trainer binary (`t2t-trainer`) using the `--model=model_name` flag. diff --git a/tensor2tensor/models/transformer.py b/tensor2tensor/models/transformer.py index 264e0570d..4fb53fe1c 100644 --- a/tensor2tensor/models/transformer.py +++ b/tensor2tensor/models/transformer.py @@ -342,7 +342,6 @@ def transformer_parsing_base(): hparams.learning_rate_warmup_steps = 16000 hparams.hidden_size = 1024 hparams.learning_rate = 0.05 - hparams.residual_dropout = 0.1 hparams.shared_embedding_and_softmax_weights = int(False) return hparams From e58639869e86ce553ed43228ac10f27456551dfc Mon Sep 17 00:00:00 2001 From: Noam Shazeer Date: Wed, 21 Jun 2017 21:17:58 -0700 Subject: [PATCH 2/5] Cap number of parallel data readers at the number of data shards. Otherwise, insufficient shuffling can lead to multiple instances of the same example in a batch, and worse quality. This is a problem in particular for single-sharded datasets. --- tensor2tensor/utils/data_reader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensor2tensor/utils/data_reader.py b/tensor2tensor/utils/data_reader.py index 0080ecaa6..0ba62ec9f 100644 --- a/tensor2tensor/utils/data_reader.py +++ b/tensor2tensor/utils/data_reader.py @@ -100,6 +100,8 @@ def examples_queue(data_sources, with tf.name_scope("examples_queue"): # Read serialized examples using slim parallel_reader. num_epochs = None if training else 1 + data_files = tf.contrib.slim.parallel_reader.get_data_files(data_sources) + num_readers = min(4 if training else 1, len(data_files)) _, example_serialized = tf.contrib.slim.parallel_reader.parallel_read( data_sources, tf.TFRecordReader, @@ -107,7 +109,7 @@ def examples_queue(data_sources, shuffle=training, capacity=2 * capacity, min_after_dequeue=capacity, - num_readers=4 if training else 1) + num_readers=num_readers) if data_items_to_decoders is None: data_items_to_decoders = { From e3e27725d4560f25e1f81077cb0200b91273c706 Mon Sep 17 00:00:00 2001 From: Ryan Sepassi Date: Thu, 22 Jun 2017 17:33:22 -0700 Subject: [PATCH 3/5] add readme toc, improve data downloads PiperOrigin-RevId: 159900539 --- README.md | 14 ++++++++++++++ tensor2tensor/data_generators/generator_utils.py | 4 +++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 69ad66ddc..f49fb20df 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,20 @@ issues](https://github.com/tensorflow/tensor2tensor/issues). And chat with us and other users on [Gitter](https://gitter.im/tensor2tensor/Lobby). +### Contents + +* [Walkthrough](#walkthrough) +* [Installation](#installation) +* [Features](#features) +* [T2T Overview](#t2t-overview) + * [Datasets](#datasets) + * [Problems and Modalities](#problems-and-modalities) + * [Models](#models) + * [Hyperparameter Sets](#hyperparameter-sets) + * [Trainer](#trainer) +* [Adding your own components](#adding-your-own-components) +* [Adding a dataset](#adding-a-dataset) + --- ## Walkthrough diff --git a/tensor2tensor/data_generators/generator_utils.py b/tensor2tensor/data_generators/generator_utils.py index 487546e16..7b6534ad2 100644 --- a/tensor2tensor/data_generators/generator_utils.py +++ b/tensor2tensor/data_generators/generator_utils.py @@ -143,7 +143,9 @@ def maybe_download(directory, filename, url): filepath = os.path.join(directory, filename) if not tf.gfile.Exists(filepath): tf.logging.info("Downloading %s to %s" % (url, filepath)) - filepath, _ = urllib.urlretrieve(url, filepath) + inprogress_filepath = filepath + ".incomplete" + inprogress_filepath, _ = urllib.urlretrieve(url, inprogress_filepath) + tf.gfile.Rename(inprogress_filepath, filepath) statinfo = os.stat(filepath) tf.logging.info("Succesfully downloaded %s, %s bytes." % (filename, statinfo.st_size)) From 854480da6d36afe53ed320ceae393eb47a8574e3 Mon Sep 17 00:00:00 2001 From: Lukasz Kaiser Date: Thu, 22 Jun 2017 18:43:42 -0700 Subject: [PATCH 4/5] internal merge PiperOrigin-RevId: 159906200 --- .../data_generators/generator_utils.py | 17 ++++++++++++++++- tensor2tensor/models/README.md | 2 +- tensor2tensor/models/common_layers.py | 7 +++++-- tensor2tensor/models/transformer.py | 9 +++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/tensor2tensor/data_generators/generator_utils.py b/tensor2tensor/data_generators/generator_utils.py index 7b6534ad2..35e61d7cc 100644 --- a/tensor2tensor/data_generators/generator_utils.py +++ b/tensor2tensor/data_generators/generator_utils.py @@ -126,6 +126,18 @@ def generate_files(generator, return output_files +def download_report_hook(count, block_size, total_size): + """Report hook for download progress. + + Args: + count: current block number + block_size: block size + total_size: total size + """ + percent = int(count * block_size * 100 / total_size) + print("\r%d%%" % percent + " completed", end="\r") + + def maybe_download(directory, filename, url): """Download filename from url unless it's already in directory. @@ -144,7 +156,10 @@ def maybe_download(directory, filename, url): if not tf.gfile.Exists(filepath): tf.logging.info("Downloading %s to %s" % (url, filepath)) inprogress_filepath = filepath + ".incomplete" - inprogress_filepath, _ = urllib.urlretrieve(url, inprogress_filepath) + inprogress_filepath, _ = urllib.urlretrieve(url, inprogress_filepath, + reporthook=download_report_hook) + # Print newline to clear the carriage return from the download progress + print() tf.gfile.Rename(inprogress_filepath, filepath) statinfo = os.stat(filepath) tf.logging.info("Succesfully downloaded %s, %s bytes." % (filename, diff --git a/tensor2tensor/models/README.md b/tensor2tensor/models/README.md index 29b88484f..69050b9b2 100644 --- a/tensor2tensor/models/README.md +++ b/tensor2tensor/models/README.md @@ -12,5 +12,5 @@ To add a model to the built-in set, create a new file (see, e.g., `neural_gpu.py`) and write your model class inheriting from `T2TModel` there and decorate it with `registry.register_model`. Import it in `models.py`. -It is now avaialable to use with the trainer binary (`t2t-trainer`) using the +It is now available to use with the trainer binary (`t2t-trainer`) using the `--model=model_name` flag. diff --git a/tensor2tensor/models/common_layers.py b/tensor2tensor/models/common_layers.py index ef6559f9e..30215e889 100644 --- a/tensor2tensor/models/common_layers.py +++ b/tensor2tensor/models/common_layers.py @@ -1079,6 +1079,7 @@ def conv_hidden_relu(inputs, hidden_size, output_size, kernel_size=(1, 1), + second_kernel_size=(1, 1), summaries=True, dropout=0.0, **kwargs): @@ -1090,7 +1091,8 @@ def conv_hidden_relu(inputs, inputs = tf.expand_dims(inputs, 2) else: is_3d = False - h = conv( + conv_f1 = conv if kernel_size == (1, 1) else separable_conv + h = conv_f1( inputs, hidden_size, kernel_size, @@ -1103,7 +1105,8 @@ def conv_hidden_relu(inputs, tf.summary.histogram("hidden_density_logit", relu_density_logit( h, list(range(inputs.shape.ndims - 1)))) - ret = conv(h, output_size, (1, 1), name="conv2", **kwargs) + conv_f2 = conv if second_kernel_size == (1, 1) else separable_conv + ret = conv_f2(h, output_size, second_kernel_size, name="conv2", **kwargs) if is_3d: ret = tf.squeeze(ret, 2) return ret diff --git a/tensor2tensor/models/transformer.py b/tensor2tensor/models/transformer.py index 4fb53fe1c..94fb0776c 100644 --- a/tensor2tensor/models/transformer.py +++ b/tensor2tensor/models/transformer.py @@ -255,6 +255,15 @@ def transformer_ffn_layer(x, hparams): hparams.filter_size, hparams.num_heads, hparams.attention_dropout) + elif hparams.ffn_layer == "conv_hidden_relu_with_sepconv": + return common_layers.conv_hidden_relu( + x, + hparams.filter_size, + hparams.hidden_size, + kernel_size=(3, 1), + second_kernel_size=(31, 1), + padding="LEFT", + dropout=hparams.relu_dropout) else: assert hparams.ffn_layer == "none" return x From 2f4d5b7f4c7894d7def087d28cb8ba707dd24d6d Mon Sep 17 00:00:00 2001 From: Ryan Sepassi Date: Thu, 22 Jun 2017 19:36:01 -0700 Subject: [PATCH 5/5] Bump version to 1.0.6 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a2d541a30..0bf0c8739 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='tensor2tensor', - version='1.0.5', + version='1.0.6', description='Tensor2Tensor', author='Google Inc.', author_email='no-reply@google.com',