Merge pull request #48 from naitoh/sfloat

Changed to use SFloat instead of DFloat for performance improvement.

examples/iris.rb

@@ -61,9 +61,9 @@ def fwd(x)
 # [7.0, 3.2, 4.7, 1.4, "Iris-versicolor"] => 50 data
 # [6.3, 3.3, 6.0, 2.5, "Iris-virginica"]  => 50 data
 
-x = Numo::DFloat.cast(x)
-y = Numo::DFloat.cast(y)
-y_onehot = Numo::DFloat.cast(y_onehot)
+x = Numo::SFloat.cast(x)
+y = Numo::SFloat.cast(y)
+y_onehot = Numo::SFloat.cast(y_onehot)
 
 x_train = x[(1..-1).step(2), true]        #=> 75 data (Iris-setosa : 25, Iris-versicolor : 25, Iris-virginica : 25)
 y_train = y_onehot[(1..-1).step(2), true] #=> 75 data (Iris-setosa : 25, Iris-versicolor : 25, Iris-virginica : 25)

lib/chainer/dataset/convert.rb

@@ -29,30 +29,51 @@ def self.concat_examples(batch, device: nil, padding: nil)
 
       def self.concat_arrays(arrays, padding)
         unless arrays[0].kind_of?(Numo::NArray)
+          # [1, 2, 3, 4] => Numo::Int32[1, 2, 3, 4]
           arrays = Numo::NArray.cast(arrays)
+          if padding
+            return concat_arrays_with_padding(arrays, padding)
+          end
+          return arrays
         end
 
         if padding
           return concat_arrays_with_padding(arrays, padding)
         end
 
-        Numo::NArray.[](*arrays.to_a.map { |arr| arr.kind_of?(Numeric) ? arr : Numo::NArray.[](*arr) })
+        # [Numo::SFloat[1, 2], Numo::SFloat[3, 4]]
+        #  => Numo::SFloat#shape=[2,2]
+        # [[1, 2], [3, 4]]
+        a = arrays.map{|arr| arr[:-, false]}
+        a[0].concatenate(*a[1..-1])
       end
 
       def self.concat_arrays_with_padding(arrays, padding)
-        shape = Numo::Int32.[](arrays[0].shape)
-        arrays[1...arrays.len].each do |array|
-          if Numo::Bit.[](shape != array.shape).any?
-            # TODO: numpy maximum
+        if arrays[0].is_a? Numo::NArray
+          shape = Numo::Int32.cast(arrays[0].shape)
+          arrays[1..-1].each do |array|
+            if Numo::Bit.[](shape != array.shape).any?
+              shape = Numo::Int32.maximum(shape, array.shape)
+            end
           end
+        else # Integer
+          shape = []
+        end
+
+        shape = shape.insert(0, arrays.size).to_a
+        if arrays[0].is_a? Numo::NArray
+          result = arrays[0].class.new(shape).fill(padding)
+        else # Integer
+          result = Numo::Int32.new(shape).fill(padding)
         end
 
-        shape = [shape.insert(0, arrays.size)]
-        result = arrays[0].dtype.[](*shape).full(padding)
         arrays.size.times do |i|
           src = arrays[i]
-          slices = src.shape.map { |s| [s] }
-          result[[i] + slices] = src
+          if src.is_a? Numo::NArray
+            result[i, 0...src.shape[0], 0...src.shape[1]] = src
+          else # Integer
+            result[i] = src
+          end
         end
 
         result

lib/chainer/datasets/mnist.rb

@@ -3,7 +3,7 @@
 module Chainer
   module Datasets
     module Mnist
-      def self.get_mnist(withlabel: true, ndim: 1, scale: 1.0, dtype: Numo::DFloat, label_dtype: Numo::Int32)
+      def self.get_mnist(withlabel: true, ndim: 1, scale: 1.0, dtype: Numo::SFloat, label_dtype: Numo::Int32)
         train_raw = retrieve_mnist_training
         train = preprocess_mnist(train_raw, withlabel, ndim, scale, dtype, label_dtype)
 

lib/chainer/functions/loss/softmax_cross_entropy.rb

@@ -84,12 +84,12 @@ def backward_cpu(inputs, grad_outputs)
 
           if y.ndim == 2
             gx = y
-            Numo::DFloat.new(t.shape[0]).seq(0).to_a.zip(Numo::DFloat.maximum(t, 0).to_a).each{|v| gx[*v] -= 1}
+            t.class.new(t.shape[0]).seq(0).to_a.zip(t.class.maximum(t, 0).to_a).each{|v| gx[*v] -= 1}
 
             if @class_weight
               shape = x.ndim.times.map { |d| d == 1 ? true : 1 }
               c = Chainer::Functions::Loss.broadcast_to(@class_weight.reshape(*shape), x.shape)
-              c = c.class.cast(Numo::DFloat.new(t.shape[0]).seq.to_a.zip(Numo::DFloat.maximum(t, 0).to_a).map{|v| c[*v]})
+              c = c.class.cast(t.class.new(t.shape[0]).seq.to_a.zip(t.class.maximum(t, 0).to_a).map{|v| c[*v]})
               gx *= Chainer::Functions::Loss.broadcast_to(c.expand_dims(1), gx.shape)
             end
 
@@ -106,12 +106,12 @@ def backward_cpu(inputs, grad_outputs)
             gx = y.reshape(y.shape[0], y.shape[1], true)
             fst_index = Numo::Int32.new(t.size).seq(0) / n_unit
             trd_index = Numo::Int32.new(t.size).seq(0) % n_unit
-            fst_index.to_a.zip(Numo::DFloat.maximum(t.flatten.dup, 0).to_a, trd_index.to_a).each{|v| gx[*v] -= 1}
+            fst_index.to_a.zip(t.class.maximum(t.flatten.dup, 0).to_a, trd_index.to_a).each{|v| gx[*v] -= 1}
             if @class_weight
               shape = x.ndim.times.map{|d| d == 1 ? true : 1}
               c = Chainer::Functions::Loss.broadcast_to(@class_weight.reshape(*shape), x.shape)
               c = c.reshape(*gx.shape)
-              c = c.class.cast(fst_index.to_a.zip(Numo::DFloat.maximum(t.flatten.dup, 0).to_a, trd_index.to_a).map{|v| c[*v]})
+              c = c.class.cast(fst_index.to_a.zip(t.class.maximum(t.flatten.dup, 0).to_a, trd_index.to_a).map{|v| c[*v]})
               c = c.reshape(y.shape[0], 1, true)
               gx *= Chainer::Functions::Loss.broadcast_to(c, gx.shape)
             end

lib/chainer/functions/noise/dropout.rb

@@ -26,9 +26,9 @@ def forward(x)
           retain_inputs([])
           unless self.instance_variable_defined?(:@mask)
             scale = x[0].class[*[1.0 / (1 - @dropout_ratio)]][0]
-            flag = Numo::DFloat.new(*x[0].shape).rand >= @dropout_ratio
+            flag = x[0].class.new(*x[0].shape).rand >= @dropout_ratio
 
-            @mask = Numo::DFloat.zeros(*x[0].shape)
+            @mask = x[0].class.zeros(*x[0].shape)
             @mask[flag] = 1
             @mask *= scale
           end

lib/chainer/initializers/init.rb

@@ -1,7 +1,7 @@
 module Chainer
   module Initializers
     def self.generate_array(initializer, shape)
-      klass = Numo::DFloat
+      klass = Numo::SFloat
       if initializer.respond_to?(:dtype) && initializer.dtype
         klass = initializer.dtype
       end

lib/chainer/initializers/normal.rb

@@ -8,7 +8,7 @@ def initialize(scale: 0.05, dtype: nil)
 
       def call(array)
         args = { loc: 0.0, scale: @scale, size: array.shape}
-        Numo::DFloat.new(array.shape).rand_norm(0.0, @scale)
+        array.class.new(array.shape).rand_norm(0.0, @scale)
       end
     end
 

lib/chainer/optimizer.rb

@@ -81,7 +81,7 @@ def serialize(serializer)
           # try to initialize the state to retrieve state entries
           @state = {}
           self_copy = self.dup
-          arr = Numo::DFloat.new(1)
+          arr = Numo::SFloat.new(1)
           self_copy.init_state(Chainer::Variable.new(arr, grad: arr))
           @state.keys.each do |key|
             @state[key] = serializer.(key.to_s, nil)

lib/chainer/parameter.rb

@@ -15,7 +15,7 @@ def initialize(initializer: nil, shape: nil, name: nil)
         else
           super(name: name)
           @initializer = initializer
-          dtype = initializer.respond_to?(:dtype) ? initializer.dtype : 'DFloat'
+          dtype = initializer.respond_to?(:dtype) ? initializer.dtype : 'SFloat'
           @grad_initializer = Chainer::Initializers.nan()
         end
       else

test/dataset/convert_test.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require 'chainer/dataset/convert'
+require 'chainer/testing/array'
+
+class TestConcatExamples < Test::Unit::TestCase
+  def get_arrays_to_concat(xumo)
+    return 5.times.map{|_| xumo::DFloat.new(2, 3).rand()}
+  end
+
+  def check_device(array, device)
+    if device
+      # T.B.I (GPU Check)
+    end
+  end
+
+  def check_concat_arrays(arrays, device: nil)
+    array = Chainer::Dataset::Convert.method(:concat_examples).call(arrays, device: device)
+    assert_equal([arrays.size] + arrays[0].shape, array.shape)
+    check_device(array, device)
+    array.to_a.zip(arrays.to_a).each do |x, y|
+      assert_true array.class.cast(x) == array.class.cast(y)
+    end
+  end
+
+  def test_concat_arrays_cpu()
+    arrays = get_arrays_to_concat(Numo)
+    check_concat_arrays(arrays)
+  end
+
+  def get_tuple_arrays_to_concat(xumo)
+    return 5.times.map{|_| [xumo::DFloat.new(2, 3).rand(), xumo::DFloat.new(3, 4).rand()]}
+  end
+
+  def check_concat_tuples(tuples, device: nil)
+    arrays = Chainer::Dataset::Convert.method(:concat_examples).call(tuples, device: device)
+    assert_equal(tuples[0].size, arrays.size)
+    arrays.size.times do |i|
+      shape = [tuples.size] + tuples[0][i].shape
+      assert_equal(shape, arrays[i].shape)
+      check_device(arrays[i], device)
+      arrays[i].to_a.zip(tuples.to_a).each do |x, y|
+        assert_true arrays[i].class.cast(x) == arrays[i].class.cast(y[i])
+      end
+    end
+  end
+
+  def test_concat_tuples_cpu()
+    tuples = get_tuple_arrays_to_concat(Numo)
+    check_concat_tuples(tuples)
+  end
+end
+
+class TestConcatExamplesWithPadding < Test::Unit::TestCase
+  def check_concat_arrays_padding(xumo)
+    arrays = [xumo::DFloat.new(3, 4).rand(), xumo::DFloat.new(2, 5).rand(), xumo::DFloat.new(4, 3).rand()]
+    array = Chainer::Dataset::Convert.method(:concat_examples).call(arrays, padding: 0)
+
+    assert_equal([3, 4, 5], array.shape)
+    assert_equal(arrays[0].class, array.class)
+    arrays = arrays.map{|a| array.class.cast(a)}
+    assert_true array[0, 0...3, 0...4].nearly_eq(arrays[0]).all?
+    assert_true array[0, 3..-1, 0..-1].nearly_eq(0).all?
+    assert_true array[0, 0..-1, 4..-1].nearly_eq(0).all?
+    assert_true array[1, 0...2, 0...5].nearly_eq(arrays[1]).all?
+    assert_true array[1, 2..-1, 0..-1].nearly_eq(0).all?
+    assert_true array[2, 0...4, 0...3].nearly_eq(arrays[2]).all?
+    assert_true array[2, 0..-1, 3..-1].nearly_eq(0).all?
+  end
+
+  def test_concat_arrays_padding_cpu()
+    check_concat_arrays_padding(Numo)
+  end
+
+  def check_concat_tuples_padding(xumo)
+    tuples = [[xumo::DFloat.new(3, 4).rand(), xumo::DFloat.new(2, 5).rand()],
+              [xumo::DFloat.new(4, 4).rand(), xumo::DFloat.new(3, 4).rand()],
+              [xumo::DFloat.new(2, 5).rand(), xumo::DFloat.new(2, 6).rand()]]
+    arrays = Chainer::Dataset::Convert.method(:concat_examples).call(tuples, padding: 0)
+
+    assert_equal(2, arrays.size)
+    assert_equal([3, 4, 5], arrays[0].shape)
+    assert_equal([3, 3, 6], arrays[1].shape)
+    assert_equal(tuples[0][0].class, arrays[0].class)
+    assert_equal(tuples[0][1].class, arrays[1].class)
+    tuples.size.times do |i|
+      tuples[i] = [tuples[i][0], tuples[i][1]]
+    end
+
+    arrays = arrays.to_a
+    assert_true arrays[0][0, 0...3, 0...4].nearly_eq(tuples[0][0]).all?
+    assert_true arrays[0][0, 3..-1, 0..-1].nearly_eq(0).all?
+    assert_true arrays[0][0, 0..-1, 4..-1].nearly_eq(0).all?
+    assert_true arrays[0][1, 0...4, 0...4].nearly_eq(tuples[1][0]).all?
+    assert_true arrays[0][1, 0..-1, 4..-1].nearly_eq(0).all?
+    assert_true arrays[0][2, 0...2, 0...5].nearly_eq(tuples[2][0]).all?
+    assert_true arrays[0][2, 2..-1, 0..-1].nearly_eq(0).all?
+    assert_true arrays[1][0, 0...2, 0...5].nearly_eq(tuples[0][1]).all?
+    assert_true arrays[1][0, 2..-1, 0..-1].nearly_eq(0).all?
+    assert_true arrays[1][0, 0..-1, 5..-1].nearly_eq(0).all?
+    assert_true arrays[1][1, 0...3, 0...4].nearly_eq(tuples[1][1]).all?
+    #assert_true arrays[1][1, 3..-1, 0..-1].nearly_eq(0).all? # range error
+    assert_true arrays[1][1, 0..-1, 4..-1].nearly_eq(0).all?
+    assert_true arrays[1][2, 0...2, 0...6].nearly_eq(tuples[2][1]).all?
+    assert_true arrays[1][2, 2..-1, 0..-1].nearly_eq(0).all?
+  end
+
+  def test_concat_tuples_padding_cpu()
+    check_concat_tuples_padding(Numo)
+  end
+end
+
+class TestConcatExamplesWithBuiltInTypes < Test::Unit::TestCase
+  data = {
+    'test1' => {padding: nil},
+    'test2' => {padding: 0}}
+
+  @@int_arrays = [1, 2, 3]
+  @@float_arrays = [1.0, 2.0, 3.0]
+
+  def check_device(array, device)
+    if device && device >= 0
+      # T.B.I (GPU Check)
+    else
+      assert_true array.is_a?(Numo::NArray)
+    end
+  end
+
+  def check_concat_arrays(arrays, device:, expected_type:)
+    array = Chainer::Dataset::Convert.method(:concat_examples).call(arrays, device: device, padding: @padding)
+    assert_equal([arrays.size], array.shape)
+    check_device(array, device)
+
+    array.to_a.zip(arrays.to_a).each do |x, y|
+      assert_true Numo::NArray.cast(y).nearly_eq(Numo::NArray.cast(x)).all?
+    end
+  end
+
+  data(data)
+  def test_concat_arrays_cpu(data)
+    @padding = data[:padding]
+
+    [-1, nil].each do |device|
+      check_concat_arrays(@@int_arrays, device: device, expected_type: Numo::Int64)
+      check_concat_arrays(@@float_arrays, device: device, expected_type: Numo::DFloat)
+    end
+  end
+end