The following example uses Pytorch's low-level API to implement a linear regression model and a DNN binary classification model.
The low-level API mainly includes tensor operations, calculation graphs and automatic differentiation.
import os
import datetime
#Print Time
def printbar():
nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print("\n"+"=========="*8 + "%s"%nowtime)
#Mac system pytorch and matplotlib running at the same time in jupyter need to change environment variables
os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"1, prepare data
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import torch
from torch import nn
#Number of samples
n = 400
# Generate test data set
X = 10*torch.rand([n,2])-5.0 #torch.rand is uniform distribution
w0 = torch.tensor([[2.0],[-3.0]])
b0 = torch.tensor([[10.0]])
Y = X@w0 + b0 + torch.normal( 0.0,2.0,size = [n,1]) # @ means matrix multiplication, increase normal disturbance# data visualization
%matplotlib inline
%config InlineBackend.figure_format ='svg'
plt.figure(figsize = (12,5))
ax1 = plt.subplot(121)
ax1.scatter(X[:,0].numpy(),Y[:,0].numpy(), c = "b",label = "samples")
ax1.legend()
plt.xlabel("x1")
plt.ylabel("y",rotation = 0)
ax2 = plt.subplot(122)
ax2.scatter(X[:,1].numpy(),Y[:,0].numpy(), c = "g",label = "samples")
ax2.legend()
plt.xlabel("x2")
plt.ylabel("y",rotation = 0)
plt.show()
# Build a data pipeline iterator
def data_iter(features, labels, batch_size=8):
num_examples = len(features)
indices = list(range(num_examples))
np.random.shuffle(indices) #The reading order of samples is random
for i in range(0, num_examples, batch_size):
indexs = torch.LongTensor(indices[i: min(i + batch_size, num_examples)])
yield features.index_select(0, indexs), labels.index_select(0, indexs)
# Test data pipeline effect
batch_size = 8
(features,labels) = next(data_iter(X,Y,batch_size))
print(features)
print(labels)tensor([[-4.3880, 1.3655],
[-0.1082, 3.9533],
[-2.6286, 2.7058],
[1.0604, -1.8646],
[-1.5805, 1.5406],
[-2.6217, -3.2342],
[2.3748, -0.6449],
[-1.2478, -2.0509]])
tensor([[-0.2069],
[-3.2494],
[-6.9620],
[17.0528],
[1.1076],
[17.2117],
[16.1081],
[14.7092]])
2, define the model
# Define model
class LinearRegression:
def __init__(self):
self.w = torch.randn_like(w0,requires_grad=True)
self.b = torch.zeros_like(b0,requires_grad=True)
#Forward spread
def forward(self,x):
return x@self.w + self.b
# Loss function
def loss_func(self,y_pred,y_true):
return torch.mean((y_pred-y_true)**2/2)
model = LinearRegression()3, training model
def train_step(model, features, labels):
predictions = model.forward(features)
loss = model.loss_func(predictions,labels)
# Backpropagation for gradient
loss.backward()
# Use torch.no_grad() to avoid gradient recording, or to avoid gradient recording by operating model.w.data
with torch.no_grad():
# Gradient descent method update parameters
model.w -= 0.001*model.w.grad
model.b -= 0.001*model.b.grad
# Gradient clear
model.w.grad.zero_()
model.b.grad.zero_()
return loss
# Test the effect of train_step
batch_size = 10
(features,labels) = next(data_iter(X,Y,batch_size))
train_step(model,features,labels)tensor(92.8199, grad_fn=<MeanBackward0>)
def train_model(model,epochs):
for epoch in range(1,epochs+1):
for features, labels in data_iter(X,Y,10):
loss = train_step(model,features,labels)
if epoch%200==0:
printbar()
print("epoch =",epoch,"loss = ",loss.item())
print("model.w =",model.w.data)
print("model.b =",model.b.data)
train_model(model,epochs = 1000)================================================= ==============================2020-07-05 08:27:57
epoch = 200 loss = 2.6340413093566895
model.w = tensor([[ 2.0283],
[-2.9632]])
model.b = tensor([[10.0748]])
================================================= ==============================2020-07-05 08:28:00
epoch = 400 loss = 2.24908709526062
model.w = tensor([[ 2.0300],
[-2.9643]])
model.b = tensor([[10.0781]])
================================================== ==============================2020-07-05 08:28:04
epoch = 600 loss = 1.510349154472351
model.w = tensor([[ 2.0290],
[-2.9630]])
model.b = tensor([[10.0781]])
================================================= ==============================2020-07-05 08:28:07
epoch = 800 loss = 1.038671851158142
model.w = tensor([[ 2.0314],
[-2.9649]])
model.b = tensor([[10.0785]])
================================================= ==============================2020-07-05 08:28:10
epoch = 1000 loss = 1.9742190837860107
model.w = tensor([[ 2.0313],
[-2.9648]])
model.b = tensor([[10.0781]])
# Result visualization
%matplotlib inline
%config InlineBackend.figure_format ='svg'
plt.figure(figsize = (12,5))
ax1 = plt.subplot(121)
ax1.scatter(X[:,0].numpy(),Y[:,0].numpy(), c = "b",label = "samples")
ax1.plot(X[:,0].numpy(),(model.w[0].data*X[:,0]+model.b[0].data).numpy(),"-r" ,linewidth = 5.0,label = "model")
ax1.legend()
plt.xlabel("x1")
plt.ylabel("y",rotation = 0)
ax2 = plt.subplot(122)
ax2.scatter(X[:,1].numpy(),Y[:,0].numpy(), c = "g",label = "samples")
ax2.plot(X[:,1].numpy(),(model.w[1].data*X[:,1]+model.b[0].data).numpy(),"-r" ,linewidth = 5.0,label = "model")
ax2.legend()
plt.xlabel("x2")
plt.ylabel("y",rotation = 0)
plt.show()
1, prepare data
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import torch
from torch import nn
%matplotlib inline
%config InlineBackend.figure_format ='svg'
#Number of positive and negative samples
n_positive,n_negative = 2000,2000
#Generate positive samples, small circle distribution
r_p = 5.0 + torch.normal(0.0,1.0,size = [n_positive,1])
theta_p = 2*np.pi*torch.rand([n_positive,1])
Xp = torch.cat([r_p*torch.cos(theta_p),r_p*torch.sin(theta_p)],axis = 1)
Yp = torch.ones_like(r_p)
#Generate negative samples, large circle distribution
r_n = 8.0 + torch.normal(0.0,1.0,size = [n_negative,1])
theta_n = 2*np.pi*torch.rand([n_negative,1])
Xn = torch.cat([r_n*torch.cos(theta_n),r_n*torch.sin(theta_n)],axis = 1)
Yn = torch.zeros_like(r_n)
#Summary sample
X = torch.cat([Xp,Xn],axis = 0)
Y = torch.cat([Yp,Yn],axis = 0)
#Visualization
plt.figure(figsize = (6,6))
plt.scatter(Xp[:,0].numpy(),Xp[:,1].numpy(),c = "r")
plt.scatter(Xn[:,0].numpy(),Xn[:,1].numpy(),c = "g")
plt.legend(["positive","negative"]);
# Build a data pipeline iterator
def data_iter(features, labels, batch_size=8):
num_examples = len(features)
indices = list(range(num_examples))
np.random.shuffle(indices) #The reading order of samples is random
for i in range(0, num_examples, batch_size):
indexs = torch.LongTensor(indices[i: min(i + batch_size, num_examples)])
yield features.index_select(0, indexs), labels.index_select(0, indexs)
# Test data pipeline effect
batch_size = 8
(features,labels) = next(data_iter(X,Y,batch_size))
print(features)
print(labels)tensor([[ 6.9914, -1.0820],
[4.8156, 4.0532],
[-1.0697, -7.4644],
[2.6291, 3.8851],
[-1.6780, -4.3390],
[-6.1495, 1.2269],
[-4.3422, 3.9552],
[-6.2265, 2.6159]])
tensor([[0.],
[1.],
[0.],
[1.],
[1.],
[1.],
[1.],
[1.]])
2, define the model
In this example, we use nn.Module to organize model variables.
class DNNModel(nn.Module):
def __init__(self):
super(DNNModel, self).__init__()
self.w1 = nn.Parameter(torch.randn(2,4))
self.b1 = nn.Parameter(torch.zeros(1,4))
self.w2 = nn.Parameter(torch.randn(4,8))
self.b2 = nn.Parameter(torch.zeros(1,8))
self.w3 = nn.Parameter(torch.randn(8,1))
self.b3 = nn.Parameter(torch.zeros(1,1))
# Forward spread
def forward(self,x):
x = torch.relu(x@self.w1 + self.b1)
x = torch.relu(x@self.w2 + self.b2)
y = torch.sigmoid(x@self.w3 + self.b3)
return y
# Loss function (binary cross entropy)
def loss_func(self,y_pred,y_true):
#Limit the predicted value above 1e-7 and below 1- (1e-7) to avoid log(0) errors
eps = 1e-7
y_pred = torch.clamp(y_pred,eps,1.0-eps)
bce =-y_true*torch.log(y_pred)-(1-y_true)*torch.log(1-y_pred)
return torch.mean(bce)
# Evaluation index (accuracy rate)
def metric_func(self,y_pred,y_true):
y_pred = torch.where(y_pred>0.5,torch.ones_like(y_pred,dtype = torch.float32),
torch.zeros_like(y_pred,dtype = torch.float32))
acc = torch.mean(1-torch.abs(y_true-y_pred))
return acc
model = DNNModel()# Test model structure
batch_size = 10
(features,labels) = next(data_iter(X,Y,batch_size))
predictions = model(features)
loss = model.loss_func(labels,predictions)
metric = model.metric_func(labels,predictions)
print("init loss:", loss.item())
print("init metric:", metric.item())init loss: 7.979694366455078
init metric: 0.50347900390625
len(list(model.parameters()))6
3, training model
def train_step(model, features, labels):
# Forward propagation for loss
predictions = model.forward(features)
loss = model.loss_func(predictions,labels)
metric = model.metric_func(predictions,labels)
# Backpropagation for gradient
loss.backward()
# Gradient descent method update parameters
for param in model.parameters():
#Note is to reassign param.data to avoid gradient recording caused by the operation here
param.data = (param.data-0.01*param.grad.data)
# Gradient clear
model.zero_grad()
return loss.item(),metric.item()
def train_model(model,epochs):
for epoch in range(1,epochs+1):
loss_list,metric_list = [],[]
for features, labels in data_iter(X,Y,20):
lossi,metrici = train_step(model,features,labels)
loss_list.append(lossi)
metric_list.append(metrici)
loss = np.mean(loss_list)
metric = np.mean(metric_list)
if epoch%100==0:
printbar()
print("epoch =",epoch,"loss = ",loss,"metric = ",metric)
train_model(model,epochs = 1000)================================================= ==============================2020-07-05 08:32:16
epoch = 100 loss = 0.24841043589636683 metric = 0.8944999960064888
================================================= =============================2020-07-05 08:32:34
epoch = 200 loss = 0.20398724960163236 metric = 0.920999992787838
================================================= ==============================2020-07-05 08:32:54
epoch = 300 loss = 0.19509393003769218 metric = 0.9239999914169311
================================================= ==============================2020-07-05 08:33:14
epoch = 400 loss = 0.19067603485658766 metric = 0.9272499939799309
================================================= ==============================2020-07-05 08:33:33
epoch = 500 loss = 0.1898010154720396 metric = 0.9237499925494194
================================================= ==============================2020-07-05 08:33:54
epoch = 600 loss = 0.19151576517149807 metric = 0.9254999926686287
================================================= ==============================2020-07-05 08:34:18
epoch = 700 loss = 0.18914461021777243 metric = 0.9274999949336052
================================================= ==============================2020-07-05 08:34:39
epoch = 800 loss = 0.18801998342387377 metric = 0.9264999932050705
================================================= ==============================2020-07-05 08:35:00
epoch = 900 loss = 0.1852504052128643 metric = 0.9249999937415123
================================================= ==============================2020-07-05 08:35:21
epoch = 1000 loss = 0.18695520935580134 metric = 0.9272499927878379
# Result visualization
fig, (ax1,ax2) = plt.subplots(nrows=1,ncols=2,figsize = (12,5))
ax1.scatter(Xp[:,0],Xp[:,1], c="r")
ax1.scatter(Xn[:,0],Xn[:,1],c = "g")
ax1.legend(["positive","negative"]);
ax1.set_title("y_true");
Xp_pred = X[torch.squeeze(model.forward(X)>=0.5)]
Xn_pred = X[torch.squeeze(model.forward(X)<0.5)]
ax2.scatter(Xp_pred[:,0],Xp_pred[:,1],c = "r")
ax2.scatter(Xn_pred[:,0],Xn_pred[:,1],c = "g")
ax2.legend(["positive","negative"]);
ax2.set_title("y_pred");
If this book is helpful to you and want to encourage the author, remember to add a star to this project, and share it with your friends 😊!
If you need to further communicate with the author on the understanding of the content of this book, please leave a message under the public account "Algorithm Food House". The author has limited time and energy and will respond as appropriate.
You can also reply to keywords in the background of the official account: Add group, join the reader exchange group and discuss with you.
