第1关：如何使用optimizer

import torch.nn as nn
import torch.optim
import torch
from torch.autograd import Variable

# Linear Regression Model
class LinearRegression(nn.Module):
    def __init__(self):
        super(LinearRegression, self).__init__()
        self.linear = nn.Linear(2, 2)  # input and output is 2 dimension
    
    def forward(self, x):
        out = self.linear(x)
        return out
model = LinearRegression()

#/********** Begin *********/

#声明一个 SGD优化器 optimizer，传入参数
optimizer= torch.optim.SGD(model.parameters(), lr = 0.1)
#利用optimizer.param_groups查看优化器的各项参数并输出lr的值。

print('lr:',optimizer.param_groups[0]['lr'])

#/********** End *********/

第2关：optim.SGD

import torch.nn as nn
import torch.optim
import torch
from torch.autograd import Variable


# Linear Regression Model
class LinearRegression(nn.Module):
    def __init__(self):
        super(LinearRegression, self).__init__()
        self.linear = nn.Linear(2, 2)  # input and output is 1 dimension
        self.linear2 = nn.Linear(2, 2)
    
    def forward(self, x):
        out = self.linear(x)
        out = self.linear2(out)
        return out
model = LinearRegression()


#/********** Begin *********/

#声明一个 SGD优化器 optimizer， 按要求设置 lr 的值
optimizer = torch.optim.SGD([
                {'params': model.linear.parameters(), 'lr': 1e-5},
                {'params':model.linear2.parameters()}
            ], lr=0.01, momentum=0.9)
#按照格式输出optimizer.param_groups的长度
print('The len of param_groups list:',len(optimizer.param_groups))
#按照格式输出linear层的lr
print("linear's lr:",optimizer.param_groups[0]['lr'])
#按照格式输出linear2层的lr
print("linear2's lr:",optimizer.param_groups[1]['lr'])
#/********** End *********/

第3关：RMSprop

import torch
from torch import nn, optim
from torch.autograd import Variable
import numpy as np
# Linear Regression Model
class LinearRegression(nn.Module):
    def __init__(self):
        super(LinearRegression, self).__init__()
        self.linear = nn.Linear(1,1)  # input and output is 1 dimension
    
    def forward(self, x):
        out = self.linear(x)
        return out

x_train = Variable(torch.randn(1,1))
y_train = Variable(torch.randn(1,1))
criterion = nn.MSELoss()

model = LinearRegression()

#/********** Begin *********/

#声明一个 RMSprop 优化器 optimizer， 按要求设置 lr,alpha 的值
optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1, alpha=0.9)  
#清空梯度
optimizer.zero_grad()
#计算Loss
loss = criterion(model(x_train), y_train)
#反向传播
loss.backward()
#更新参数
optimizer.step()
#按照格式输出optimizer的lr
print("optimizer's lr:",optimizer.param_groups[0]['lr'])
##按照格式输出optimizer的alpha
print("optimizer's alpha:",optimizer.param_groups[0]['alpha'])
#/********** End *********/

第4关：Adam

import torch
from torch import nn, optim
from torch.autograd import Variable
import numpy as np
# Linear Regression Model
class LinearRegression(nn.Module):
    def __init__(self):
        super(LinearRegression, self).__init__()
        self.linear = nn.Linear(2,2)  # input and output is 2 dimension
    
    def forward(self, x):
        out = self.linear(x)
        return out

x_train = Variable(torch.from_numpy(np.array([[1,2],[3,4]], dtype=np.float32)))
y_train = Variable(torch.from_numpy(np.array([[1,5],[2,8]], dtype=np.float32)))

model_Adam1 = LinearRegression()
model_Adam2 = LinearRegression()
models = [model_Adam1,model_Adam2]
#/********** Begin *********/

#声明一个Adam优化器 optimizer1， 设置 lr为0.2，betas为(0.9,0.9)
opt_Adam1 = torch.optim.Adam(model_Adam1.parameters(),lr=0.2,betas=(0.9,0.9))
#声明一个Adam优化器 optimizer2， 设置 lr为0.001，betas为(0.9,0.9)
opt_Adam2 = torch.optim.Adam(model_Adam2.parameters(),lr=0.001,betas=(0.9,0.9))

optimizers = [opt_Adam1,opt_Adam2]
losses_his = [[],[]]
loss_func = nn.MSELoss()

for epoch in range(10):
    # 对每个优化器, 优化属于他的神经网络
    for model,opt, l_his in zip(models,optimizers, losses_his):
        output = model(x_train)
        loss = loss_func(output, y_train)
        opt.zero_grad()
        loss.backward()
        opt.step()
        l_his.append(loss.data[0])
loss1 = sum(losses_his[0])
loss2 = sum(losses_his[1])

#利用 if-else 结构判断 loss1、loss2的大小
#若loss1小于loss2，输出“opt_Adam1 is better than opt_Adam2”；
#否则输出“opt_Adam2 is better than opt_Adam1”。
if loss1 > loss2:
    print("opt_Adam2 is  better than opt_Adam1")
else:
    print("opt_Adam1 is  better than opt_Adam2")

#/********** End *********/

第5关：优化器总结

import torch
import torch.utils.data as Data
import torch.nn.functional as F
from torch.autograd import Variable

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

import warnings
warnings.filterwarnings('ignore')

import os,sys
path = os.path.split(os.path.abspath(os.path.realpath(sys.argv[0])))[0] + os.path.sep
print("validation path:" ,path)

#定义参数
LR = 0.01
BATCH_SIZE = 32
EPOCH = 10

x = torch.unsqueeze(torch.linspace(-1, 1, 100), dim=1)
y = x.pow(2) + 0.1*torch.randn(x.size())
torch_dataset = Data.TensorDataset(data_tensor=x, target_tensor=y)
loader = Data.DataLoader(dataset=torch_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2,)

# 默认的 network 形式
class Net(torch.nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.hidden = torch.nn.Linear(1, 40)
        self.predict = torch.nn.Linear(40, 1)
    
    def forward(self, x):
        #隐藏层的激活函数
        x = F.relu(self.hidden(x))
        #线性输出
        x = self.predict(x)
        return x


net_SGD         = Net()
net_Momentum    = Net()
net_RMSprop     = Net()
net_Adam        = Net()
nets = [net_SGD, net_Momentum, net_RMSprop, net_Adam]

#/********** Begin *********/
opt_SGD         = torch.optim.SGD(net_SGD.parameters(), lr=LR)

# 声明优化器opt_Momentum，传入对应的模型参数，lr 赋值为 LR，momentum为0.7
opt_Momentum = torch.optim.SGD(net_Momentum.parameters(),lr=LR,momentum=0.7)
# 声明优化器opt_RMSprop，传入对应的模型参数，lr 赋值为 LR，alpha为0.9
opt_RMSprop = torch.optim.RMSprop(net_RMSprop.parameters(),lr=LR,alpha=0.9)
# 声明优化器opt_Adam，传入对应的模型参数，lr 赋值为 LR，betas为(0.9, 0.99)
opt_Adam = torch.optim.Adam(net_Adam.parameters(),lr=LR,betas=(0.9,0.99))
#/********** End *********/
optimizers = [opt_SGD, opt_Momentum, opt_RMSprop, opt_Adam]
loss_func = torch.nn.MSELoss()
losses_his = [[], [], [], []]

#训练循环
for epoch in range(EPOCH):
    for step, (batch_x, batch_y) in enumerate(loader):
        b_x = Variable(batch_x)
        b_y = Variable(batch_y)
        
        for net, opt, l_his in zip(nets, optimizers, losses_his):
            output = net(b_x)
            loss = loss_func(output, b_y)

         #/********** Begin *********/
            opt.zero_grad()
            #反向传播 计算梯度
            loss.backward()
            #更新梯度
            opt.step()
            l_his.append(loss.data[0])
         #/********** End *********/
         
         
#画图
labels = ['SGD', 'Momentum', 'RMSprop', 'Adam']
for i, l_his in enumerate(losses_his):
    plt.plot(l_his, label=labels[i])
plt.legend(loc='best')
plt.xlabel('Steps')
plt.ylabel('Loss')
plt.ylim((0, 0.2))
plt.savefig(path + "outputimages/mylossTest.png")