目錄
1. 聯(lián)邦學(xué)習(xí)介紹
2. 實(shí)驗(yàn)流程
3. 數(shù)據(jù)加載
4. 模型構(gòu)建
5. 數(shù)據(jù)采樣函數(shù)
6. 模型訓(xùn)練
1. 聯(lián)邦學(xué)習(xí)介紹
聯(lián)邦學(xué)習(xí)是一種分布式機(jī)器學(xué)習(xí)方法，中心節(jié)點(diǎn)為server（服務(wù)器），各分支節(jié)點(diǎn)為本地的client（設(shè)備）。聯(lián)邦學(xué)習(xí)的模式是在各分支節(jié)點(diǎn)分別利用本地?cái)?shù)據(jù)訓(xùn)練模型，再將訓(xùn)練好的模型匯合到中心節(jié)點(diǎn)，獲得一個(gè)更好的全局模型。
聯(lián)邦學(xué)習(xí)的提出是為了充分利用用戶的數(shù)據(jù)特征訓(xùn)練效果更佳的模型，同時(shí)，為了保證隱私，聯(lián)邦學(xué)習(xí)在訓(xùn)練過程中，server和clients之間通信的是模型的參數(shù)（或梯度、參數(shù)更新量），本地的數(shù)據(jù)不會(huì)上傳到服務(wù)器。
本項(xiàng)目主要是升級(jí)1.8版本的聯(lián)邦學(xué)習(xí)fedavg算法至2.3版本，內(nèi)容取材于基于PaddlePaddle實(shí)現(xiàn)聯(lián)邦學(xué)習(xí)算法FedAvg - 飛槳AI Studio星河社區(qū)
2. 實(shí)驗(yàn)流程
聯(lián)邦學(xué)習(xí)的基本流程是：
1. server初始化模型參數(shù)，所有的clients將這個(gè)初始模型下載到本地；
2. clients利用本地產(chǎn)生的數(shù)據(jù)進(jìn)行SGD訓(xùn)練；
3. 選取K個(gè)clients將訓(xùn)練得到的模型參數(shù)上傳到server；
4. server對(duì)得到的模型參數(shù)整合，所有的clients下載新的模型。
5. 重復(fù)執(zhí)行2-5，直至收斂或達(dá)到預(yù)期要求文章來源：http://www.zghlxwxcb.cn/news/detail-731116.html
import os
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import random
import time
import paddle
import paddle.nn as nn
import numpy as np
from paddle.io import Dataset,DataLoader
import paddle.nn.functional as F
3. 數(shù)據(jù)加載
mnist_data_train=np.load('data/data2489/train_mnist.npy')
mnist_data_test=np.load('data/data2489/test_mnist.npy')
print('There are {} images for training'.format(len(mnist_data_train)))
print('There are {} images for testing'.format(len(mnist_data_test)))
# 數(shù)據(jù)和標(biāo)簽分離（便于后續(xù)處理）
Label=[int(i[0]) for i in mnist_data_train]
Data=[i[1:] for i in mnist_data_train]
There are 60000 images for training
There are 10000 images for testing
4. 模型構(gòu)建
class CNN(nn.Layer):
def __init__(self):
super(CNN,self).__init__()
self.conv1=nn.Conv2D(1,32,5)
self.relu = nn.ReLU()
self.pool1=nn.MaxPool2D(kernel_size=2,stride=2)
self.conv2=nn.Conv2D(32,64,5)
self.pool2=nn.MaxPool2D(kernel_size=2,stride=2)
self.fc1=nn.Linear(1024,512)
self.fc2=nn.Linear(512,10)
# self.softmax = nn.Softmax()
def forward(self,inputs):
x = self.conv1(inputs)
x = self.relu(x)
x = self.pool1(x)
x = self.conv2(x)
x = self.relu(x)
x = self.pool2(x)
x=paddle.reshape(x,[-1,1024])
x = self.relu(self.fc1(x))
y = self.fc2(x)
return y
5. 數(shù)據(jù)采樣函數(shù)
# 均勻采樣，分配到各個(gè)client的數(shù)據(jù)集都是IID且數(shù)量相等的
def IID(dataset, clients):
num_items_per_client = int(len(dataset)/clients)
client_dict = {}
image_idxs = [i for i in range(len(dataset))]
for i in range(clients):
client_dict[i] = set(np.random.choice(image_idxs, num_items_per_client, replace=False)) # 為每個(gè)client隨機(jī)選取數(shù)據(jù)
image_idxs = list(set(image_idxs) - client_dict[i]) # 將已經(jīng)選取過的數(shù)據(jù)去除
client_dict[i] = list(client_dict[i])
return client_dict
# 非均勻采樣，同時(shí)各個(gè)client上的數(shù)據(jù)分布和數(shù)量都不同
def NonIID(dataset, clients, total_shards, shards_size, num_shards_per_client):
shard_idxs = [i for i in range(total_shards)]
client_dict = {i: np.array([], dtype='int64') for i in range(clients)}
idxs = np.arange(len(dataset))
data_labels = Label
label_idxs = np.vstack((idxs, data_labels)) # 將標(biāo)簽和數(shù)據(jù)ID堆疊
label_idxs = label_idxs[:, label_idxs[1,:].argsort()]
idxs = label_idxs[0,:]
for i in range(clients):
rand_set = set(np.random.choice(shard_idxs, num_shards_per_client, replace=False))
shard_idxs = list(set(shard_idxs) - rand_set)
for rand in rand_set:
client_dict[i] = np.concatenate((client_dict[i], idxs[rand*shards_size:(rand+1)*shards_size]), axis=0) # 拼接
return client_dict
class MNISTDataset(Dataset):
def __init__(self, data,label):
self.data = data
self.label = label
def __getitem__(self, idx):
image=np.array(self.data[idx]).astype('float32')
image=np.reshape(image,[1,28,28])
label=np.array(self.label[idx]).astype('int64')
return image, label
def __len__(self):
return len(self.label)
6. 模型訓(xùn)練
class ClientUpdate(object):
def __init__(self, data, label, batch_size, learning_rate, epochs):
dataset = MNISTDataset(data,label)
self.train_loader = DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
drop_last=True)
self.learning_rate = learning_rate
self.epochs = epochs
def train(self, model):
optimizer=paddle.optimizer.SGD(learning_rate=self.learning_rate,parameters=model.parameters())
criterion = nn.CrossEntropyLoss(reduction='mean')
model.train()
e_loss = []
for epoch in range(1,self.epochs+1):
train_loss = []
for image,label in self.train_loader:
# image=paddle.to_tensor(image)
# label=paddle.to_tensor(label.reshape([label.shape[0],1]))
output=model(image)
loss= criterion(output,label)
# print(loss)
loss.backward()
optimizer.step()
optimizer.clear_grad()
train_loss.append(loss.numpy()[0])
t_loss=sum(train_loss)/len(train_loss)
e_loss.append(t_loss)
total_loss=sum(e_loss)/len(e_loss)
return model.state_dict(), total_loss
train_x = np.array(Data)
train_y = np.array(Label)
BATCH_SIZE = 32
# 通信輪數(shù)
rounds = 100
# client比例
C = 0.1
# clients數(shù)量
K = 100
# 每次通信在本地訓(xùn)練的epoch
E = 5
# batch size
batch_size = 10
# 學(xué)習(xí)率
lr=0.001
# 數(shù)據(jù)切分
iid_dict = IID(mnist_data_train, 100)
def training(model, rounds, batch_size, lr, ds,L, data_dict, C, K, E, plt_title, plt_color):
global_weights = model.state_dict()
train_loss = []
start = time.time()
# clients與server之間通信
for curr_round in range(1, rounds+1):
w, local_loss = [], []
m = max(int(C*K), 1) # 隨機(jī)選取參與更新的clients
S_t = np.random.choice(range(K), m, replace=False)
for k in S_t:
# print(data_dict[k])
sub_data = ds[data_dict[k]]
sub_y = L[data_dict[k]]
local_update = ClientUpdate(sub_data,sub_y, batch_size=batch_size, learning_rate=lr, epochs=E)
weights, loss = local_update.train(model)
w.append(weights)
local_loss.append(loss)
# 更新global weights
weights_avg = w[0]
for k in weights_avg.keys():
for i in range(1, len(w)):
# weights_avg[k] += (num[i]/sum(num))*w[i][k]
weights_avg[k]=weights_avg[k]+w[i][k]
weights_avg[k]=weights_avg[k]/len(w)
global_weights[k].set_value(weights_avg[k])
# global_weights = weights_avg
# print(global_weights)
#模型加載最新的參數(shù)
model.load_dict(global_weights)
loss_avg = sum(local_loss) / len(local_loss)
if curr_round % 10 == 0:
print('Round: {}... \tAverage Loss: {}'.format(curr_round, np.round(loss_avg, 5)))
train_loss.append(loss_avg)
end = time.time()
fig, ax = plt.subplots()
x_axis = np.arange(1, rounds+1)
y_axis = np.array(train_loss)
ax.plot(x_axis, y_axis, 'tab:'+plt_color)
ax.set(xlabel='Number of Rounds', ylabel='Train Loss',title=plt_title)
ax.grid()
fig.savefig(plt_title+'.jpg', format='jpg')
print("Training Done!")
print("Total time taken to Train: {}".format(end-start))
return model.state_dict()
#導(dǎo)入模型
mnist_cnn = CNN()
mnist_cnn_iid_trained = training(mnist_cnn, rounds, batch_size, lr, train_x,train_y, iid_dict, C, K, E, "MNIST CNN on IID Dataset", "orange")
文章來源地址http://www.zghlxwxcb.cn/news/detail-731116.html
Round: 10... Average Loss: [0.024]
Round: 20... Average Loss: [0.015]
Round: 30... Average Loss: [0.008]
Round: 40... Average Loss: [0.003]
Round: 50... Average Loss: [0.004]
Round: 60... Average Loss: [0.002]
Round: 70... Average Loss: [0.002]
Round: 80... Average Loss: [0.002]
Round: 90... Average Loss: [0.001]
Round: 100... Average Loss: [0.]
Training Done!
Total time taken to Train: 759.6239657402039到了這里，關(guān)于paddle2.3-基于聯(lián)邦學(xué)習(xí)實(shí)現(xiàn)FedAVg算法-CNN的文章就介紹完了。如果您還想了解更多內(nèi)容，請(qǐng)?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！
