kb4020102(KB4012212)

前沿拓展：

kb4020102

1、Windows10创意者版积累性更新15063.332（KB4020过试料客长双矿员待护富102）已经于5月26日向创意者正式版用户推送，该补丁包含了很多问题的问题。

前言

实现一台服务器的百万并发，服务器支撑百万连接会出现哪些问题，如何排查与解决这些问题是本文的重点

服务器能够同时建立连接的数量不是并发量，它只是并发量一个基础。服务器的并发量：一个服务器能够同时承载客户端的数量；承载：服务器能够稳定的维持这些连接，能够响应请求，在200ms内返回响应就认为是ok的，其中这200ms包括数据库的**作，网络带宽，内存**作，日志等时间。测试介绍

服务器采用 1台 centos7 12G 1核虚拟机

客户端采用 2台 centos7 3G 1核虚拟机

服务器代码：单reactor单线程，IO多路复用使用epoll

客户端代码：IO多路复用使用epoll，每个客户端发51w个连接，每个连接发送一次数据，读取一次数据之后不再发送数据

服务器代码

由于fd的数量未知，这里设计ntyreactor 里面包含 eventblock ，eventblock 包含1024个fd。每个fd通过 fd/1024**到在第几个eventblock，通过fd%1024**到在eventblock第几个位置。

struct ntyevent {
int fd;
int events;
void *arg;

NCALLBACK callback;

int status;
char buffer[BUFFER_LENGTH];
int length;
};
struct eventblock {
struct eventblock *next;
struct ntyevent *events;
};

struct ntyreactor {
int epfd;
int blkcnt;
struct eventblock *evblk;
};

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>

#define BUFFER_LENGTH 4096
#define MAX_EPOLL_EVENTS 1024
#define SERVER_PORT 8081
#define PORT_COUNT 100

typedef int (*NCALLBACK)(int, int, void *);

struct ntyevent {
int fd;
int events;
void *arg;

NCALLBACK callback;

int status;
char buffer[BUFFER_LENGTH];
int length;
};
struct eventblock {
struct eventblock *next;
struct ntyevent *events;
};

struct ntyreactor {
int epfd;
int blkcnt;
struct eventblock *evblk;
};

int recv_cb(int fd, int events, void *arg);

int send_cb(int fd, int events, void *arg);

struct ntyevent *ntyreactor_find_event_idx(struct ntyreactor *reactor, int sockfd);

void nty_event_set(struct ntyevent *ev, int fd, NCALLBACK *callback, void *arg) {
ev->fd = fd;
ev->callback = callback;
ev->events = 0;
ev->arg = arg;
}

int nty_event_add(int epfd, int events, struct ntyevent *ev) {
struct epoll_event ep_ev = {0, {0}};
ep_ev.data.ptr = ev;
ep_ev.events = ev->events = events;
int op;
if (ev->status == 1) {
op = EPOLL_CTL_MOD;
}
else {
op = EPOLL_CTL_ADD;
ev->status = 1;
}
if (epoll_ctl(epfd, op, ev->fd, &ep_ev) < 0) {
printf("event add failed [fd=%d], events[%d]n", ev->fd, events);
return -1;
}
return 0;
}

int nty_event_del(int epfd, struct ntyevent *ev) {
struct epoll_event ep_ev = {0, {0}};
if (ev->status != 1) {
return -1;
}
ep_ev.data.ptr = ev;
ev->status = 0;
epoll_ctl(epfd, EPOLL_CTL_DEL, ev->fd, &ep_ev);
return 0;
}

int recv_cb(int fd, int events, void *arg) {
struct ntyreactor *reactor = (struct ntyreactor *) arg;
struct ntyevent *ev = ntyreactor_find_event_idx(reactor, fd);
int len = recv(fd, ev->buffer, BUFFER_LENGTH, 0); //
nty_event_del(reactor->epfd, ev);

if (len > 0) {
ev->length = len;
ev->buffer[len] = '';
// printf("recv[%d]:%sn", fd, ev->buffer);
printf("recv fd=[%dn", fd);

nty_event_set(ev, fd, send_cb, reactor);
nty_event_add(reactor->epfd, EPOLLOUT, ev);
}
else if (len == 0) {
close(ev->fd);
//printf("[fd=%d] pos[%ld], closedn", fd, ev-reactor->events);
}
else {
close(ev->fd);
// printf("recv[fd=%d] error[%d]:%sn", fd, errno, strerror(errno));
}
return len;
}

int send_cb(int fd, int events, void *arg) {
struct ntyreactor *reactor = (struct ntyreactor *) arg;
struct ntyevent *ev = ntyreactor_find_event_idx(reactor, fd);

int len = send(fd, ev->buffer, ev->length, 0);
if (len > 0) {
// printf("send[fd=%d], [%d]%sn", fd, len, ev->buffer);
printf("send fd=[%dn]", fd);

nty_event_del(reactor->epfd, ev);
nty_event_set(ev, fd, recv_cb, reactor);
nty_event_add(reactor->epfd, EPOLLIN, ev);
}
else {
nty_event_del(reactor->epfd, ev);
close(ev->fd);
printf("send[fd=%d] error %sn", fd, strerror(errno));
}
return len;
}

int accept_cb(int fd, int events, void *arg) {//非阻塞
struct ntyreactor *reactor = (struct ntyreactor *) arg;
if (reactor == NULL) return -1;

struct sockaddr_in client_addr;
socklen_t len = sizeof(client_addr);

int clientfd;
if ((clientfd = accept(fd, (struct sockaddr *) &client_addr, &len)) == -1) {
printf("accept: %sn", strerror(errno));
return -1;
}
if ((fcntl(clientfd, F_SETFL, O_NONBLOCK)) < 0) {
printf("%s: fcntl nonblocking failed, %dn", __func__, MAX_EPOLL_EVENTS);
return -1;
}
struct ntyevent *event = ntyreactor_find_event_idx(reactor, clientfd);

nty_event_set(event, clientfd, recv_cb, reactor);
nty_event_add(reactor->epfd, EPOLLIN, event);

printf("new connect [%s:%d], pos[%d]n",
inet_ntoa(client_addr.sin_addr), ntohs(client_addr.sin_port), clientfd);
return 0;
}

int init_sock(short port) {
int fd = socket(AF_INET, SOCK_STREAM, 0);
fcntl(fd, F_SETFL, O_NONBLOCK);
struct sockaddr_in server_addr;
memset(&server_addr, 0, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
server_addr.sin_port = htons(port);

bind(fd, (struct sockaddr *) &server_addr, sizeof(server_addr));

if (listen(fd, 20) < 0) {
printf("listen failed : %sn", strerror(errno));
}
return fd;
}

int ntyreactor_alloc(struct ntyreactor *reactor) {
if (reactor == NULL) return -1;
if (reactor->evblk == NULL) return -1;

struct eventblock *blk = reactor->evblk;
while (blk->next != NULL) {
blk = blk->next;
}

struct ntyevent *evs = (struct ntyevent *) malloc((MAX_EPOLL_EVENTS) * sizeof(struct ntyevent));
if (evs == NULL) {
printf("ntyreactor_alloc ntyevents failedn");
return -2;
}
memset(evs, 0, (MAX_EPOLL_EVENTS) * sizeof(struct ntyevent));

struct eventblock *block = (struct eventblock *) malloc(sizeof(struct eventblock));
if (block == NULL) {
printf("ntyreactor_alloc eventblock failedn");
return -2;
}
memset(block, 0, sizeof(struct eventblock));

block->events = evs;
block->next = NULL;

blk->next = block;
reactor->blkcnt++; //
return 0;
}

struct ntyevent *ntyreactor_find_event_idx(struct ntyreactor *reactor, int sockfd) {
int blkidx = sockfd / MAX_EPOLL_EVENTS;

while (blkidx >= reactor->blkcnt) {
ntyreactor_alloc(reactor);
}
int i = 0;
struct eventblock *blk = reactor->evblk;
while (i++ < blkidx && blk != NULL) {
blk = blk->next;
}
return &blk->events[sockfd % MAX_EPOLL_EVENTS];
}

int ntyreactor_init(struct ntyreactor *reactor) {
if (reactor == NULL) return -1;
memset(reactor, 0, sizeof(struct ntyreactor));

reactor->epfd = epoll_create(1);
if (reactor->epfd <= 0) {
printf("create epfd in %s err %sn", __func__, strerror(errno));
return -2;
}

block->events = evs;
block->next = NULL;

reactor->evblk = block;
reactor->blkcnt = 1;
return 0;
}

int ntyreactor_destory(struct ntyreactor *reactor) {
close(reactor->epfd);
//free(reactor->events);

struct eventblock *blk = reactor->evblk;
struct eventblock *blk_next = NULL;

while (blk != NULL) {
blk_next = blk->next;
free(blk->events);
free(blk);
blk = blk_next;
}
return 0;
}

int ntyreactor_addlistener(struct ntyreactor *reactor, int sockfd, NCALLBACK *acceptor) {
if (reactor == NULL) return -1;
if (reactor->evblk == NULL) return -1;

struct ntyevent *event = ntyreactor_find_event_idx(reactor, sockfd);

nty_event_set(event, sockfd, acceptor, reactor);
nty_event_add(reactor->epfd, EPOLLIN, event);
return 0;
}

_Noreturn int ntyreactor_run(struct ntyreactor *reactor) {
if (reactor == NULL) return -1;
if (reactor->epfd < 0) return -1;
if (reactor->evblk == NULL) return -1;

struct epoll_event events[MAX_EPOLL_EVENTS + 1];

int i;

while (1) {
int nready = epoll_wait(reactor->epfd, events, MAX_EPOLL_EVENTS, 1000);
if (nready < 0) {
printf("epoll_wait error, exitn");
continue;
}
for (i = 0; i < nready; i++) {
struct ntyevent *ev = (struct ntyevent *) events[i].data.ptr;
if ((events[i].events & EPOLLIN) && (ev->events & EPOLLIN)) {
ev->callback(ev->fd, events[i].events, ev->arg);
}
if ((events[i].events & EPOLLOUT) && (ev->events & EPOLLOUT)) {
ev->callback(ev->fd, events[i].events, ev->arg);
}
}
}
}

// <remoteip, remoteport, localip, localport,protocol>
int main(int argc, char *argv[]) {
unsigned short port = SERVER_PORT; // listen 8081
if (argc == 2) {
port = atoi(argv[1]);
}
struct ntyreactor *reactor = (struct ntyreactor *) malloc(sizeof(struct ntyreactor));
ntyreactor_init(reactor);
int i = 0;
int sockfds[PORT_COUNT] = {0};
for (i = 0; i < PORT_COUNT; i++) {
sockfds[i] = init_sock(port + i);
ntyreactor_addlistener(reactor, sockfds[i], accept_cb);
}
ntyreactor_run(reactor);
ntyreactor_destory(reactor);
for (i = 0; i < PORT_COUNT; i++) {
close(sockfds[i]);
}
free(reactor);
return 0;
}

我们只需要对net.ipv4.tcp_mem，net.ipv4.tcp_wmem，net.ipv4.tcp_rmem进行适合的修改即可

# 编辑内核参数配置文件
vim /etc/sysctl.conf

# 添加以下内容
# 最小值默认值最大值
net.ipv4.tcp_mem = 252144 524288 786432 # tcp协议栈的大小，单位为内存页（4K），分别是 1G 2G 3G，如果大于2G，tcp协议栈会进行一定的优化
net.ipv4.tcp_wmem = 1024 1024 2048 # tcp接收缓存区(用于tcp接受滑动窗口)的最小值，默认值和最大值（单位byte）1k 1k 2k,每一个连接fd都有一个接收缓存区
net.ipv4.tcp_rmem = 1024 1024 2048 # tcp发送缓存区(用于tcp发送滑动窗口)的最小值，默认值和最大值（单位byte）1k 1k 2k,每一个连接fd都有一个发送缓存区

# 总缓存 = （每个fd发送缓存区 + 每个fd接收缓存区） * fd数量
# （1024byte + 1024byte ） * 100w 约等于 2G

如果服务器是用来接收大文件，传输量很大的时候，就要把send buffer和read buffer调大。

如果服务器只是接收小数据字符的时候。把buffer调小是为了把fd的数量做到更多，并发数量能做到更大。如果buffer调大的话，内存会不够。

百万并发测试结果

出现的问题小编综合来说

想要实现服务器百万并发：

一个进程能够打开文件描述符的数量open files 和 file-max 改成100w以上在不同的环境下要看开放的端口够不够socket fd — < 源IP地址 , 源端口 , 目的IP地址 , 目的端口 , 协议 >设置netfilter允许对外最大连接数量100w以上根据内存和场景，适当调整net.ipv4.tcp_mem，net.ipv4.tcp_wmem，net.ipv4.tcp_rmem

原文地址：Linux服务器百万并发实现与问题排查