cat /proc/sys/fs/file-max

#define __FD_SETSIZE    1024

#include <sys/epoll.h>

int epoll_create(int size);
int epoll_ctl(int epfd, int op, int fd, struct epoll_events* event);
int epoll_wait(int epfd, struct epoll_event* events, int maxevents, int timeout);

import socket
import select

EOL1 = b'\n\n'
EOL2 = b'\n\r\n'
response = b'HTTP/1.0 200 OK\r\nDate: Mon, 1 Jan 1996 01:01:01 GMT\r\n'
response += b'Content-Type: text/plain\r\nContent-Length: 13\r\n\r\n'
response += b'Hello, world!'

# 創(chuàng)建套接字對象并綁定監(jiān)聽端口
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serversocket.bind(('0.0.0.0', 8080))
serversocket.listen(1)
serversocket.setblocking(0)

# 創(chuàng)建epoll對象，并注冊socket對象的 epoll可讀事件
epoll = select.epoll()
epoll.register(serversocket.fileno(), select.EPOLLIN)

try:
    connections = {}
    requests = {}
    responses = {}
    while True:
        # 主循環(huán)，epoll的系統(tǒng)調(diào)用，一旦有網(wǎng)絡(luò)IO事件發(fā)生，poll調(diào)用返回。這是和select系統(tǒng)調(diào)用的關(guān)鍵區(qū)別
        events = epoll.poll(1)
        # 通過事件通知獲得監(jiān)聽的文件描述符，進(jìn)而處理
        for fileno, event in events:
            # 注冊監(jiān)聽的socket對象可讀，獲取連接，并注冊連接的可讀事件
            if fileno == serversocket.fileno():
                connection, address = serversocket.accept()
                connection.setblocking(0)
                epoll.register(connection.fileno(), select.EPOLLIN)
                connections[connection.fileno()] = connection
                requests[connection.fileno()] = b''
                responses[connection.fileno()] = response
            elif event & select.EPOLLIN:
                # 連接對象可讀，處理客戶端發(fā)生的信息，并注冊連接對象可寫
                requests[fileno] += connections[fileno].recv(1024)
                if EOL1 in requests[fileno] or EOL2 in requests[fileno]:
                    epoll.modify(fileno, select.EPOLLOUT)
                    print('-' * 40 + '\n' + requests[fileno].decode()[:-2])
            elif event & select.EPOLLOUT:
                # 連接對象可寫事件發(fā)生，發(fā)送數(shù)據(jù)到客戶端
                byteswritten = connections[fileno].send(responses[fileno])
                responses[fileno] = responses[fileno][byteswritten:]
                if len(responses[fileno]) == 0:
                    epoll.modify(fileno, 0)
                    connections[fileno].shutdown(socket.SHUT_RDWR)
            elif event & select.EPOLLHUP:
                epoll.unregister(fileno)
                connections[fileno].close()
                del connections[fileno]
finally:
    epoll.unregister(serversocket.fileno())
    epoll.close()
    serversocket.close()

import errno
import functools
import tornado.ioloop
import socket


def handle_connection(connection, address):
    """ 處理請求，返回數(shù)據(jù)給客戶端 """
    data = connection.recv(2014)
    print data
    connection.send(data)


def connection_ready(sock, fd, events):
    """ 事件回調(diào)函數(shù)，主要用于socket可讀事件，用于獲取socket的鏈接 """
    while True:
        try:
            connection, address = sock.accept()
        except socket.error as e:
            if e.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN):
                raise
            return
        connection.setblocking(0)
        handle_connection(connection, address)


if __name__ == '__main__':
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.setblocking(0)
    sock.bind(("", 5000))
    sock.listen(128)
    # 使用tornado封裝好的epoll接口，即IOLoop對象
    io_loop = tornado.ioloop.IOLoop.current()
    callback = functools.partial(connection_ready, sock)
    # io_loop對象注冊網(wǎng)絡(luò)io文件描述符和回調(diào)函數(shù)與io事件的綁定
    io_loop.add_handler(sock.fileno(), callback, io_loop.READ)
    io_loop.start()

__poll_t poll(struct file *fp, poll_table *wait)

/*
 *  fs/eventpoll.c (Efficient event retrieval implementation)
 *  Copyright (C) 2001,...,2009	 Davide Libenzi
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  Davide Libenzi <davidel@xmailserver.org>
 *
 */
/*
 * 在深入了解epoll的實現(xiàn)之前, 先來了解內(nèi)核的3個方面.
 * 1. 等待隊列 waitqueue
 * 我們簡單解釋一下等待隊列:
 * 隊列頭(wait_queue_head_t)往往是資源生產(chǎn)者,
 * 隊列成員(wait_queue_t)往往是資源消費者,
 * 當(dāng)頭的資源ready后, 會逐個執(zhí)行每個成員指定的回調(diào)函數(shù),
 * 來通知它們資源已經(jīng)ready了, 等待隊列大致就這個意思.
 * 2. 內(nèi)核的poll機(jī)制
 * 被Poll的fd, 必須在實現(xiàn)上支持內(nèi)核的Poll技術(shù),
 * 比如fd是某個字符設(shè)備,或者是個socket, 它必須實現(xiàn)
 * file_operations中的poll操作, 給自己分配有一個等待隊列頭.
 * 主動poll fd的某個進(jìn)程必須分配一個等待隊列成員, 添加到
 * fd的對待隊列里面去, 并指定資源ready時的回調(diào)函數(shù).
 * 用socket做例子, 它必須有實現(xiàn)一個poll操作, 這個Poll是
 * 發(fā)起輪詢的代碼必須主動調(diào)用的, 該函數(shù)中必須調(diào)用poll_wait(),
 * poll_wait會將發(fā)起者作為等待隊列成員加入到socket的等待隊列中去.
 * 這樣socket發(fā)生狀態(tài)變化時可以通過隊列頭逐個通知所有關(guān)心它的進(jìn)程.
 * 這一點必須很清楚的理解, 否則會想不明白epoll是如何
 * 得知fd的狀態(tài)發(fā)生變化的.
 * 3. epollfd本身也是個fd, 所以它本身也可以被epoll,
 * 可以猜測一下它是不是可以無限嵌套epoll下去... 
 *
 * epoll基本上就是使用了上面的1,2點來完成.
 * 可見epoll本身并沒有給內(nèi)核引入什么特別復(fù)雜或者高深的技術(shù),
 * 只不過是已有功能的重新組合, 達(dá)到了超過select的效果.
 */
/* 
 * 相關(guān)的其它內(nèi)核知識:
 * 1. fd我們知道是文件描述符, 在內(nèi)核態(tài), 與之對應(yīng)的是struct file結(jié)構(gòu),
 * 可以看作是內(nèi)核態(tài)的文件描述符.
 * 2. spinlock, 自旋鎖, 必須要非常小心使用的鎖,
 * 尤其是調(diào)用spin_lock_irqsave()的時候, 中斷關(guān)閉, 不會發(fā)生進(jìn)程調(diào)度,
 * 被保護(hù)的資源其它CPU也無法訪問. 這個鎖是很強(qiáng)力的, 所以只能鎖一些
 * 非常輕量級的操作.
 * 3. 引用計數(shù)在內(nèi)核中是非常重要的概念,
 * 內(nèi)核代碼里面經(jīng)常有些release, free釋放資源的函數(shù)幾乎不加任何鎖,
 * 這是因為這些函數(shù)往往是在對象的引用計數(shù)變成0時被調(diào)用,
 * 既然沒有進(jìn)程在使用在這些對象, 自然也不需要加鎖.
 * struct file 是持有引用計數(shù)的.
 */
/* --- epoll相關(guān)的數(shù)據(jù)結(jié)構(gòu) --- */
/*
 * This structure is stored inside the "private_data" member of the file
 * structure and rapresent the main data sructure for the eventpoll
 * interface.
 */
/* 每創(chuàng)建一個epollfd, 內(nèi)核就會分配一個eventpoll與之對應(yīng), 可以說是
 * 內(nèi)核態(tài)的epollfd. */
struct eventpoll {
	/* Protect the this structure access */
	spinlock_t lock;
	/*
	 * This mutex is used to ensure that files are not removed
	 * while epoll is using them. This is held during the event
	 * collection loop, the file cleanup path, the epoll file exit
	 * code and the ctl operations.
	 */
	/* 添加, 修改或者刪除監(jiān)聽fd的時候, 以及epoll_wait返回, 向用戶空間
	 * 傳遞數(shù)據(jù)時都會持有這個互斥鎖, 所以在用戶空間可以放心的在多個線程
	 * 中同時執(zhí)行epoll相關(guān)的操作, 內(nèi)核級已經(jīng)做了保護(hù). */
	struct mutex mtx;
	/* Wait queue used by sys_epoll_wait() */
	/* 調(diào)用epoll_wait()時, 我們就是"睡"在了這個等待隊列上... */
	wait_queue_head_t wq;
	/* Wait queue used by file->poll() */
	/* 這個用于epollfd本事被poll的時候... */
	wait_queue_head_t poll_wait;
	/* List of ready file descriptors */
	/* 所有已經(jīng)ready的epitem都在這個鏈表里面 */
	struct list_head rdllist;
	/* RB tree root used to store monitored fd structs */
	/* 所有要監(jiān)聽的epitem都在這里 */
	struct rb_root rbr;
	/*
		這是一個單鏈表鏈接著所有的struct epitem當(dāng)event轉(zhuǎn)移到用戶空間時
	 */
	 * This is a single linked list that chains all the "struct epitem" that
	 * happened while transfering ready events to userspace w/out
	 * holding ->lock.
	 */
	struct epitem *ovflist;
	/* The user that created the eventpoll descriptor */
	/* 這里保存了一些用戶變量, 比如fd監(jiān)聽數(shù)量的最大值等等 */
	struct user_struct *user;
};
/*
 * Each file descriptor added to the eventpoll interface will
 * have an entry of this type linked to the "rbr" RB tree.
 */
/* epitem 表示一個被監(jiān)聽的fd */
struct epitem {
	/* RB tree node used to link this structure to the eventpoll RB tree */
	/* rb_node, 當(dāng)使用epoll_ctl()將一批fds加入到某個epollfd時, 內(nèi)核會分配
	 * 一批的epitem與fds們對應(yīng), 而且它們以rb_tree的形式組織起來, tree的root
	 * 保存在epollfd, 也就是struct eventpoll中. 
	 * 在這里使用rb_tree的原因我認(rèn)為是提高查找,插入以及刪除的速度.
	 * rb_tree對以上3個操作都具有O(lgN)的時間復(fù)雜度 */
	struct rb_node rbn;
	/* List header used to link this structure to the eventpoll ready list */
	/* 鏈表節(jié)點, 所有已經(jīng)ready的epitem都會被鏈到eventpoll的rdllist中 */
	struct list_head rdllink;
	/*
	 * Works together "struct eventpoll"->ovflist in keeping the
	 * single linked chain of items.
	 */
	/* 這個在代碼中再解釋... */
	struct epitem *next;
	/* The file descriptor information this item refers to */
	/* epitem對應(yīng)的fd和struct file */
	struct epoll_filefd ffd;
	/* Number of active wait queue attached to poll operations */
	int nwait;
	/* List containing poll wait queues */
	struct list_head pwqlist;
	/* The "container" of this item */
	/* 當(dāng)前epitem屬于哪個eventpoll */
	struct eventpoll *ep;
	/* List header used to link this item to the "struct file" items list */
	struct list_head fllink;
	/* The structure that describe the interested events and the source fd */
	/* 當(dāng)前的epitem關(guān)系哪些events, 這個數(shù)據(jù)是調(diào)用epoll_ctl時從用戶態(tài)傳遞過來 */
	struct epoll_event event;
};
struct epoll_filefd {
	struct file *file;
	int fd;
};
/* poll所用到的鉤子Wait structure used by the poll hooks */
struct eppoll_entry {
	/* List header used to link this structure to the "struct epitem" */
	struct list_head llink;
	/* The "base" pointer is set to the container "struct epitem" */
	struct epitem *base;
	/*
	 * Wait queue item that will be linked to the target file wait
	 * queue head.
	 */
	wait_queue_t wait;
	/* The wait queue head that linked the "wait" wait queue item */
	wait_queue_head_t *whead;
};
/* Wrapper struct used by poll queueing */
struct ep_pqueue {
	poll_table pt;
	struct epitem *epi;
};
/* Used by the ep_send_events() function as callback private data */
struct ep_send_events_data {
	int maxevents;
	struct epoll_event __user *events;
};

/* --- 代碼注釋 --- */
/* 你沒看錯, 這就是epoll_create()的真身, 基本啥也不干直接調(diào)用epoll_create1了,
 * 另外你也可以發(fā)現(xiàn), size這個參數(shù)其實是沒有任何用處的... */
SYSCALL_DEFINE1(epoll_create, int, size)
{
        if (size <= 0)
                return -EINVAL;
        return sys_epoll_create1(0);
}
/* 這才是真正的epoll_create啊~~ */
SYSCALL_DEFINE1(epoll_create1, int, flags)
{
	int error;
	struct eventpoll *ep = NULL;//主描述符
	/* Check the EPOLL_* constant for consistency.  */
	/* 這句沒啥用處... */
	BUILD_BUG_ON(EPOLL_CLOEXEC != O_CLOEXEC);
	/* 對于epoll來講, 目前唯一有效的FLAG就是CLOEXEC */
	if (flags & ~EPOLL_CLOEXEC)
		return -EINVAL;
	/*
	 * Create the internal data structure ("struct eventpoll").
	 */
	/* 分配一個struct eventpoll, 分配和初始化細(xì)節(jié)我們隨后深聊~ */
	error = ep_alloc(&ep);
	if (error < 0)
		return error;
	/*
	 * Creates all the items needed to setup an eventpoll file. That is,
	 * a file structure and a free file descriptor.
	 */
	/* 這里是創(chuàng)建一個匿名fd, 說起來就話長了...長話短說:
	 * epollfd本身并不存在一個真正的文件與之對應(yīng), 所以內(nèi)核需要創(chuàng)建一個
	 * "虛擬"的文件, 并為之分配真正的struct file結(jié)構(gòu), 而且有真正的fd.
	 * 這里2個參數(shù)比較關(guān)鍵:
	 * eventpoll_fops, fops就是file operations, 就是當(dāng)你對這個文件(這里是虛擬的)進(jìn)行操作(比如讀)時,
	 * fops里面的函數(shù)指針指向真正的操作實現(xiàn), 類似C++里面虛函數(shù)和子類的概念.
	 * epoll只實現(xiàn)了poll和release(就是close)操作, 其它文件系統(tǒng)操作都有VFS全權(quán)處理了.
	 * ep, ep就是struct epollevent, 它會作為一個私有數(shù)據(jù)保存在struct file的private指針里面.
	 * 其實說白了, 就是為了能通過fd找到struct file, 通過struct file能找到eventpoll結(jié)構(gòu).
	 * 如果懂一點Linux下字符設(shè)備驅(qū)動開發(fā), 這里應(yīng)該是很好理解的,
	 * 推薦閱讀 <Linux device driver 3rd>
	 */
	error = anon_inode_getfd("[eventpoll]", &eventpoll_fops, ep,
				 O_RDWR | (flags & O_CLOEXEC));
	if (error < 0)
		ep_free(ep);
	return error;
}
/* 
* 創(chuàng)建好epollfd后, 接下來我們要往里面添加fd咯
* 來看epoll_ctl
* epfd 就是epollfd
* op ADD,MOD,DEL
* fd 需要監(jiān)聽的描述符
* event 我們關(guān)心的events
*/
SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd,
		struct epoll_event __user *, event)
{
	int error;
	struct file *file, *tfile;
	struct eventpoll *ep;
	struct epitem *epi;
	struct epoll_event epds;
	error = -EFAULT;
	/* 
	 * 錯誤處理以及從用戶空間將epoll_event結(jié)構(gòu)copy到內(nèi)核空間.
	 */
	if (ep_op_has_event(op) &&
	    copy_from_user(&epds, event, sizeof(struct epoll_event)))
		goto error_return;
	/* Get the "struct file *" for the eventpoll file */
	/* 取得struct file結(jié)構(gòu), epfd既然是真正的fd, 那么內(nèi)核空間
	 * 就會有與之對于的一個struct file結(jié)構(gòu)
	 * 這個結(jié)構(gòu)在epoll_create1()中, 由函數(shù)anon_inode_getfd()分配 */
	error = -EBADF;
	file = fget(epfd);
	if (!file)
		goto error_return;
	/* Get the "struct file *" for the target file */
	/* 我們需要監(jiān)聽的fd, 它當(dāng)然也有個struct file結(jié)構(gòu), 上下2個不要搞混了哦 */
	tfile = fget(fd);
	if (!tfile)
		goto error_fput;
	/* The target file descriptor must support poll */
	error = -EPERM;
	/* 如果監(jiān)聽的文件不支持poll, 那就沒轍了.
	 * 你知道什么情況下, 文件會不支持poll嗎?
	 */
	if (!tfile->f_op || !tfile->f_op->poll)
		goto error_tgt_fput;
	/*
	 * We have to check that the file structure underneath the file descriptor
	 * the user passed to us _is_ an eventpoll file. And also we do not permit
	 * adding an epoll file descriptor inside itself.
	 */
	error = -EINVAL;
	/* epoll不能自己監(jiān)聽自己... */
	if (file == tfile || !is_file_epoll(file))
		goto error_tgt_fput;
	/*
	 * At this point it is safe to assume that the "private_data" contains
	 * our own data structure.
	 */
	/* 取到我們的eventpoll結(jié)構(gòu), 來自與epoll_create1()中的分配 */
	ep = file->private_data;
	/* 接下來的操作有可能修改數(shù)據(jù)結(jié)構(gòu)內(nèi)容, 鎖之~ */
	mutex_lock(&ep->mtx);
	/*
	 * Try to lookup the file inside our RB tree, Since we grabbed "mtx"
	 * above, we can be sure to be able to use the item looked up by
	 * ep_find() till we release the mutex.
	 */
	/* 對于每一個監(jiān)聽的fd, 內(nèi)核都有分配一個epitem結(jié)構(gòu),
	 * 而且我們也知道, epoll是不允許重復(fù)添加fd的,
	 * 所以我們首先查找該fd是不是已經(jīng)存在了.
	 * ep_find()其實就是RBTREE查找, 跟C++STL的map差不多一回事, O(lgn)的時間復(fù)雜度.
	 */
	epi = ep_find(ep, tfile, fd);
	error = -EINVAL;
	switch (op) {
		/* 首先我們關(guān)心添加 */
	case EPOLL_CTL_ADD:
		if (!epi) {
			/* 之前的find沒有找到有效的epitem, 證明是第一次插入, 接受!
			 * 這里我們可以知道, POLLERR和POLLHUP事件內(nèi)核總是會關(guān)心的
			 * */
			epds.events |= POLLERR | POLLHUP;
			/* rbtree插入, 詳情見ep_insert()的分析
			 * 其實我覺得這里有insert的話, 之前的find應(yīng)該
			 * 是可以省掉的... */
			error = ep_insert(ep, &epds, tfile, fd);
		} else
			/* 找到了!? 重復(fù)添加! */
			error = -EEXIST;
		break;
		/* 刪除和修改操作都比較簡單 */
	case EPOLL_CTL_DEL:
		if (epi)
			error = ep_remove(ep, epi);
		else
			error = -ENOENT;
		break;
	case EPOLL_CTL_MOD:
		if (epi) {
			epds.events |= POLLERR | POLLHUP;
			error = ep_modify(ep, epi, &epds);
		} else
			error = -ENOENT;
		break;
	}
	mutex_unlock(&ep->mtx);
error_tgt_fput:
	fput(tfile);
error_fput:
	fput(file);
error_return:
	return error;
}
/* 分配一個eventpoll結(jié)構(gòu) */
static int ep_alloc(struct eventpoll **pep)
{
	int error;
	struct user_struct *user;
	struct eventpoll *ep;
	/* 獲取當(dāng)前用戶的一些信息, 比如是不是root啦, 最大監(jiān)聽fd數(shù)目啦 */
	user = get_current_user();
	error = -ENOMEM;
	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
	if (unlikely(!ep))
		goto free_uid;
	/* 這些都是初始化啦 */
	spin_lock_init(&ep->lock);
	mutex_init(&ep->mtx);
	init_waitqueue_head(&ep->wq);//初始化自己睡在的等待隊列
	init_waitqueue_head(&ep->poll_wait);//初始化
	INIT_LIST_HEAD(&ep->rdllist);//初始化就緒鏈表
	ep->rbr = RB_ROOT;
	ep->ovflist = EP_UNACTIVE_PTR;
	ep->user = user;
	*pep = ep;
	return 0;
free_uid:
	free_uid(user);
	return error;
}
/*
 * Must be called with "mtx" held.
 */
/* 
 * ep_insert()在epoll_ctl()中被調(diào)用, 完成往epollfd里面添加一個監(jiān)聽fd的工作
 * tfile是fd在內(nèi)核態(tài)的struct file結(jié)構(gòu)
 */
static int ep_insert(struct eventpoll *ep, struct epoll_event *event,
		     struct file *tfile, int fd)
{
	int error, revents, pwake = 0;
	unsigned long flags;
	struct epitem *epi;
	struct ep_pqueue epq;
	/* 查看是否達(dá)到當(dāng)前用戶的最大監(jiān)聽數(shù) */
	if (unlikely(atomic_read(&ep->user->epoll_watches) >=
		     max_user_watches))
		return -ENOSPC;
	/* 從著名的slab中分配一個epitem */
	if (!(epi = kmem_***_alloc(epi_***, GFP_KERNEL)))
		return -ENOMEM;
	/* Item initialization follow here ... */
	/* 這些都是相關(guān)成員的初始化... */
	INIT_LIST_HEAD(&epi->rdllink);
	INIT_LIST_HEAD(&epi->fllink);
	INIT_LIST_HEAD(&epi->pwqlist);
	epi->ep = ep;
	/* 這里保存了我們需要監(jiān)聽的文件fd和它的file結(jié)構(gòu) */
	ep_set_ffd(&epi->ffd, tfile, fd);
	epi->event = *event;
	epi->nwait = 0;
	/* 這個指針的初值不是NULL哦... */
	epi->next = EP_UNACTIVE_PTR;
	/* Initialize the poll table using the queue callback */
	/* 好, 我們終于要進(jìn)入到poll的正題了 */
	epq.epi = epi;
	/* 初始化一個poll_table
	 * 其實就是指定調(diào)用poll_wait(注意不是epoll_wait!!!)時的回調(diào)函數(shù),和我們關(guān)心哪些events,
	 * ep_ptable_queue_proc()就是我們的回調(diào)啦, 初值是所有event都關(guān)心 */
	init_poll_funcptr(&epq.pt, ep_ptable_queue_proc);
	/*
	 * Attach the item to the poll hooks and get current event bits.
	 * We can safely use the file* here because its usage count has
	 * been increased by the caller of this function. Note that after
	 * this operation completes, the poll callback can start hitting
	 * the new item.
	 */
	/* 這一部很關(guān)鍵, 也比較難懂, 完全是內(nèi)核的poll機(jī)制導(dǎo)致的...
	 * 首先, f_op->poll()一般來說只是個wrapper, 它會調(diào)用真正的poll實現(xiàn),
	 * 拿UDP的socket來舉例, 這里就是這樣的調(diào)用流程: f_op->poll(), sock_poll(),
	 * udp_poll(), datagram_poll(), sock_poll_wait(), 最后調(diào)用到我們上面指定的
	 * ep_ptable_queue_proc()這個回調(diào)函數(shù)...(好深的調(diào)用路徑...).
	 * 完成這一步, 我們的epitem就跟這個socket關(guān)聯(lián)起來了, 當(dāng)它有狀態(tài)變化時,
	 * 會通過ep_poll_callback()來通知.
	 * 最后, 這個函數(shù)還會查詢當(dāng)前的fd是不是已經(jīng)有啥event已經(jīng)ready了, 有的話
	 * 會將event返回. */
	revents = tfile->f_op->poll(tfile, &epq.pt);
	/*
	 * We have to check if something went wrong during the poll wait queue
	 * install process. Namely an allocation for a wait queue failed due
	 * high memory pressure.
	 */
	error = -ENOMEM;
	if (epi->nwait < 0)
		goto error_unregister;
	/* Add the current item to the list of active epoll hook for this file */
	/* 這個就是每個文件會將所有監(jiān)聽自己的epitem鏈起來 */
	spin_lock(&tfile->f_lock);
	list_add_tail(&epi->fllink, &tfile->f_ep_links);
	spin_unlock(&tfile->f_lock);
	/*
	 * Add the current item to the RB tree. All RB tree operations are
	 * protected by "mtx", and ep_insert() is called with "mtx" held.
	 */
	/* 都搞定后, 將epitem插入到對應(yīng)的eventpoll中去 */
	ep_rbtree_insert(ep, epi);
	/* We have to drop the new item inside our item list to keep track of it */
	spin_lock_irqsave(&ep->lock, flags);
	/* If the file is already "ready" we drop it inside the ready list */
	/* 到達(dá)這里后, 如果我們監(jiān)聽的fd已經(jīng)有事件發(fā)生, 那就要處理一下 */
	if ((revents & event->events) && !ep_is_linked(&epi->rdllink)) {
		/* 將當(dāng)前的epitem加入到ready list中去 */
		list_add_tail(&epi->rdllink, &ep->rdllist);
		/* Notify waiting tasks that events are available */
		/* 誰在epoll_wait, 就喚醒它... */
		if (waitqueue_active(&ep->wq))
			wake_up_locked(&ep->wq);
		/* 誰在epoll當(dāng)前的epollfd, 也喚醒它... */
		if (waitqueue_active(&ep->poll_wait))
			pwake++;
	}
	spin_unlock_irqrestore(&ep->lock, flags);
	atomic_inc(&ep->user->epoll_watches);
	/* We have to call this outside the lock */
	if (pwake)
		ep_poll_safewake(&ep->poll_wait);
	return 0;
error_unregister:
	ep_unregister_pollwait(ep, epi);
	/*
	 * We need to do this because an event could have been arrived on some
	 * allocated wait queue. Note that we don't care about the ep->ovflist
	 * list, since that is used/cleaned only inside a section bound by "mtx".
	 * And ep_insert() is called with "mtx" held.
	 */
	spin_lock_irqsave(&ep->lock, flags);
	if (ep_is_linked(&epi->rdllink))
		list_del_init(&epi->rdllink);
	spin_unlock_irqrestore(&ep->lock, flags);
	kmem_***_free(epi_***, epi);
	return error;
}
/*
 * This is the callback that is used to add our wait queue to the
 * target file wakeup lists.
 */
/* 
 * 該函數(shù)在調(diào)用f_op->poll()時會被調(diào)用.
 * 也就是epoll主動poll某個fd時, 用來將epitem與指定的fd關(guān)聯(lián)起來的.
 * 關(guān)聯(lián)的辦法就是使用等待隊列(waitqueue)
 */
static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
				 poll_table *pt)
{
	struct epitem *epi = ep_item_from_epqueue(pt);
	struct eppoll_entry *pwq;
	if (epi->nwait >= 0 && (pwq = kmem_***_alloc(pwq_***, GFP_KERNEL))) {
		/* 初始化等待隊列, 指定ep_poll_callback為喚醒時的回調(diào)函數(shù),
		 * 當(dāng)我們監(jiān)聽的fd發(fā)生狀態(tài)改變時, 也就是隊列頭被喚醒時,
		 * 指定的回調(diào)函數(shù)將會被調(diào)用. */
		init_waitqueue_func_entry(&pwq->wait, ep_poll_callback);
		pwq->whead = whead;
		pwq->base = epi;
		/* 將剛分配的等待隊列成員加入到頭中, 頭是由fd持有的 */
		add_wait_queue(whead, &pwq->wait);
		list_add_tail(&pwq->llink, &epi->pwqlist);
		/* nwait記錄了當(dāng)前epitem加入到了多少個等待隊列中,
		 * 我認(rèn)為這個值最大也只會是1... */
		epi->nwait++;
	} else {
		/* We have to signal that an error occurred */
		epi->nwait = -1;
	}
}
/*
 * This is the callback that is passed to the wait queue wakeup
 * machanism. It is called by the stored file descriptors when they
 * have events to report.
 */
/* 
 * 這個是關(guān)鍵性的回調(diào)函數(shù), 當(dāng)我們監(jiān)聽的fd發(fā)生狀態(tài)改變時, 它會被調(diào)用.
 * 參數(shù)key被當(dāng)作一個unsigned long整數(shù)使用, 攜帶的是events.
 */
static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync, void *key)
{
	int pwake = 0;
	unsigned long flags;
	struct epitem *epi = ep_item_from_wait(wait);//從等待隊列獲取epitem.需要知道哪個進(jìn)程掛載到這個設(shè)備
	struct eventpoll *ep = epi->ep;//獲取
	spin_lock_irqsave(&ep->lock, flags);
	/*
	 * If the event mask does not contain any poll(2) event, we consider the
	 * descriptor to be disabled. This condition is likely the effect of the
	 * EPOLLONESHOT bit that disables the descriptor when an event is received,
	 * until the next EPOLL_CTL_MOD will be issued.
	 */
	if (!(epi->event.events & ~EP_PRIVATE_BITS))
		goto out_unlock;
	/*
	 * Check the events coming with the callback. At this stage, not
	 * every device reports the events in the "key" parameter of the
	 * callback. We need to be able to handle both cases here, hence the
	 * test for "key" != NULL before the event match test.
	 */
	/* 沒有我們關(guān)心的event... */
	if (key && !((unsigned long) key & epi->event.events))
		goto out_unlock;
	/*
	 * If we are trasfering events to userspace, we can hold no locks
	 * (because we're accessing user memory, and because of linux f_op->poll()
	 * semantics). All the events that happens during that period of time are
	 * chained in ep->ovflist and requeued later on.
	 */
	/* 
	 * 這里看起來可能有點費解, 其實干的事情比較簡單:
	 * 如果該callback被調(diào)用的同時, epoll_wait()已經(jīng)返回了,
	 * 也就是說, 此刻應(yīng)用程序有可能已經(jīng)在循環(huán)獲取events,
	 * 這種情況下, 內(nèi)核將此刻發(fā)生event的epitem用一個單獨的鏈表
	 * 鏈起來, 不發(fā)給應(yīng)用程序, 也不丟棄, 而是在下一次epoll_wait
	 * 時返回給用戶.
	 */
	if (unlikely(ep->ovflist != EP_UNACTIVE_PTR)) {
		if (epi->next == EP_UNACTIVE_PTR) {
			epi->next = ep->ovflist;
			ep->ovflist = epi;
		}
		goto out_unlock;
	}
	/* If this file is already in the ready list we exit soon */
	/* 將當(dāng)前的epitem放入ready list */
	if (!ep_is_linked(&epi->rdllink))
		list_add_tail(&epi->rdllink, &ep->rdllist);
	/*
	 * Wake up ( if active ) both the eventpoll wait list and the ->poll()
	 * wait list.
	 */
	/* 喚醒epoll_wait... */
	if (waitqueue_active(&ep->wq))
		wake_up_locked(&ep->wq);
	/* 如果epollfd也在被poll, 那就喚醒隊列里面的所有成員. */
	if (waitqueue_active(&ep->poll_wait))
		pwake++;
out_unlock:
	spin_unlock_irqrestore(&ep->lock, flags);
	/* We have to call this outside the lock */
	if (pwake)
		ep_poll_safewake(&ep->poll_wait);
	return 1;
}
/*
 * Implement the event wait interface for the eventpoll file. It is the kernel
 * part of the user space epoll_wait(2).
 */
SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events,
		int, maxevents, int, timeout)
{
	int error;
	struct file *file;
	struct eventpoll *ep;
	/* The maximum number of event must be greater than zero */
	if (maxevents <= 0 || maxevents > EP_MAX_EVENTS)
		return -EINVAL;
	/* Verify that the area passed by the user is writeable */
	/* 這個地方有必要說明一下:
	 * 內(nèi)核對應(yīng)用程序采取的策略是"絕對不信任",
	 * 所以內(nèi)核跟應(yīng)用程序之間的數(shù)據(jù)交互大都是copy, 不允許(也時候也是不能...)指針引用.
	 * epoll_wait()需要內(nèi)核返回數(shù)據(jù)給用戶空間, 內(nèi)存由用戶程序提供,
	 * 所以內(nèi)核會用一些手段來驗證這一段內(nèi)存空間是不是有效的.
	 */
	if (!access_ok(VERIFY_WRITE, events, maxevents * sizeof(struct epoll_event))) {
		error = -EFAULT;
		goto error_return;
	}
	/* Get the "struct file *" for the eventpoll file */
	error = -EBADF;
	/* 獲取epollfd的struct file, epollfd也是文件嘛 */
	file = fget(epfd);
	if (!file)
		goto error_return;
	/*
	 * We have to check that the file structure underneath the fd
	 * the user passed to us _is_ an eventpoll file.
	 */
	error = -EINVAL;
	/* 檢查一下它是不是一個真正的epollfd... */
	if (!is_file_epoll(file))
		goto error_fput;
	/*
	 * At this point it is safe to assume that the "private_data" contains
	 * our own data structure.
	 */
	/* 獲取eventpoll結(jié)構(gòu) */
	ep = file->private_data;
	/* Time to fish for events ... */
	/* OK, 睡覺, 等待事件到來~~ */
	error = ep_poll(ep, events, maxevents, timeout);
error_fput:
	fput(file);
error_return:
	return error;
}
/* 這個函數(shù)真正將執(zhí)行epoll_wait的進(jìn)程帶入睡眠狀態(tài)... */
static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
		   int maxevents, long timeout)
{
	int res, eavail;
	unsigned long flags;
	long jtimeout;
	wait_queue_t wait;//等待隊列
	/*
	 * Calculate the timeout by checking for the "infinite" value (-1)
	 * and the overflow condition. The passed timeout is in milliseconds,
	 * that why (t * HZ) / 1000.
	 */
	/* 計算睡覺時間, 毫秒要轉(zhuǎn)換為HZ */
	jtimeout = (timeout < 0 || timeout >= EP_MAX_MSTIMEO) ?
		MAX_SCHEDULE_TIMEOUT : (timeout * HZ + 999) / 1000;
retry:
	spin_lock_irqsave(&ep->lock, flags);
	res = 0;
	/* 如果ready list不為空, 就不睡了, 直接干活... */
	if (list_empty(&ep->rdllist)) {
		/*
		 * We don't have any available event to return to the caller.
		 * We need to sleep here, and we will be wake up by
		 * ep_poll_callback() when events will become available.
		 */
		/* OK, 初始化一個等待隊列, 準(zhǔn)備直接把自己掛起,
		 * 注意current是一個宏, 代表當(dāng)前進(jìn)程 */
		init_waitqueue_entry(&wait, current);//初始化等待隊列,wait表示當(dāng)前進(jìn)程
		__add_wait_queue_exclusive(&ep->wq, &wait);//掛載到ep結(jié)構(gòu)的等待隊列
		for (;;) {
			/*
			 * We don't want to sleep if the ep_poll_callback() sends us
			 * a wakeup in between. That's why we set the task state
			 * to TASK_INTERRUPTIBLE before doing the checks.
			 */
			/* 將當(dāng)前進(jìn)程設(shè)置位睡眠, 但是可以被信號喚醒的狀態(tài),
			 * 注意這個設(shè)置是"將來時", 我們此刻還沒睡! */
			set_current_state(TASK_INTERRUPTIBLE);
			/* 如果這個時候, ready list里面有成員了,
			 * 或者睡眠時間已經(jīng)過了, 就直接不睡了... */
			if (!list_empty(&ep->rdllist) || !jtimeout)
				break;
			/* 如果有信號產(chǎn)生, 也起床... */
			if (signal_pending(current)) {
				res = -EINTR;
				break;
			}
			/* 啥事都沒有,解鎖, 睡覺... */
			spin_unlock_irqrestore(&ep->lock, flags);
			/* jtimeout這個時間后, 會被喚醒,
			 * ep_poll_callback()如果此時被調(diào)用,
			 * 那么我們就會直接被喚醒, 不用等時間了... 
			 * 再次強(qiáng)調(diào)一下ep_poll_callback()的調(diào)用時機(jī)是由被監(jiān)聽的fd
			 * 的具體實現(xiàn), 比如socket或者某個設(shè)備驅(qū)動來決定的,
			 * 因為等待隊列頭是他們持有的, epoll和當(dāng)前進(jìn)程
			 * 只是單純的等待...
			 **/
			jtimeout = schedule_timeout(jtimeout);//睡覺
			spin_lock_irqsave(&ep->lock, flags);
		}
		__remove_wait_queue(&ep->wq, &wait);
		/* OK 我們醒來了... */
		set_current_state(TASK_RUNNING);
	}
	/* Is it worth to try to dig for events ? */
	eavail = !list_empty(&ep->rdllist) || ep->ovflist != EP_UNACTIVE_PTR;
	spin_unlock_irqrestore(&ep->lock, flags);
	/*
	 * Try to transfer events to user space. In case we get 0 events and
	 * there's still timeout left over, we go trying again in search of
	 * more luck.
	 */
	/* 如果一切正常, 有event發(fā)生, 就開始準(zhǔn)備數(shù)據(jù)copy給用戶空間了... */
	if (!res && eavail &&
	    !(res = ep_send_events(ep, events, maxevents)) && jtimeout)
		goto retry;
	return res;
}
/* 這個簡單, 我們直奔下一個... */
static int ep_send_events(struct eventpoll *ep,
			  struct epoll_event __user *events, int maxevents)
{
	struct ep_send_events_data esed;
	esed.maxevents = maxevents;
	esed.events = events;
	return ep_scan_ready_list(ep, ep_send_events_proc, &esed);
}
/**
 * ep_scan_ready_list - Scans the ready list in a way that makes possible for
 *                      the scan code, to call f_op->poll(). Also allows for
 *                      O(NumReady) performance.
 *
 * @ep: Pointer to the epoll private data structure.
 * @sproc: Pointer to the scan callback.
 * @priv: Private opaque data passed to the @sproc callback.
 *
 * Returns: The same integer error code returned by the @sproc callback.
 */
static int ep_scan_ready_list(struct eventpoll *ep,
			      int (*sproc)(struct eventpoll *,
					   struct list_head *, void *),
			      void *priv)
{
	int error, pwake = 0;
	unsigned long flags;
	struct epitem *epi, *nepi;
	LIST_HEAD(txlist);
	/*
	 * We need to lock this because we could be hit by
	 * eventpoll_release_file() and epoll_ctl().
	 */
	mutex_lock(&ep->mtx);
	/*
	 * Steal the ready list, and re-init the original one to the
	 * empty list. Also, set ep->ovflist to NULL so that events
	 * happening while looping w/out locks, are not lost. We cannot
	 * have the poll callback to queue directly on ep->rdllist,
	 * because we want the "sproc" callback to be able to do it
	 * in a lockless way.
	 */
	spin_lock_irqsave(&ep->lock, flags);
	/* 這一步要注意, 首先, 所有監(jiān)聽到events的epitem都鏈到rdllist上了,
	 * 但是這一步之后, 所有的epitem都轉(zhuǎn)移到了txlist上, 而rdllist被清空了,
	 * 要注意哦, rdllist已經(jīng)被清空了! */
	list_splice_init(&ep->rdllist, &txlist);
	/* ovflist, 在ep_poll_callback()里面我解釋過, 此時此刻我們不希望
	 * 有新的event加入到ready list中了, 保存后下次再處理... */
	ep->ovflist = NULL;
	spin_unlock_irqrestore(&ep->lock, flags);
	/*
	 * Now call the callback function.
	 */
	/* 在這個回調(diào)函數(shù)里面處理每個epitem
	 * sproc 就是 ep_send_events_proc, 下面會注釋到. */
	error = (*sproc)(ep, &txlist, priv);
	spin_lock_irqsave(&ep->lock, flags);
	/*
	 * During the time we spent inside the "sproc" callback, some
	 * other events might have been queued by the poll callback.
	 * We re-insert them inside the main ready-list here.
	 */
	/* 現(xiàn)在我們來處理ovflist, 這些epitem都是我們在傳遞數(shù)據(jù)給用戶空間時
	 * 監(jiān)聽到了事件. */
	for (nepi = ep->ovflist; (epi = nepi) != NULL;
	     nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {
		/*
		 * We need to check if the item is already in the list.
		 * During the "sproc" callback execution time, items are
		 * queued into ->ovflist but the "txlist" might already
		 * contain them, and the list_splice() below takes care of them.
		 */
		/* 將這些直接放入readylist */
		if (!ep_is_linked(&epi->rdllink))
			list_add_tail(&epi->rdllink, &ep->rdllist);
	}
	/*
	 * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after
	 * releasing the lock, events will be queued in the normal way inside
	 * ep->rdllist.
	 */
	ep->ovflist = EP_UNACTIVE_PTR;
	/*
	 * Quickly re-inject items left on "txlist".
	 */
	/* 上一次沒有處理完的epitem, 重新插入到ready list */
	list_splice(&txlist, &ep->rdllist);
	/* ready list不為空, 直接喚醒... */
	if (!list_empty(&ep->rdllist)) {
		/*
		 * Wake up (if active) both the eventpoll wait list and
		 * the ->poll() wait list (delayed after we release the lock).
		 */
		if (waitqueue_active(&ep->wq))
			wake_up_locked(&ep->wq);
		if (waitqueue_active(&ep->poll_wait))
			pwake++;
	}
	spin_unlock_irqrestore(&ep->lock, flags);
	mutex_unlock(&ep->mtx);
	/* We have to call this outside the lock */
	if (pwake)
		ep_poll_safewake(&ep->poll_wait);
	return error;
}
/* 該函數(shù)作為callbakc在ep_scan_ready_list()中被調(diào)用
 * head是一個鏈表, 包含了已經(jīng)ready的epitem,
 * 這個不是eventpoll里面的ready list, 而是上面函數(shù)中的txlist.
 */
static int ep_send_events_proc(struct eventpoll *ep, struct list_head *head,
			       void *priv)
{
	struct ep_send_events_data *esed = priv;
	int eventcnt;
	unsigned int revents;
	struct epitem *epi;
	struct epoll_event __user *uevent;
	/*
	 * We can loop without lock because we are passed a task private list.
	 * Items cannot vanish during the loop because ep_scan_ready_list() is
	 * holding "mtx" during this call.
	 */
	/* 掃描整個鏈表... */
	for (eventcnt = 0, uevent = esed->events;
	     !list_empty(head) && eventcnt < esed->maxevents;) {
		/* 取出第一個成員 */
		epi = list_first_entry(head, struct epitem, rdllink);
		/* 然后從鏈表里面移除 */
		list_del_init(&epi->rdllink);
		/* 讀取events, 
		 * 注意events我們ep_poll_callback()里面已經(jīng)取過一次了, 為啥還要再取?
		 * 1. 我們當(dāng)然希望能拿到此刻的最新數(shù)據(jù), events是會變的~
		 * 2. 不是所有的poll實現(xiàn), 都通過等待隊列傳遞了events, 有可能某些驅(qū)動壓根沒傳
		 * 必須主動去讀取. */
		revents = epi->ffd.file->f_op->poll(epi->ffd.file, NULL) &
			epi->event.events;
		if (revents) {
			/* 將當(dāng)前的事件和用戶傳入的數(shù)據(jù)都copy給用戶空間,
			 * 就是epoll_wait()后應(yīng)用程序能讀到的那一堆數(shù)據(jù). */
			if (__put_user(revents, &uevent->events) ||
			    __put_user(epi->event.data, &uevent->data)) {
				list_add(&epi->rdllink, head);
				return eventcnt ? eventcnt : -EFAULT;
			}
			eventcnt++;
			uevent++;
			if (epi->event.events & EPOLLONESHOT)
				epi->event.events &= EP_PRIVATE_BITS;
			else if (!(epi->event.events & EPOLLET)) {
				/* 嘿嘿, EPOLLET和非ET的區(qū)別就在這一步之差呀~
				 * 如果是ET, epitem是不會再進(jìn)入到readly list,
				 * 除非fd再次發(fā)生了狀態(tài)改變, ep_poll_callback被調(diào)用.
				 * 如果是非ET, 不管你還有沒有有效的事件或者數(shù)據(jù),
				 * 都會被重新插入到ready list, 再下一次epoll_wait
				 * 時, 會立即返回, 并通知給用戶空間. 當(dāng)然如果這個
				 * 被監(jiān)聽的fds確實沒事件也沒數(shù)據(jù)了, epoll_wait會返回一個0,
				 * 空轉(zhuǎn)一次.
				 */
				list_add_tail(&epi->rdllink, &ep->rdllist);
			}
		}
	}
	return eventcnt;
}
/* ep_free在epollfd被close時調(diào)用,
 * 釋放一些資源而已, 比較簡單 */
static void ep_free(struct eventpoll *ep)
{
	struct rb_node *rbp;
	struct epitem *epi;
	/* We need to release all tasks waiting for these file */
	if (waitqueue_active(&ep->poll_wait))
		ep_poll_safewake(&ep->poll_wait);
	/*
	 * We need to lock this because we could be hit by
	 * eventpoll_release_file() while we're freeing the "struct eventpoll".
	 * We do not need to hold "ep->mtx" here because the epoll file
	 * is on the way to be removed and no one has references to it
	 * anymore. The only hit might come from eventpoll_release_file() but
	 * holding "epmutex" is sufficent here.
	 */
	mutex_lock(&epmutex);
	/*
	 * Walks through the whole tree by unregistering poll callbacks.
	 */
	for (rbp = rb_first(&ep->rbr); rbp; rbp = rb_next(rbp)) {
		epi = rb_entry(rbp, struct epitem, rbn);
		ep_unregister_pollwait(ep, epi);
	}
	/*
	 * Walks through the whole tree by freeing each "struct epitem". At this
	 * point we are sure no poll callbacks will be lingering around, and also by
	 * holding "epmutex" we can be sure that no file cleanup code will hit
	 * us during this operation. So we can avoid the lock on "ep->lock".
	 */
	/* 之所以在關(guān)閉epollfd之前不需要調(diào)用epoll_ctl移除已經(jīng)添加的fd,
	 * 是因為這里已經(jīng)做了... */
	while ((rbp = rb_first(&ep->rbr)) != NULL) {
		epi = rb_entry(rbp, struct epitem, rbn);
		ep_remove(ep, epi);
	}
	mutex_unlock(&epmutex);
	mutex_destroy(&ep->mtx);
	free_uid(ep->user);
	kfree(ep);
}
/* File callbacks that implement the eventpoll file behaviour */
static const struct file_operations eventpoll_fops = {
	.release	= ep_eventpoll_release,
	.poll		= ep_eventpoll_poll
};
/* Fast test to see if the file is an evenpoll file */
static inline int is_file_epoll(struct file *f)
{
	return f->f_op == &eventpoll_fops;
}
/* OK, eventpoll我認(rèn)為比較重要的函數(shù)都注釋完了... */

int epoll_create(int size); //監(jiān)聽個數(shù)

int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)

EPOLL_CTL_ADD：注冊新的fd到epfd中；
EPOLL_CTL_MOD：修改已經(jīng)注冊的fd的監(jiān)聽事件；
EPOLL_CTL_DEL：從epfd中刪除一個fd；

struct epoll_event {  
    __uint32_t events; /* Epoll events */
    epoll_data_t data; /* User data variable */
};

int epoll_wait(int epfd, struct epoll_event * events, int maxevents, int timeout)

#define LISTEN_MAX     5
#define EPOLL_FDSIZE   LISTEN_MAX
#define EPOLL_EVENTS   20
#define CLIENT_NUM     3

void EpollAddEvent(int epollfd, int fd, int event)
{
    PRINT("epollfd:%d add fd:%d(event:%d)\n", epollfd, fd, event);
    struct epoll_event ev;
    ev.events = event;
    ev.data.fd = fd;
    epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &ev);
}

void TcpServerThread()
{
	//------------socket
	int sockfd = socket(AF_UNIX, SOCK_STREAM, 0);
	if (sockfd < 0)
	{
		PRINT("create socket fail\n");
		return;
	}
	PRINT("create socketfd:%d\n", sockfd);

	struct sockaddr_un addr;
	memset (&addr, 0, sizeof(addr));
	addr.sun_family = AF_UNIX;
	strcpy(addr.sun_path, UNIX_TCP_SOCKET_ADDR);

	//------------bind
	if (bind(sockfd, (struct sockaddr *)&addr, sizeof(addr)))
	{
		PRINT("bind fail\n");
		return;
	}
	PRINT("bind ok\n");

	//------------listen
	if (listen(sockfd, LISTEN_MAX))
	{
		PRINT("listen fail\n");
		return;
	}
	PRINT("listen ok\n");

	//------------epoll---------------
	int epollfd = epoll_create(EPOLL_FDSIZE);
	if (epollfd < 0)
	{
		PRINT("epoll create fail\n");
		return;
	}
	PRINT("epoll create fd:%d\n", epollfd);

	EpollAddEvent(epollfd, sockfd, EPOLLIN);

	struct epoll_event events[EPOLL_EVENTS];
	while(1)
	{
		PRINT("epoll wait...\n");
		int num = epoll_wait(epollfd, events, EPOLL_EVENTS, -1);
		PRINT("epoll wait done, num:%d\n", num);
		for (int i = 0;i < num;i++)
		{
			int fd = events[i].data.fd;
			if (EPOLLIN == events[i].events)
			{
				//接受客戶端的連接請求
				if (fd == sockfd)
				{
					//------------accept
					int clientfd = accept(sockfd, NULL, NULL);
					if (clientfd == -1)
					{
						PRINT("accpet error\n");
					}
					else
					{
						PRINT("=====> accept new clientfd:%d\n", clientfd);
						
						EpollAddEvent(epollfd, clientfd, EPOLLIN);
					}
				}
				//讀取客戶端發(fā)來的數(shù)據(jù)
				else
				{
					char buf[BUF_SIZE] = {0};
					//------------recv
					size_t size = recv(fd, buf, BUF_SIZE, 0);
					//size = read(clientfd, buf, BUF_SIZE);
					if (size > 0)
					{
						PRINT("recv from clientfd:%d, msg:%s\n", fd, buf);
					}
				}
			}
		}
	}

	PRINT("end\n");
}

int main()
{
	unlink(UNIX_TCP_SOCKET_ADDR);

	//創(chuàng)建一個服務(wù)端
	thread thServer(TcpServerThread);

	//創(chuàng)建多個客戶端
	thread thClinet[CLIENT_NUM];
	for (int i=0; i<CLIENT_NUM; i++)
	{
		thClinet[i] = thread(TcpClientThread);
		sleep(1);
	}

	while(1)
	{
		sleep(5);
	}
}

[TcpServerThread] create socketfd:3
[TcpServerThread] bind ok
[TcpClientThread] create socketfd:4
[TcpServerThread] listen ok
[TcpServerThread] epoll create fd:5
[EpollAddEvent] epollfd:5 add fd:3(event:1)
[TcpServerThread] epoll wait...
[TcpClientThread] create socketfd:6
[TcpClientThread] connect ok
[TcpServerThread] epoll wait done, num:1
[TcpServerThread] =====> accept new clientfd:7
[EpollAddEvent] epollfd:5 add fd:7(event:1)
[TcpServerThread] epoll wait...
[TcpServerThread] epoll wait done, num:1
[TcpServerThread] recv from clientfd:7, msg:helloTCP(fd:4)1
[TcpServerThread] epoll wait...
[TcpClientThread] create socketfd:8
[TcpClientThread] connect ok
[TcpServerThread] epoll wait done, num:2
[TcpServerThread] recv from clientfd:7, msg:helloTCP(fd:4)2
[TcpServerThread] =====> accept new clientfd:9
[EpollAddEvent] epollfd:5 add fd:9(event:1)
[TcpServerThread] epoll wait...
[TcpServerThread] epoll wait done, num:1
[TcpServerThread] recv from clientfd:9, msg:helloTCP(fd:6)3
[TcpServerThread] epoll wait...
[TcpClientThread] connect ok
[TcpServerThread] epoll wait done, num:3
[TcpServerThread] =====> accept new clientfd:10
[EpollAddEvent] epollfd:5 add fd:10(event:1)
[TcpServerThread] recv from clientfd:7, msg:helloTCP(fd:4)5
[TcpServerThread] recv from clientfd:9, msg:helloTCP(fd:6)6
[TcpServerThread] epoll wait...
[TcpServerThread] epoll wait done, num:1
[TcpServerThread] recv from clientfd:10, msg:helloTCP(fd:8)4
[TcpServerThread] epoll wait...
[TcpServerThread] epoll wait done, num:3
[TcpServerThread] recv from clientfd:10, msg:helloTCP(fd:8)7
[TcpServerThread] recv from clientfd:7, msg:helloTCP(fd:4)8
[TcpServerThread] recv from clientfd:9, msg:helloTCP(fd:6)9
[TcpServerThread] epoll wait...
[TcpServerThread] epoll wait done, num:1
[TcpServerThread] recv from clientfd:7, msg:helloTCP(fd:4)10
[TcpServerThread] epoll wait...
[TcpServerThread] epoll wait done, num:2
[TcpServerThread] recv from clientfd:9, msg:helloTCP(fd:6)12
[TcpServerThread] recv from clientfd:10, msg:helloTCP(fd:8)11
[TcpServerThread] epoll wait...
[TcpServerThread] epoll wait done, num:3
[TcpServerThread] recv from clientfd:10, msg:helloTCP(fd:8)14
[TcpServerThread] recv from clientfd:7, msg:helloTCP(fd:4)13
[TcpServerThread] recv from clientfd:9, msg:helloTCP(fd:6)15
[TcpServerThread] epoll wait...
[TcpServerThread] epoll wait done, num:3
[TcpServerThread] recv from clientfd:10, msg:helloTCP(fd:8)16
[TcpServerThread] recv from clientfd:7, msg:helloTCP(fd:4)17
[TcpServerThread] recv from clientfd:9, msg:helloTCP(fd:6)18
[TcpServerThread] epoll wait...