学步园

因需要改写将服务器读取文件同步IO为异步IO,因此在自己机子上编写了这个根据文件列表随机不间断读取文件块得操作

环境：CentOS4.8 512M内存,2个1.8 cpu

编译器:g++4.5

库：librt

文件列表格式：文件绝对路径+文件名 空格 文件总长度

提示：最好选取文件大点

/root/work/data/[www.dy2018.com]jixieshi.rmvb 371324360

/root/work/data/oracle9i_system01.dbf 398467072

编译： g++4.5 chlaws_aio.c -lrt

程序如下：

程序还可以大幅度优化,这里只是简单实现下,有问题可以留言,转载请注明文章出处.

#include <iostream><br /> #include <map><br /> #include <iterator><br /> #include <list><br /> #include <string><br /> #include <ctime><br /> #include <cstdlib><br /> #include <fstream><br /> #include <errno.h><br /> #include <aio.h><br /> #include <unistd.h><br /> #include <sys/types.h><br /> #include <cstring><br /> #include <strings.h><br /> #include <stdlib.h><br /> using namespace std;<br /> struct AIO_INFO<br /> {<br /> int reqblock;<br /> string filesize;<br /> string filename;<br /> struct aiocb *my_aiocb;<br /> };<br /> struct aiocb aiocb_list[100];<br /> int block = 1024*256;<br /> int main()<br /> {<br /> map<string,int>file_list;<br /> ifstream infile("/root/work/etc/file_list.txt",ios::in);<br /> if( infile )<br /> {<br /> string line;<br /> while(getline(infile,line))<br /> {<br /> if(line.size()>2 && line.at(line.size()-2)=='/r')<br /> {<br /> line = line.substr(0,line.size()-2);<br /> }<br /> if(line.size()>1 && line.at(line.size()-1)=='/n')<br /> {<br /> line = line.substr(0,line.size()-1);<br /> }<br /> if(line.size() > 0)<br /> {<br /> size_t pos = line.find(' ');<br /> if(pos != string::npos)<br /> {<br /> string file = line.substr(0,pos);<br /> string size = line.substr(pos+1,line.size());<br /> file_list.insert( make_pair(file, atoi(size.c_str()) ) );<br /> }<br /> }<br /> }<br /> }<br /> if(file_list.size() <= 0)<br /> {<br /> cout << "no file" << endl;<br /> return 1;<br /> }<br /> else<br /> {<br /> cout << file_list.begin()->first << "/t";<br /> cout << file_list.begin()->second << "/n";<br /> }</p> <p> //FILE *fp=fopen(file,"rb+");<br /> int ix=0;<br /> int nread=0,nret=0;<br /> //char rbuf[100][1024*256];<br /> //fd_map:filename-fd<br /> map<string,int> fd_map;<br /> fd_map.clear();<br /> struct AIO_INFO aio_info;<br /> list<struct AIO_INFO> aio_list;<br /> list<int>err_fd;<br /> srand(time(NULL));</p> <p> int last_time = 0;<br /> int now;<br /> int count = 0;<br /> while(1)<br /> {</p> <p> now = time(NULL);<br /> if(now - last_time > 120)<br /> {<br /> cout << "aio_list size:" << aio_list.size() << endl;<br /> count ++;<br /> if (count >= 5)<br /> break;<br /> last_time = now;<br /> }<br /> fd_map.clear();<br /> aio_list.clear();<br /> ix = 0;<br /> aio_list.clear();<br /> list<int>::iterator tmp_fd = err_fd.begin();<br /> cout << "err_fd size:" << err_fd.size() << endl;<br /> while( tmp_fd != err_fd.end())<br /> {<br /> close(*tmp_fd);<br /> err_fd.erase(tmp_fd++);<br /> }<br /> for( ; ix<100; ix++){<br /> map<string,int>::iterator file_iter = file_list.begin();<br /> int req = rand()%file_list.size();</p> <p> //add fd to fd_map<br /> int fd=-1;<br /> //int need_insert_fd_map=0;<br /> while(req--)file_iter++;<br /> string reqfile;<br /> if(file_iter != file_list.end())<br /> reqfile=file_iter->first;<br /> else<br /> continue;</p> <p> if(reqfile.size() == 0) break;<br /> map<string,int>::iterator fd_iter = fd_map.find(reqfile);<br /> if(fd_iter != fd_map.end())<br /> {<br /> fd = fd_iter->second;<br /> }<br /> else<br /> {<br /> fd = open(reqfile.c_str(),O_RDONLY);<br /> if(fd == -1) {cout << "open error:" << strerror(errno) << endl; exit(-1);}<br /> else fd_map.insert(make_pair(reqfile,fd));<br /> //need_insert_fd_map = 1;<br /> }</p> <p> int filesize = file_iter->second;<br /> int blk = rand()%(filesize/block);</p> <p> //cout << "req file:" << reqfile << "/tblock:" << blk << endl;<br /> //struct aiocb *my_aiocb =(struct aiocb*)malloc(sizeof(struct aiocb));<br /> //bzero(&aio_info.my_aiocb,sizeof(struct aiocb));<br /> //aio_info.my_aiocb = (struct aiocb*)malloc(sizeof(struct aiocb));<br /> bzero(&aiocb_list[ix],sizeof(struct aiocb));<br /> aio_info.my_aiocb = &aiocb_list[ix];<br /> aio_info.my_aiocb->aio_fildes = fd;<br /> aio_info.my_aiocb->aio_buf = (char*)malloc(block+1);<br /> aio_info.my_aiocb->aio_nbytes = block;<br /> aio_info.my_aiocb->aio_offset = blk*block;</p> <p> //memcpy((char*)&(aio_info.my_aiocb),(char*)&my_aiocb,sizeof (my_aiocb));<br /> //aio_info.my_aiocb = &my_aiocb;<br /> aio_info.reqblock=blk;<br /> aio_info.filename=reqfile;<br /> aio_info.filesize=filesize;<br /> //aio_list.insert(aio_list.begin(),aio_info);</p> <p> //printf("aio_read , aiocb address:%x/n",&aio_info.my_aiocb);<br /> int ret = aio_read(&aiocb_list[ix]);<br /> if(ret == -1)<br /> {<br /> cout << "aio_read error " << "reqfile:" << reqfile << " block:" << blk << endl;<br /> //free((void*)aio_info.my_aiocb);<br /> //bzero(&aio_info,sizeof(struct AIO_INFO));<br /> /*<br /> map<string,int>::iterator del_fd = fd_map.find(reqfile);<br /> if(del_fd != fd_map.end())<br /> {<br /> close(del_fd->second);<br /> fd_map.erase(del_fd);<br /> }<br /> */<br /> err_fd.insert(err_fd.begin(),fd);<br /> continue;<br /> }</p> <p> cout << "req file:" << reqfile << "/tblock:" << blk << endl;<br /> aio_list.insert(aio_list.begin(),aio_info);<br /> //if(need_insert_fd_map == 1) fd_map.insert(make_pair(reqfile,fd));<br /> /*<br /> while(aio_error(&my_aiocb) == EINPROGRESS);</p> <p> ret = aio_return(&my_aiocb);<br /> if(ret > 0)<br /> cout << "aio_return ok, ret=" << ret <<"/n";<br /> else<br /> cout << "aio_return error/n";<br /> */</p> <p> }<br /> if(aio_list.size() == 0)<br /> {<br /> continue;<br /> }<br /> int ret = -1;<br /> int wait_last = 0;<br /> int wait_now = 0;<br /> count = 0;<br /> while(1)<br /> {<br /> wait_now = time(NULL);<br /> if(wait_now - wait_last > 3)<br /> {<br /> cout << "[in while] wait 30s,aio_list size:" << aio_list.size()<br /> << "fd_map size:" << fd_map.size() << endl;<br /> if( count ++ > 3)<br /> {<br /> cout << "timeout exit while loop /n";<br /> list<struct AIO_INFO>::iterator del_it = aio_list.begin();<br /> while(del_it!=aio_list.end())<br /> {<br /> free((void*)del_it->my_aiocb->aio_buf);<br /> //free((void*)del_it->my_aiocb);<br /> aio_list.erase(del_it++);<br /> }<br /> break;<br /> }<br /> if(count == 1) usleep(1000);<br /> wait_last = wait_now;<br /> }<br /> if(aio_list.size() == 0)<br /> {<br /> cout << "[in while] the aio_list size : 0,exit while loop/n";<br /> break;<br /> }</p> <p> list<struct AIO_INFO>::iterator aio_iter = aio_list.begin();<br /> for( ;aio_iter!=aio_list.end();)<br /> {<br /> ret = -1;<br /> // cout << "in aio_list,fd:" << aio_iter->my_aiocb.aio_fildes<br /> // << " filename:" << aio_iter->filename << " block:" << aio_iter->reqblock << endl;<br /> //printf("aio_read , aiocb address:%x/n",&aio_iter->my_aiocb);<br /> if( (ret=aio_error((aio_iter->my_aiocb))) == -1)<br /> {<br /> free((void*)aio_iter->my_aiocb->aio_buf);<br /> //free((void*)aio_iter->my_aiocb);<br /> aio_list.erase(aio_iter++);<br /> cout << "aio_error" << endl;<br /> continue;<br /> }<br /> else if( ret == EINPROGRESS )<br /> {<br /> aio_iter++;<br /> //cout << "async io this req not ready" << endl;<br /> continue;<br /> }<br /> else if(ret == 0)<br /> {<br /> //cout << "request :"<<aio_iter->filename << "/tblock:" << aio_iter->reqblock << "/taio_error ok/n";<br /> ret = aio_return((aio_iter->my_aiocb));<br /> if(ret == block)<br /> {<br /> cout << "aio_return ok, this req:" << aio_iter->filename<br /> << "/treq block:" << aio_iter->reqblock << endl;<br /> free((void*)aio_iter->my_aiocb->aio_buf);<br /> //free((void*)aio_iter->my_aiocb);<br /> aio_list.erase(aio_iter++);<br /> }<br /> else<br /> {<br /> cout << "aio_return error,ret :" << ret << endl;<br /> free((void*)aio_iter->my_aiocb->aio_buf);<br /> //free((void*)aio_iter->my_aiocb);<br /> aio_list.erase(aio_iter++);<br /> }<br /> }<br /> }<br /> }//end while wait aio return<br /> //free buf<br /> list<struct AIO_INFO>::iterator aio_iter = aio_list.begin();<br /> for( ; aio_iter!=aio_list.end() ; )<br /> {<br /> free((void*)aio_iter->my_aiocb->aio_buf);<br /> //free((void*)aio_iter->my_aiocb);<br /> aio_list.erase(aio_iter++);<br /> }<br /> if(aio_list.size() != 0)<br /> {<br /> cout << "error:aio_list size != 0" << endl;<br /> }<br /> //close fd<br /> map<string,int>::iterator fd_iter = fd_map.begin();<br /> for( ; fd_iter!=fd_map.end() ; )<br /> {<br /> cout << "close fd:" << fd_iter->second << endl;<br /> close(fd_iter->second);<br /> fd_map.erase(fd_iter++);<br /> }<br /> usleep(2*1000);<br /> }</p> <p> return 0;<br /> }<br />