如下面一段文本：
Alice Emma has long flowing red hair. Her Daddy says when the wind blows through her hair, it looks almost alive, like a fiery bird in flight. A beautiful fiery bird, he tells her, magical but untamed. "Daddy, shush, there is no such thing, "she tells him, at the same time wanting him to tell her more. Shyly, she asks, "I mean, Daddy, is there?"
实现将其拆分为一个一个的单词，保存其在文中的行号及在行中的位置，并且剔除文中的标点：
vector<string> *retrieve_text()
{
string file_name;
cout << "please enter file name: ";
cin >> file_name;
// 打开文本文件以便输入
ifstream infile( file_name.c_str(), ios::in );
if ( ! infile )
{
cerr << "oops! unable to open file " << file_name << " -- bailing out!\n";
exit( -1 );
}
else
cout << '\n';
vector<string> *lines_of_text= new vector<string>;
string textline;
typedef pair<string::size_type, int> stats;
stats maxline;
int linenum = 0;
while ( getline( infile, textline, '\n' ))
{
cout << "line read: " << textline << '\n'<<endl;
if ( maxline.first < textline.size() )
{
maxline.first = textline.size();
maxline.second = linenum;
}
lines_of_text->push_back( textline );
linenum++;
}
cout<<"number of lines :"<<linenum<<endl;
cout<<"maxinum length :"<<maxline.first<<endl;
cout<<"longest line :"<<(*lines_of_text)[maxline.second]<<endl;
return lines_of_text;
}
函数retrieve_text()实现文本的按行读入，记录其行数及最大行中的单词数量；
*******************************************************************************************************
void filter_text( vector<string> *words, string filter )
{
vector<string>::iterator iter = words ->begin();
vector<string>::iterator iter_end = words ->end();
// 如果用户没有提供 filter, 则缺省使用最小集
if ( ! filter.size() )
filter.insert( 0, "\".," );
while ( iter != iter_end )
{
string::size_type pos = 0;
// 对于找到的每个元素, 将其删除
while (( pos = (*iter).find_first_of( filter, pos ))!= string::npos )
(*iter).erase(pos,1);
iter++;
}
}
函数filter_text()实现标点符号的剔除功能；
*****************************************************************************************************
typedef pair<short,short> location;
typedef vector<location> loc;
typedef vector<string> text;
typedef pair<text*,loc*> text_loc;
text_loc* separate_words( const vector<string> *text_file )
{
// words: 包含独立单词的集合
// locations: 包含相关的行/列信息
vector<string> *words = new vector<string>;
vector<location> *locations = new vector<location>;
short line_pos = 0; // current line number
// iterate through each line of text
for ( ; line_pos < text_file->size(); ++line_pos )
{
// textline: 待处理的当前文本行
// word_pos: 文本行中的当前列位置
short word_pos = 0;
string textline = (*text_file)[ line_pos ];
string::size_type pos = 0, prev_pos = 0;
//cout<<"textline:"<<textline<<endl;
while (( pos = textline.find_first_of( ' ', pos ))!= string::npos )
{
// 存储当前单词子串的拷贝
string word=textline.substr( prev_pos, pos - prev_pos);
words->push_back(word);//textline.substr( prev_pos, pos - prev_pos ));
// 将行/列信息存储为 pair
locations ->push_back(make_pair( line_pos, word_pos ));
// 为下一次迭代修改位置信息
++word_pos;
prev_pos = ++pos;
}
// 现在处理最后一个单词
words->push_back(textline.substr( prev_pos, pos - prev_pos ));
locations->push_back(make_pair( line_pos, word_pos ));
}
//处理标点符号
filter_text(words,"\",.;:!?)(\\/" );
//for(int i=0;i<words->size();i++)
// cout<<(*words)[i]<<" ";
for(int i=0;i<locations->size();i++)
{
cout<<"line: "<<(*locations)[i].first<<" word positon:"<<(*locations)[i].second<<" word:"<<(*words)[i]<<endl;
}
return new text_loc( words, locations );
}
本函数实现文本的单词分割，返回值记录了文件单词及其位置信息；