我正在开发一个C / C ++应用程序(在Visual Studio 2010中),我需要对逗号分隔的字符串进行标记,我希望它尽可能快。目前我正在使用strtok_s
。我对strtok_s
和sscanf
进行了一些测试,看起来像strtok_s
更快(除非我写了一个可怕的实现:))但我想知道是否有人可以提出更快的替代方案。
您可以做的最好的事情是确保只通过字符串一次,并动态构建输出。开始将字符拉入临时缓冲区,当遇到分隔符时将临时缓冲区保存到输出集合,清除临时缓冲区,冲洗并重复。
这是执行此操作的基本实现。
template<class C=char>
struct basic_token
{
typedef std::basic_string<C> token_string;
typedef unsigned long size_type;
token_string token_, delim_;
basic_token(const token_string& token, const token_string& delim = token_string());
};
template<class C>
basic_token<C>::basic_token(const token_string& token, const token_string& delim)
: token_(token),
delim_(delim)
{
}
typedef basic_token<char> token;
template<class Char, class Iter> void tokenize(const std::basic_string<Char>& line, const Char* delims, Iter itx)
{
typedef basic_token<Char> Token;
typedef std::basic_string<Char> TString;
for( TString::size_type tok_begin = 0, tok_end = line.find_first_of(delims, tok_begin);
tok_begin != TString::npos; tok_end = line.find_first_of(delims, tok_begin) )
{
if( tok_end == TString::npos )
{
(*itx++) = Token(TString(&line[tok_begin]));
tok_begin = tok_end;
}
else
{
(*itx++) = Token(TString(&line[tok_begin], &line[tok_end]), TString(1, line[tok_end]));
tok_begin = tok_end + 1;
}
}
}
template<class Char, class Iter> void tokenize(const Char* line, const Char* delim, Iter itx)
{
tokenize(std::basic_string<Char>(line), delim, itx);
}
template<class Stream, class Token> Stream& operator<<(Stream& os, const Token& tok)
{
os << tok.token_ << "\t[" << tok.delim_ << "]";
return os;
}
...你会像这样使用:
string raw = "35=BW|49=TEST|1346=REQ22|1355=2|1182=88500|1183=88505|10=087^";
vector<stoken> tokens;
tokenize(raw, "|", back_inserter(tokens));
copy(tokens.begin(), tokens.end(), ostream_iterator<stoken>(cout, "\n"));
输出是:
35=BW [|]
49=TEST [|]
1346=REQ22 [|]
1355=2 [|]
1182=88500 [|]
1183=88505 [|]
10=087^ []
我想提醒你,strtok及其同类存在风险,你可以获得比你想要的更多不同数量的代币。
one|two|three would yield 3 tokens
而
one|||three would yield 2.
mmhmm的测试没有正确使用精神,他的语法是缺陷。
#include <cstdio>
#include <cstring>
#include <iostream>
#include <string>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/io.hpp>
#include <boost/spirit/include/qi.hpp>
/****************************strtok_r************************/
typedef struct sTokenDataC {
char *time;
char *symb;
float bid;
float ask;
int bidSize;
int askSize;
} tokenDataC;
tokenDataC parseTick( char *line, char *parseBuffer )
{
tokenDataC tokenDataOut;
tokenDataOut.time = strtok_r( line,",", &parseBuffer );
tokenDataOut.symb = strtok_r( nullptr,",", &parseBuffer );
tokenDataOut.bid = atof(strtok_r( nullptr,",", &parseBuffer ));
tokenDataOut.ask = atof(strtok_r( nullptr , ",", &parseBuffer ));
tokenDataOut.bidSize = atoi(strtok_r( nullptr,",", &parseBuffer ));
tokenDataOut.askSize = atoi(strtok_r( nullptr, ",", &parseBuffer ));
return tokenDataOut;
}
void test_strcpy_s(int iteration)
{
char *testStringC = new char[64];
char *lineBuffer = new char[64];
printf("test_strcpy_s....\n");
strcpy(testStringC,"09:30:00,TEST,13.24,15.32,10,14");
{
timeEstimate<> es;
tokenDataC tokenData2;
for(int i = 0; i < iteration; i++)
{
strcpy(lineBuffer, testStringC);//this is more realistic since this has to happen because I want to preserve the line
tokenData2 = parseTick(lineBuffer, testStringC);
//std::cout<<*tokenData2.time<<", "<<*tokenData2.symb<<",";
//std::cout<<tokenData2.bid<<", "<<tokenData2.ask<<", "<<tokenData2.bidSize<<", "<<tokenData2.askSize<<std::endl;
}
}
delete[] lineBuffer;
delete[] testStringC;
}
/****************************strtok_r************************/
/****************************spirit::qi*********************/
namespace qi = boost::spirit::qi;
struct tokenDataCPP
{
std::string time;
std::string symb;
float bid;
float ask;
int bidSize;
int askSize;
void clearTimeSymb(){
time.clear();
symb.clear();
}
};
BOOST_FUSION_ADAPT_STRUCT(
tokenDataCPP,
(std::string, time)
(std::string, symb)
(float, bid)
(float, ask)
(int, bidSize)
(int, askSize)
)
void test_spirit_qi(int iteration)
{
std::string const strs("09:30:00,TEST,13.24,15.32,10,14");
tokenDataCPP data;
auto myString = *~qi::char_(",");
auto parser = myString >> "," >> myString >> "," >> qi::float_ >> "," >> qi::float_ >> "," >> qi::int_ >> "," >> qi::int_;
{
std::cout<<("test_spirit_qi....\n");
timeEstimate<> es;
for(int i = 0; i < iteration; ++i){
qi::parse(std::begin(strs), std::end(strs), parser, data);
//std::cout<<data.time<<", "<<data.symb<<", ";
//std::cout<<data.bid<<", "<<data.ask<<", "<<data.bidSize<<", "<<data.askSize<<std::endl;
data.clearTimeSymb();
}
}
}
/****************************spirit::qi*********************/
int main()
{
int const ITERATIONS = 500 * 10000;
test_strcpy_s(ITERATIONS);
test_spirit_qi(ITERATIONS);
}
由于clang ++没有strtok_s,我使用strtok_r替换它迭代500 * 10k,时间是
他们的时代差不多,差别不大。
编译器,clang ++ 3.2,-O2
timeEstime的代码
这应该非常快,没有临时缓冲区,它也分配空的tokes。
template <class char_t, class char_traits_t,
class char_allocator_t, class string_allocator_t>
inline void _tokenize(
const std::basic_string<char_t, char_traits_t, char_allocator_t>& _Str,
const char_t& _Tok,
std::vector<std::basic_string<char_t, char_traits_t, char_allocator_t>,
string_allocator_t>& _Tokens,
const size_t& _HintSz=10)
{
_Tokens.reserve(_HintSz);
const char_t* _Beg(&_Str[0]), *_End(&_Str[_Str.size()]);
for (const char_t* _Ptr=_Beg; _Ptr<_End; ++_Ptr)
{
if (*_Ptr == _Tok)
{
_Tokens.push_back(
std::basic_string<char_t, char_traits_t,
char_allocator_t>(_Beg, _Ptr));
_Beg = 1+_Ptr;
}
}
_Tokens.push_back(
std::basic_string<char_t, char_traits_t,
char_allocator_t>(_Beg, _End));
}
在对每个建议的候选人进行测试和计时之后,结果是strtok显然是最快的。虽然我对测试的热爱并不感到惊讶,但是值得探索其他选择。 [注意:代码被抛在一起编辑欢迎:)]
鉴于:
typedef struct sTokenDataC {
char *time;
char *symb;
float bid;
float ask;
int bidSize;
int askSize;
} tokenDataC;
tokenDataC parseTick( char *line, char *parseBuffer )
{
tokenDataC tokenDataOut;
tokenDataOut.time = strtok_s( line,",", &parseBuffer );
tokenDataOut.symb = strtok_s( null,",", &parseBuffer );
tokenDataOut.bid = atof(strtok_s( null,",", &parseBuffer ));
tokenDataOut.ask = atof(strtok_s( null , ",", &parseBuffer ));
tokenDataOut.bidSize = atoi(strtok_s( null,",", &parseBuffer ));
tokenDataOut.askSize = atoi(strtok_s( null, ",", &parseBuffer ));
return tokenDataOut;
}
char *testStringC = new char[64];
strcpy(testStringC,"09:30:00,TEST,13.24,15.32,10,14");
int _tmain(int argc, _TCHAR* argv[])
{
char *lineBuffer = new char[64];
printf("Testing method2....\n");
for(int i = 0; i < ITERATIONS; i++)
{
strcpy(lineBuffer,testStringC);//this is more realistic since this has to happen because I want to preserve the line
tokenData2 = parseTick(lineBuffer,parseBuffer);
}
}
vs呼叫John Diblings通过以下方式:
struct sTokenDataCPP
{
std::basic_string<char> time;
std::basic_string<char> symb;
float bid;
float ask;
int bidSize;
int askSize;
};
std::vector<myToken> tokens1;
tokenDataCPP tokenData;
printf("Testing method1....\n");
for(int i = 0; i < ITERATIONS; i++)
{
tokens1.clear();
tokenize(raw, ",", std::back_inserter(tokens1));
tokenData.time.assign(tokens1.at(0).token_);
tokenData.symb.assign(tokens1.at(1).token_);
tokenData.ask = atof(tokens1.at(2).token_.c_str());
tokenData.bid = atof(tokens1.at(3).token_.c_str());
tokenData.askSize = atoi(tokens1.at(4).token_.c_str());
tokenData.bidSize = atoi(tokens1.at(5).token_.c_str());
}
vs一个简单的boost.spirit.qi实现,定义语法如下:
template <typename Iterator>
struct tick_parser : grammar<Iterator, tokenDataCPP(), boost::spirit::ascii::space_type>
{
tick_parser() : tick_parser::base_type(start)
{
my_string %= lexeme[+(boost::spirit::ascii::char_ ) ];
start %=
my_string >> ','
>> my_string >> ','
>> float_ >> ','
>> float_ >> ','
>> int_ >> ','
>> int_
;
}
rule<Iterator, std::string(), boost::spirit::ascii::space_type> my_string;
rule<Iterator, sTokenDataCPP(), boost::spirit::ascii::space_type> start;
};
ITERATIONS设置为500k:strtok版本:2s john的版本:115s提升:172s
我可以发布完整的代码是人们想要这个,我只是不想占用巨大的空间