Like thie code:
Tokenizer tk;
tk.InitFromBuffer(_T("15 + 25 * 2"));
wxString s;
while (tk.NotEOF())
{
s += tk.GetToken();
}
The s Value is:
Will lost the last char '2'.
This patch can fix it:
wxString Tokenizer::DoGetToken()
{
...
else if (wxIsdigit(c))
{
// numbers
+ int count = 0;
while (NotEOF() && CharInString(CurrentChar(), _T("0123456789.abcdefABCDEFXxLl")))
{
MoveToNextChar();
+ ++count;
}
- if (IsEOF())
+ if (IsEOF() && count == 0)
return wxEmptyString;
}
Morten, your opinion?
Maybe a better way, just comment these two lines!
if (IsEOF())
return wxEmptyString;
Even "IsEOF()" is true, "m_Buffer.Mid (start, m_TokenIndex - start);" can be also work!
if (c == '_' || wxIsalpha(c))
{
// keywords, identifiers, etc.
// operator== is cheaper than wxIsalnum, also MoveToNextChar already includes IsEOF
while ( ( (c == '_') || (wxIsalnum(c)) )
&& MoveToNextChar() )
c = CurrentChar(); // repeat
// if (IsEOF())
// return wxEmptyString;
needReplace = true;
str = m_Buffer.Mid(start, m_TokenIndex - start);
}
#ifdef __WXMSW__ // This is a Windows only bug!
else if (c == 178 || c == 179 || c == 185) // fetch ?and ?
{
str = c;
MoveToNextChar();
}
#endif
else if (wxIsdigit(c))
{
// numbers
while (NotEOF() && CharInString(CurrentChar(), _T("0123456789.abcdefABCDEFXxLl")))
{
MoveToNextChar();
}
// if (IsEOF())
// return wxEmptyString;
str = m_Buffer.Mid(start, m_TokenIndex - start);
m_IsOperator = false;
}