Found a bug in Tokenzier

[Loaden]

Like thie code:

Code

    Tokenizer tk;
    tk.InitFromBuffer(_T("15 + 25 * 2"));
    wxString s;
    while (tk.NotEOF())
    {
        s += tk.GetToken();
    }

The s Value is:

Code

15+25*

Will lost the last char '2'.

This patch can fix it:

Code

wxString Tokenizer::DoGetToken()
{
...
    else if (wxIsdigit(c))
    {
        // numbers
+        int count = 0;
        while (NotEOF() && CharInString(CurrentChar(), _T("0123456789.abcdefABCDEFXxLl")))
        {
            MoveToNextChar();
+            ++count;
        }

-        if (IsEOF())
+        if (IsEOF() && count == 0)
            return wxEmptyString;
        }

Morten, your opinion?

[Loaden]

Maybe a better way, just comment these two lines!

Code

        if (IsEOF())
            return wxEmptyString;

Even "IsEOF()" is true, "m_Buffer.Mid (start, m_TokenIndex - start);" can be also work!

Code

    if (c == '_' || wxIsalpha(c))
    {
        // keywords, identifiers, etc.

        // operator== is cheaper than wxIsalnum, also MoveToNextChar already includes IsEOF
        while (    ( (c == '_') || (wxIsalnum(c)) )
               &&  MoveToNextChar() )
            c = CurrentChar(); // repeat

//        if (IsEOF())
//            return wxEmptyString;

        needReplace = true;
        str = m_Buffer.Mid(start, m_TokenIndex - start);
    }
#ifdef __WXMSW__ // This is a Windows only bug!
    else if (c == 178 || c == 179 || c == 185) // fetch ?and ?
    {
        str = c;
        MoveToNextChar();
    }
#endif
    else if (wxIsdigit(c))
    {
        // numbers
        while (NotEOF() && CharInString(CurrentChar(), _T("0123456789.abcdefABCDEFXxLl")))
        {
            MoveToNextChar();
        }

//        if (IsEOF())
//            return wxEmptyString;

        str = m_Buffer.Mid(start, m_TokenIndex - start);
        m_IsOperator = false;
    }