Developer forums (C::B DEVELOPMENT STRICTLY!) > CodeCompletion redesign

Found a bug in Tokenzier

(1/1)

Loaden:
Like thie code:

--- Code: ---    Tokenizer tk;
    tk.InitFromBuffer(_T("15 + 25 * 2"));
    wxString s;
    while (tk.NotEOF())
    {
        s += tk.GetToken();
    }
--- End code ---
The s Value is:

--- Code: ---15+25*
--- End code ---

Will lost the last char '2'.

This patch can fix it:

--- Code: ---wxString Tokenizer::DoGetToken()
{
...
    else if (wxIsdigit(c))
    {
        // numbers
+        int count = 0;
        while (NotEOF() && CharInString(CurrentChar(), _T("0123456789.abcdefABCDEFXxLl")))
        {
            MoveToNextChar();
+            ++count;
        }

-        if (IsEOF())
+        if (IsEOF() && count == 0)
            return wxEmptyString;
        }

--- End code ---
Morten, your opinion?

Loaden:
Maybe a better way, just comment these two lines!

--- Code: ---        if (IsEOF())
            return wxEmptyString;
--- End code ---
Even "IsEOF()" is true, "m_Buffer.Mid (start, m_TokenIndex - start);" can be also work!

--- Code: ---    if (c == '_' || wxIsalpha(c))
    {
        // keywords, identifiers, etc.

        // operator== is cheaper than wxIsalnum, also MoveToNextChar already includes IsEOF
        while (    ( (c == '_') || (wxIsalnum(c)) )
               &&  MoveToNextChar() )
            c = CurrentChar(); // repeat

//        if (IsEOF())
//            return wxEmptyString;

        needReplace = true;
        str = m_Buffer.Mid(start, m_TokenIndex - start);
    }
#ifdef __WXMSW__ // This is a Windows only bug!
    else if (c == 178 || c == 179 || c == 185) // fetch ?and ?
    {
        str = c;
        MoveToNextChar();
    }
#endif
    else if (wxIsdigit(c))
    {
        // numbers
        while (NotEOF() && CharInString(CurrentChar(), _T("0123456789.abcdefABCDEFXxLl")))
        {
            MoveToNextChar();
        }

//        if (IsEOF())
//            return wxEmptyString;

        str = m_Buffer.Mid(start, m_TokenIndex - start);
        m_IsOperator = false;
    }
--- End code ---

Navigation

[0] Message Index