lexruby.cxx

Wxpython Implemented on Windows CE, Source code

    int firstWordPosn = skipWhitespace(lineStartPosn, lt2StartPos, styler);
    if (firstWordPosn >= lt2StartPos) {
        return definitely_not_a_here_doc;
    }
    prevStyle = styler.StyleAt(firstWordPosn);
    // If we have '<<' following a keyword, it's not a heredoc
    if (prevStyle != SCE_RB_IDENTIFIER) {
        return definitely_not_a_here_doc;
    }
    int newStyle = prevStyle;
    // Some compilers incorrectly warn about uninit newStyle
    for (firstWordPosn += 1; firstWordPosn <= lt2StartPos; firstWordPosn += 1) {
        // Inner loop looks at the name
        for (; firstWordPosn <= lt2StartPos; firstWordPosn += 1) {
            newStyle = styler.StyleAt(firstWordPosn);
            if (newStyle != prevStyle) {
                break;
            }
        }
        // Do we have '::' or '.'?
        if (firstWordPosn < lt2StartPos && newStyle == SCE_RB_OPERATOR) {
            char ch = styler[firstWordPosn];
            if (ch == '.') {
                // yes
            } else if (ch == ':') {
                if (styler.StyleAt(++firstWordPosn) != SCE_RB_OPERATOR) {
                    return definitely_not_a_here_doc;
                } else if (styler[firstWordPosn] != ':') {
                    return definitely_not_a_here_doc;
                }
            } else {
                break;
            }
        } else {
            break;
        }
    }
    // Skip next batch of white-space
    firstWordPosn = skipWhitespace(firstWordPosn, lt2StartPos, styler);
    if (firstWordPosn != lt2StartPos) {
        // Have [[^ws[identifier]ws[*something_else*]ws<<
        return definitely_not_a_here_doc;
    }
    // OK, now 'j' will point to the current spot moving ahead
	int j = firstWordPosn + 1;
    if (styler.StyleAt(j) != SCE_RB_OPERATOR || styler[j] != '<') {
        // This shouldn't happen
        return definitely_not_a_here_doc;
    }
    int nextLineStartPosn = styler.LineStart(lineStart + 1);
    if (nextLineStartPosn >= lengthDoc) {
        return definitely_not_a_here_doc;
    }
    j = skipWhitespace(j + 1, nextLineStartPosn, styler);
    if (j >= lengthDoc) {
        return definitely_not_a_here_doc;
    }
    bool allow_indent;
    int target_start, target_end;
    // From this point on no more styling, since we're looking ahead
    if (styler[j] == '-') {
        allow_indent = true;
        j++;
    } else {
        allow_indent = false;
    }

    // Allow for quoted targets.
    char target_quote = 0;
    switch (styler[j]) {
    case '\'':
    case '"':
    case '`':
        target_quote = styler[j];
        j += 1;
    }
    
    if (isSafeAlnum(styler[j])) {
        // Init target_end because some compilers think it won't
        // be initialized by the time it's used
        target_start = target_end = j;
        j++;
    } else {
        return definitely_not_a_here_doc;
    }
    for (; j < lengthDoc; j++) {
        if (!isSafeAlnum(styler[j])) {
            if (target_quote && styler[j] != target_quote) {
                // unquoted end
                return definitely_not_a_here_doc;
            }

            // And for now make sure that it's a newline
            // don't handle arbitrary expressions yet
            
            target_end = j;
			if (target_quote) {
				// Now we can move to the character after the string delimiter.
				j += 1;
			}
            j = skipWhitespace(j, lengthDoc, styler);
            if (j >= lengthDoc) {
                return definitely_not_a_here_doc;
            } else {
                char ch = styler[j];
                if (ch == '#' || isEOLChar(ch)) {
                    // This is OK, so break and continue;
                    break;
                } else {
                    return definitely_not_a_here_doc;
                }
            }
        }
    }

    // Just look at the start of each line
    int last_line = styler.GetLine(lengthDoc - 1);
    // But don't go too far
    if (last_line > lineStart + 50) {
        last_line = lineStart + 50;
    }
    for (int line_num = lineStart + 1; line_num <= last_line; line_num++) {
        if (allow_indent) {
            j = skipWhitespace(styler.LineStart(line_num), lengthDoc, styler);
        } else {
            j = styler.LineStart(line_num);
        }
        // target_end is one past the end
        if (haveTargetMatch(j, lengthDoc, target_start, target_end, styler)) {
            // We got it
            return looks_like_a_here_doc;
        }
    }
    return definitely_not_a_here_doc;
}

//todo: if we aren't looking at a stdio character,
// move to the start of the first line that is not in a 
// multi-line construct

static void synchronizeDocStart(unsigned int& startPos,
                                int &length,
                                int &initStyle,
                                Accessor &styler,
                                bool skipWhiteSpace=false) {

    styler.Flush();
    int style = actual_style(styler.StyleAt(startPos));
    switch (style) {
        case SCE_RB_STDIN:
        case SCE_RB_STDOUT:
        case SCE_RB_STDERR:
            // Don't do anything else with these.
            return;
    }
    
    int pos = startPos;
    // Quick way to characterize each line
    int lineStart;
    for (lineStart = styler.GetLine(pos); lineStart > 0; lineStart--) {
        // Now look at the style before the previous line's EOL
        pos = styler.LineStart(lineStart) - 1;
        if (pos <= 10) {
            lineStart = 0;
            break;
        }
        char ch = styler.SafeGetCharAt(pos);
        char chPrev = styler.SafeGetCharAt(pos - 1);
        if (ch == '\n' && chPrev == '\r') {
            pos--;
        }
        if (styler.SafeGetCharAt(pos - 1) == '\\') {
            // Continuation line -- keep going
        } else if (actual_style(styler.StyleAt(pos)) != SCE_RB_DEFAULT) {
            // Part of multi-line construct -- keep going
        } else if (currLineContainsHereDelims(pos, styler)) {
            // Keep going, with pos and length now pointing
            // at the end of the here-doc delimiter
        } else if (skipWhiteSpace && isEmptyLine(pos, styler)) {
            // Keep going
        } else {
            break;
        }
    }
    pos = styler.LineStart(lineStart);
    length += (startPos - pos);
    startPos = pos;
    initStyle = SCE_RB_DEFAULT;
}

static void ColouriseRbDoc(unsigned int startPos, int length, int initStyle,
						   WordList *keywordlists[], Accessor &styler) {

	// Lexer for Ruby often has to backtrack to start of current style to determine
	// which characters are being used as quotes, how deeply nested is the
	// start position and what the termination string is for here documents
    
	WordList &keywords = *keywordlists[0];

	class HereDocCls {
	public:
		int State;
        // States
        // 0: '<<' encountered
		// 1: collect the delimiter
        // 1b: text between the end of the delimiter and the EOL
		// 2: here doc text (lines after the delimiter)
		char Quote;		// the char after '<<'
		bool Quoted;		// true if Quote in ('\'','"','`')
		int DelimiterLength;	// strlen(Delimiter)
		char Delimiter[256];	// the Delimiter, limit of 256: from Perl
        bool CanBeIndented;
		HereDocCls() {
			State = 0;
			DelimiterLength = 0;
			Delimiter[0] = '\0';
            CanBeIndented = false;
		}
	};
	HereDocCls HereDoc;	

	class QuoteCls {
		public:
		int  Count;
		char Up;
		char Down;
		QuoteCls() {
			this->New();
		}
		void New() {
			Count = 0;
			Up    = '\0';
			Down  = '\0';
		}
		void Open(char u) {
			Count++;
			Up    = u;
			Down  = opposite(Up);
		}
	};
	QuoteCls Quote;

    int numDots = 0;  // For numbers --
                      // Don't start lexing in the middle of a num

    synchronizeDocStart(startPos, length, initStyle, styler, // ref args
                        false);

	bool preferRE = true;
    int state = initStyle;
	int lengthDoc = startPos + length;

	char prevWord[MAX_KEYWORD_LENGTH + 1]; // 1 byte for zero
	prevWord[0] = '\0';
	if (length == 0)
		return;

	char chPrev = styler.SafeGetCharAt(startPos - 1);
	char chNext = styler.SafeGetCharAt(startPos);
	// Ruby uses a different mask because bad indentation is marked by oring with 32
	styler.StartAt(startPos, 127);
	styler.StartSegment(startPos);

    static int q_states[] = {SCE_RB_STRING_Q,
                             SCE_RB_STRING_QQ,
                             SCE_RB_STRING_QR,
                             SCE_RB_STRING_QW,
                             SCE_RB_STRING_QW,
                             SCE_RB_STRING_QX};
    static const char* q_chars = "qQrwWx";
    
	for (int i = startPos; i < lengthDoc; i++) {
		char ch = chNext;
		chNext = styler.SafeGetCharAt(i + 1);
		char chNext2 = styler.SafeGetCharAt(i + 2);

        if (styler.IsLeadByte(ch)) {
			chNext = chNext2;
			chPrev = ' ';
			i += 1;
			continue;
		}
		
        // skip on DOS/Windows
        //No, don't, because some things will get tagged on,
        // so we won't recognize keywords, for example
#if 0
		if (ch == '\r' && chNext == '\n') {
	    	continue;
        }
#endif
            
        if (HereDoc.State == 1 && isEOLChar(ch)) {
			// Begin of here-doc (the line after the here-doc delimiter):
			HereDoc.State = 2;
			styler.ColourTo(i-1, state);
            // Don't check for a missing quote, just jump into
            // the here-doc state
            state = SCE_RB_HERE_Q;
        }

        // Regular transitions
		if (state == SCE_RB_DEFAULT) {
            if (isSafeDigit(ch)) {
            	styler.ColourTo(i - 1, state);
				state = SCE_RB_NUMBER;
                numDots = 0;
            } else if (isHighBitChar(ch) || iswordstart(ch)) {
            	styler.ColourTo(i - 1, state);
				state = SCE_RB_WORD;
			} else if (ch == '#') {
				styler.ColourTo(i - 1, state);
				state = SCE_RB_COMMENTLINE;
			} else if (ch == '=') {
				// =begin indicates the start of a comment (doc) block
                if (i == 0 || isEOLChar(chPrev)
                    && chNext == 'b'
                    && styler.SafeGetCharAt(i + 2) == 'e'
                    && styler.SafeGetCharAt(i + 3) == 'g'
                    && styler.SafeGetCharAt(i + 4) == 'i'
                    && styler.SafeGetCharAt(i + 5) == 'n'
                    && !isSafeWordcharOrHigh(styler.SafeGetCharAt(i + 6))) {
                    styler.ColourTo(i - 1, state);
                    state = SCE_RB_POD;
				} else {
					styler.ColourTo(i - 1, state);
					styler.ColourTo(i, SCE_RB_OPERATOR);
					preferRE = true;
				}
			} else if (ch == '"') {
				styler.ColourTo(i - 1, state);
				state = SCE_RB_STRING;
				Quote.New();
				Quote.Open(ch);
			} else if (ch == '\'') {
                styler.ColourTo(i - 1, state);
                state = SCE_RB_CHARACTER;
                Quote.New();
                Quote.Open(ch);
			} else if (ch == '`') {
				styler.ColourTo(i - 1, state);
				state = SCE_RB_BACKTICKS;
				Quote.New();
				Quote.Open(ch);
			} else if (ch == '@') {
                // Instance or class var
				styler.ColourTo(i - 1, state);
                if (chNext == '@') {
                    state = SCE_RB_CLASS_VAR;
                    advance_char(i, ch, chNext, chNext2); // pass by ref
                } else {
                    state = SCE_RB_INSTANCE_VAR;
                }
			} else if (ch == '$') {
                // Check for a builtin global
				styler.ColourTo(i - 1, state);
                // Recognize it bit by bit
                state = SCE_RB_GLOBAL;
            } else if (ch == '/' && preferRE) {
                // Ambigous operator
				styler.ColourTo(i - 1, state);
				state = SCE_RB_REGEX;
                Quote.New();
                Quote.Open(ch);
			} else if (ch == '<' && chNext == '<' && chNext2 != '=') {

                // Recognise the '<<' symbol - either a here document or a binary op
				styler.ColourTo(i - 1, state);
                i++;
                chNext = chNext2;
				styler.ColourTo(i, SCE_RB_OPERATOR);

                if (! (strchr("\"\'`_-", chNext2) || isSafeAlpha(chNext2))) {
                    // It's definitely not a here-doc,
                    // based on Ruby's lexer/parser in the
                    // heredoc_identifier routine.
                    // Nothing else to do.
                } else if (preferRE) {
                    if (sureThisIsHeredoc(i - 1, styler, prevWord)) {
                        state = SCE_RB_HERE_DELIM;
                        HereDoc.State = 0;
                    }
                    // else leave it in default state
                } else {
                    if (sureThisIsNotHeredoc(i - 1, styler)) {
                        // leave state as default