textresourcedecoder.cpp

linux下开源浏览器WebKit的源码,市面上的很多商用浏览器都是移植自WebKit

    return pos != dataEnd;
}

void TextResourceDecoder::checkForBOM(const char* data, size_t len)
{
    // Check for UTF-16/32 or UTF-8 BOM mark at the beginning, which is a sure sign of a Unicode encoding.

    if (m_source == UserChosenEncoding) {
        // FIXME: Maybe a BOM should override even a user-chosen encoding.
        m_checkedForBOM = true;
        return;
    }

    // Check if we have enough data.
    size_t bufferLength = m_buffer.size();
    if (bufferLength + len < 4)
        return;

    m_checkedForBOM = true;

    // Extract the first four bytes.
    // Handle the case where some of bytes are already in the buffer.
    // The last byte is always guaranteed to not be in the buffer.
    const unsigned char* udata = reinterpret_cast<const unsigned char*>(data);
    unsigned char c1 = bufferLength >= 1 ? m_buffer[0] : *udata++;
    unsigned char c2 = bufferLength >= 2 ? m_buffer[1] : *udata++;
    unsigned char c3 = bufferLength >= 3 ? m_buffer[2] : *udata++;
    ASSERT(bufferLength < 4);
    unsigned char c4 = *udata;

    // Check for the BOM.
    if (c1 == 0xFF && c2 == 0xFE) {
        if (c3 !=0 || c4 != 0)
            setEncoding(UTF16LittleEndianEncoding(), AutoDetectedEncoding);
        else 
            setEncoding(UTF32LittleEndianEncoding(), AutoDetectedEncoding);
    }
    else if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF)
        setEncoding(UTF8Encoding(), AutoDetectedEncoding);
    else if (c1 == 0xFE && c2 == 0xFF)
        setEncoding(UTF16BigEndianEncoding(), AutoDetectedEncoding);
    else if (c1 == 0 && c2 == 0 && c3 == 0xFE && c4 == 0xFF)
        setEncoding(UTF32BigEndianEncoding(), AutoDetectedEncoding);
}

bool TextResourceDecoder::checkForCSSCharset(const char* data, size_t len, bool& movedDataToBuffer)
{
    if (m_source != DefaultEncoding) {
        m_checkedForCSSCharset = true;
        return true;
    }

    size_t oldSize = m_buffer.size();
    m_buffer.grow(oldSize + len);
    memcpy(m_buffer.data() + oldSize, data, len);

    movedDataToBuffer = true;

    if (m_buffer.size() > 8) { // strlen("@charset") == 8
        const char* dataStart = m_buffer.data();
        const char* dataEnd = dataStart + m_buffer.size();

        if (dataStart[0] == '@' && dataStart[1] == 'c' && dataStart[2] == 'h' && dataStart[3] == 'a' && dataStart[4] == 'r' && 
            dataStart[5] == 's' && dataStart[6] == 'e' && dataStart[7] == 't') {
    
            dataStart += 8;
            const char* pos = dataStart;
            if (!skipWhitespace(pos, dataEnd))
                return false;

            if (*pos == '"' || *pos == '\'') {
                char quotationMark = *pos;
                ++pos;
                dataStart = pos;
            
                while (pos < dataEnd && *pos != quotationMark)
                    ++pos;
                if (pos == dataEnd)
                    return false;

                int encodingNameLength = pos - dataStart + 1;
                
                ++pos;
                if (!skipWhitespace(pos, dataEnd))
                    return false;

                if (*pos == ';')
                    setEncoding(findTextEncoding(dataStart, encodingNameLength), EncodingFromCSSCharset);
            }
        }
        m_checkedForCSSCharset = true;
        return true;
    }
    return false;
}

// Other browsers allow comments in the head section, so we need to also.
// It's important not to look for tags inside the comments.
static inline void skipComment(const char*& ptr, const char* pEnd)
{
    const char* p = ptr;
    // Allow <!-->; other browsers do.
    if (*p == '>') {
        p++;
    } else {
        while (p != pEnd) {
            if (*p == '-') {
                // This is the real end of comment, "-->".
                if (p[1] == '-' && p[2] == '>') {
                    p += 3;
                    break;
                }
                // This is the incorrect end of comment that other browsers allow, "--!>".
                if (p[1] == '-' && p[2] == '!' && p[3] == '>') {
                    p += 4;
                    break;
                }
            }
            p++;
        }
    }
    ptr = p;
}

const int bytesToCheckUnconditionally = 1024; // That many input bytes will be checked for meta charset even if <head> section is over.

bool TextResourceDecoder::checkForHeadCharset(const char* data, size_t len, bool& movedDataToBuffer)
{
    if (m_source != DefaultEncoding) {
        m_checkedForHeadCharset = true;
        return true;
    }

    // This is not completely efficient, since the function might go
    // through the HTML head several times.

    size_t oldSize = m_buffer.size();
    m_buffer.grow(oldSize + len);
    memcpy(m_buffer.data() + oldSize, data, len);

    movedDataToBuffer = true;

    const char* ptr = m_buffer.data();
    const char* pEnd = ptr + m_buffer.size();

    // Is there enough data available to check for XML declaration?
    if (m_buffer.size() < 8)
        return false;

    // Handle XML declaration, which can have encoding in it. This encoding is honored even for HTML documents.
    // It is an error for an XML declaration not to be at the start of an XML document, and it is ignored in HTML documents in such case.
    if (ptr[0] == '<' && ptr[1] == '?' && ptr[2] == 'x' && ptr[3] == 'm' && ptr[4] == 'l') {
        const char* xmlDeclarationEnd = ptr;
        while (xmlDeclarationEnd != pEnd && *xmlDeclarationEnd != '>')
            ++xmlDeclarationEnd;
        if (xmlDeclarationEnd == pEnd)
            return false;
        // No need for +1, because we have an extra "?" to lose at the end of XML declaration.
        int len;
        int pos = findXMLEncoding(ptr, xmlDeclarationEnd - ptr, len);
        if (pos != -1)
            setEncoding(findTextEncoding(ptr + pos, len), EncodingFromXMLHeader);
        // continue looking for a charset - it may be specified in an HTTP-Equiv meta
    } else if (ptr[0] == '<' && ptr[1] == 0 && ptr[2] == '?' && ptr[3] == 0 && ptr[4] == 'x' && ptr[5] == 0) {
        setEncoding(UTF16LittleEndianEncoding(), AutoDetectedEncoding);
        return true;
    } else if (ptr[0] == 0 && ptr[1] == '<' && ptr[2] == 0 && ptr[3] == '?' && ptr[4] == 0 && ptr[5] == 'x') {
        setEncoding(UTF16BigEndianEncoding(), AutoDetectedEncoding);
        return true;
    } else if (ptr[0] == '<' && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == 0 && ptr[4] == '?' && ptr[5] == 0 && ptr[6] == 0 && ptr[7] == 0) {
        setEncoding(UTF32LittleEndianEncoding(), AutoDetectedEncoding);
        return true;
    } else if (ptr[0] == 0 && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == '<' && ptr[4] == 0 && ptr[5] == 0 && ptr[6] == 0 && ptr[7] == '?') {
        setEncoding(UTF32BigEndianEncoding(), AutoDetectedEncoding);
        return true;
    }

    // we still don't have an encoding, and are in the head
    // the following tags are allowed in <head>:
    // SCRIPT|STYLE|META|LINK|OBJECT|TITLE|BASE
    
    // We stop scanning when a tag that is not permitted in <head>
    // is seen, rather when </head> is seen, because that more closely
    // matches behavior in other browsers; more details in
    // <http://bugs.webkit.org/show_bug.cgi?id=3590>.
    
    // Additionally, we ignore things that looks like tags in <title>, <script> and <noscript>; see
    // <http://bugs.webkit.org/show_bug.cgi?id=4560>, <http://bugs.webkit.org/show_bug.cgi?id=12165>
    // and <http://bugs.webkit.org/show_bug.cgi?id=12389>.

    // Since many sites have charset declarations after <body> or other tags that are disallowed in <head>,
    // we don't bail out until we've checked at least bytesToCheckUnconditionally bytes of input.

    AtomicStringImpl* enclosingTagName = 0;
    bool inHeadSection = true; // Becomes false when </head> or any tag not allowed in head is encountered.

    // the HTTP-EQUIV meta has no effect on XHTML
    if (m_contentType == XML)
        return true;

    while (ptr + 3 < pEnd) { // +3 guarantees that "<!--" fits in the buffer - and certainly we aren't going to lose any "charset" that way.
        if (*ptr == '<') {
            bool end = false;
            ptr++;

            // Handle comments.
            if (ptr[0] == '!' && ptr[1] == '-' && ptr[2] == '-') {
                ptr += 3;
                skipComment(ptr, pEnd);
                if (ptr - m_buffer.data() >= bytesToCheckUnconditionally && !inHeadSection) {
                    // Some pages that test bandwidth from within the browser do it by having
                    // huge comments and measuring the time they take to load. Repeatedly scanning
                    // these comments can take a lot of CPU time.
                    m_checkedForHeadCharset = true;
                    return true;
                }
                continue;
            }

            if (*ptr == '/') {
                ++ptr;
                end = true;
            }

            // Grab the tag name, but mostly ignore namespaces.
            bool sawNamespace = false;
            char tagBuffer[20];
            int len = 0;
            while (len < 19) {
                if (ptr == pEnd)
                    return false;
                char c = *ptr;
                if (c == ':') {
                    len = 0;
                    sawNamespace = true;
                    ptr++;
                    continue;
                }
                if (c >= 'a' && c <= 'z' || c >= '0' && c <= '9')
                    ;
                else if (c >= 'A' && c <= 'Z')
                    c += 'a' - 'A';
                else
                    break;
                tagBuffer[len++] = c;
                ptr++;
            }
            tagBuffer[len] = 0;
            AtomicString tag(tagBuffer);
            
            if (enclosingTagName) {
                if (end && tag.impl() == enclosingTagName)
                    enclosingTagName = 0;
            } else {
                if (tag == titleTag)
                    enclosingTagName = titleTag.localName().impl();
                else if (tag == scriptTag)
                    enclosingTagName = scriptTag.localName().impl();
                else if (tag == noscriptTag)
                    enclosingTagName = noscriptTag.localName().impl();
            }
            
            // Find where the opening tag ends.
            const char* tagContentStart = ptr;
            if (!end) {
                while (ptr != pEnd && *ptr != '>') {
                    if (*ptr == '\'' || *ptr == '"') {
                        char quoteMark = *ptr;
                        ++ptr;
                        while (ptr != pEnd && *ptr != quoteMark)
                            ++ptr;
                        if (ptr == pEnd)
                            return false;
                    }
                    ++ptr;
                }
                if (ptr == pEnd)
                    return false;
                ++ptr;
            }
            
            if (!end && tag == metaTag && !sawNamespace) {
                const char* str = tagContentStart;
                int length = ptr - tagContentStart;
                int pos = 0;
                while (pos < length) {
                    int charsetPos = findIgnoringCase(str + pos, length - pos, "charset");
                    if (charsetPos == -1)
                        break;
                    pos += charsetPos + 7;
                    // skip whitespace
                    while (pos < length && str[pos] <= ' ')
                        pos++;
                    if (pos == length)
                        break;
                    if (str[pos++] != '=')
                        continue;
                    while (pos < length &&
                            (str[pos] <= ' ') || str[pos] == '=' || str[pos] == '"' || str[pos] == '\'')
                        pos++;

                    // end ?
                    if (pos == length)
                        break;
                    int end = pos;
                    while (end < length &&
                           str[end] != ' ' && str[end] != '"' && str[end] != '\'' &&
                           str[end] != ';' && str[end] != '>')
                        end++;
                    setEncoding(findTextEncoding(str + pos, end - pos), EncodingFromMetaTag);
                    if (m_source == EncodingFromMetaTag)
                        return true;

                    if (end >= length || str[end] == '/' || str[end] == '>')
                        break;

                    pos = end + 1;
                }
            } else {
                if (!enclosingTagName && tag != scriptTag && tag != noscriptTag && tag != styleTag
                    && tag != linkTag && tag != metaTag && tag != objectTag && tag != titleTag && tag != baseTag
                    && (end || tag != htmlTag) && (end || tag != headTag) && isASCIIAlpha(tagBuffer[0])) {
                    inHeadSection = false;
                }

                if (ptr - m_buffer.data() >= bytesToCheckUnconditionally && !inHeadSection) {
                    m_checkedForHeadCharset = true;
                    return true;
                }
            }
        } else
            ++ptr;
    }
    return false;
}

void TextResourceDecoder::detectJapaneseEncoding(const char* data, size_t len)
{
    switch (KanjiCode::judge(data, len)) {
        case KanjiCode::JIS:
            setEncoding("ISO-2022-JP", AutoDetectedEncoding);
            break;
        case KanjiCode::EUC:
            setEncoding("EUC-JP", AutoDetectedEncoding);
            break;
        case KanjiCode::SJIS:
            setEncoding("Shift_JIS", AutoDetectedEncoding);
            break;
        case KanjiCode::ASCII:
        case KanjiCode::UTF16:
        case KanjiCode::UTF8:
            break;
    }
}

String TextResourceDecoder::decode(const char* data, size_t len)
{
    if (!m_checkedForBOM)
        checkForBOM(data, len);

    bool movedDataToBuffer = false;

    if (m_contentType == CSS && !m_checkedForCSSCharset)
        if (!checkForCSSCharset(data, len, movedDataToBuffer))
            return "";

    if ((m_contentType == HTML || m_contentType == XML) && !m_checkedForHeadCharset) // HTML and XML
        if (!checkForHeadCharset(data, len, movedDataToBuffer))
            return "";

    // Do the auto-detect if our default encoding is one of the Japanese ones.
    // FIXME: It seems wrong to change our encoding downstream after we have already done some decoding.
    if (m_source != UserChosenEncoding && m_source != AutoDetectedEncoding && encoding().isJapanese())
        detectJapaneseEncoding(data, len);

    ASSERT(encoding().isValid());

    if (m_buffer.isEmpty())
        return m_decoder.decode(data, len, false, m_contentType == XML, m_sawError);

    if (!movedDataToBuffer) {
        size_t oldSize = m_buffer.size();
        m_buffer.grow(oldSize + len);
        memcpy(m_buffer.data() + oldSize, data, len);
    }

    String result = m_decoder.decode(m_buffer.data(), m_buffer.size(), false, m_contentType == XML && !m_useLenientXMLDecoding, m_sawError);
    m_buffer.clear();
    return result;
}

String TextResourceDecoder::flush()
{
    String result = m_decoder.decode(m_buffer.data(), m_buffer.size(), true, m_contentType == XML && !m_useLenientXMLDecoding, m_sawError);
    m_buffer.clear();
    m_decoder.reset(m_decoder.encoding());
    return result;
}

}