index.cpp

来自「奇趣公司比较新的qt/emd版本」· C++ 代码 · 共 573 行 · 第 1/2 页

/****************************************************************************
**
** Copyright (C) 1992-2007 Trolltech ASA. All rights reserved.
**
** This file is part of the Qt Assistant of the Qt Toolkit.
**
** This file may be used under the terms of the GNU General Public
** License version 2.0 as published by the Free Software Foundation
** and appearing in the file LICENSE.GPL included in the packaging of
** this file.  Please review the following information to ensure GNU
** General Public Licensing requirements will be met:
** http://trolltech.com/products/qt/licenses/licensing/opensource/
**
** If you are unsure which license is appropriate for your use, please
** review the following information:
** http://trolltech.com/products/qt/licenses/licensing/licensingoverview
** or contact the sales department at [e-mail].
**
** In addition, as a special exception, Trolltech gives you certain
** additional rights. These rights are described in the Trolltech GPL
** Exception version 1.0, which can be found at
** http://www.trolltech.com/products/qt/gplexception/ and in the file
** GPL_EXCEPTION.txt in this package.
**
** In addition, as a special exception, Trolltech, as the sole copyright
** holder for Qt Designer, grants users of the Qt/Eclipse Integration
** plug-in the right for the Qt/Eclipse Integration to link to
** functionality provided by Qt Designer and its related libraries.
**
** Trolltech reserves all rights not expressly granted herein.
**
** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
**
****************************************************************************/

#include "index.h"

#include <QFile>
#include <QDir>
#include <QStringList>
#include <QApplication>
#include <QByteArray>
#include <QTextStream>
#include <QtAlgorithms>
#include <QUrl>
#include <QTextCodec>
#include <ctype.h>
#include <QTextDocument>

struct Term {
    Term() : frequency(-1) {}
    Term( const QString &t, int f, QVector<Document> l ) : term( t ), frequency( f ), documents( l ) {}
    QString term;
    int frequency;
    QVector<Document>documents;
    bool operator<( const Term &i2 ) const { return frequency < i2.frequency; }
};

QDataStream &operator>>( QDataStream &s, Document &l )
{
    s >> l.docNumber;
    s >> l.frequency;
    return s;
}

QDataStream &operator<<( QDataStream &s, const Document &l )
{
    s << (qint16)l.docNumber;
    s << (qint16)l.frequency;
    return s;
}

Index::Index( const QString &dp, const QString &hp )
    : QObject( 0 ), docPath( dp )
{
    Q_UNUSED(hp);

    alreadyHaveDocList = false;
    lastWindowClosed = false;
    connect( qApp, SIGNAL(lastWindowClosed()),
             this, SLOT(setLastWinClosed()) );
}

Index::Index( const QStringList &dl, const QString &hp )
    : QObject( 0 )
{
    Q_UNUSED(hp);
    docList = dl;
    alreadyHaveDocList = true;
    lastWindowClosed = false;
    connect( qApp, SIGNAL(lastWindowClosed()),
             this, SLOT(setLastWinClosed()) );
}

void Index::setLastWinClosed()
{
    lastWindowClosed = true;
}

void Index::setDictionaryFile( const QString &f )
{
    dictFile = f;
}

void Index::setDocListFile( const QString &f )
{
    docListFile = f;
}

void Index::setDocList( const QStringList &lst )
{
    docList = lst;
}

int Index::makeIndex()
{
    if ( !alreadyHaveDocList )
        setupDocumentList();
    if ( docList.isEmpty() )
        return 1;
    QStringList::Iterator it = docList.begin();
    int steps = docList.count() / 100;
    if ( !steps )
        steps++;
    int prog = 0;
    for ( int i = 0; it != docList.end(); ++it, ++i ) {
        if ( lastWindowClosed ) {
            return -1;
        }
        QUrl url(*it);
        parseDocument( url.toLocalFile(), i );
        if ( i%steps == 0 ) {
            prog++;
            emit indexingProgress( prog );
        }
    }
    return 0;
}

void Index::setupDocumentList()
{
    QDir d( docPath );
    QStringList filters;
    filters.append(QLatin1String("*.html"));
    QStringList lst = d.entryList(filters);
    QStringList::ConstIterator it = lst.constBegin();
    for ( ; it != lst.constEnd(); ++it )
        docList.append( QLatin1String("file:") + docPath + QLatin1String("/") + *it );
}

void Index::insertInDict( const QString &str, int docNum )
{
    if ( str == QLatin1String("amp") || str == QLatin1String("nbsp"))
        return;
    Entry *e = 0;
    if ( dict.count() )
        e = dict[ str ];

    if ( e ) {
        if ( e->documents.last().docNumber != docNum )
            e->documents.append( Document(docNum, 1 ) );
        else
            e->documents.last().frequency++;
    } else {
        dict.insert( str, new Entry( docNum ) );
    }
}

QString Index::getCharsetForDocument(QFile *file)
{
    QTextStream s(file);
    QString contents = s.readAll();

    QString encoding;
    int start = contents.indexOf(QLatin1String("<meta"), 0, Qt::CaseInsensitive);
    if (start > 0) {
        int end = contents.indexOf(QLatin1String(">"), start);
        QString meta = contents.mid(start+5, end-start);
        meta = meta.toLower();
        QRegExp r(QLatin1String("charset=([^\"\\s]+)"));
        if (r.indexIn(meta) != -1) {
            encoding = r.cap(1);        
        }
    }

    file->seek(0);
    if (encoding.isEmpty())
        return QLatin1String("utf-8");
    return encoding;
}

void Index::parseDocument( const QString &filename, int docNum )
{
    QFile file( filename );
    if ( !file.open(QFile::ReadOnly) ) {
        qWarning( "can not open file %s", qPrintable(filename) );
        return;
    }

    QTextStream s(&file);
    QString en = getCharsetForDocument(&file);
    s.setCodec(QTextCodec::codecForName(en.toLatin1().constData()));

    QString text = s.readAll();
    if (text.isNull())
        return;

    bool valid = true;
    const QChar *buf = text.unicode();
    QChar str[64];
    QChar c = buf[0];
    int j = 0;
    int i = 0;
    while ( j < text.length() ) {
        if ( c == QLatin1Char('<') || c == QLatin1Char('&') ) {
            valid = false;
            if ( i > 1 )
                insertInDict( QString(str,i), docNum );
            i = 0;
            c = buf[++j];
            continue;
        }
        if ( ( c == QLatin1Char('>') || c == QLatin1Char(';') ) && !valid ) {
            valid = true;
            c = buf[++j];
            continue;
        }
        if ( !valid ) {
            c = buf[++j];
            continue;
        }
        if ( ( c.isLetterOrNumber() || c == QLatin1Char('_') ) && i < 63 ) {
            str[i] = c.toLower();
            ++i;
        } else {
            if ( i > 1 )
                insertInDict( QString(str,i), docNum );
            i = 0;
        }
        c = buf[++j];
    }
    if ( i > 1 )
        insertInDict( QString(str,i), docNum );
    file.close();
}

void Index::writeDict()
{
    QFile f( dictFile );
    if ( !f.open(QFile::WriteOnly ) )
        return;
    QDataStream s( &f );
    for(QHash<QString, Entry *>::Iterator it = dict.begin(); it != dict.end(); ++it) {
        s << it.key();
        s << it.value()->documents.count();
        s << it.value()->documents;
    }
    f.close();
    writeDocumentList();
}

void Index::writeDocumentList()
{
    QFile f( docListFile );
    if ( !f.open(QFile::WriteOnly ) )
        return;
    QDataStream s( &f );
    s << docList;
}

void Index::readDict()
{
    QFile f( dictFile );
    if ( !f.open(QFile::ReadOnly ) )
        return;

    dict.clear();
    QDataStream s( &f );
    QString key;
    int numOfDocs;
    QVector<Document> docs;
    while ( !s.atEnd() ) {
        s >> key;
        s >> numOfDocs;
        docs.resize(numOfDocs);
        s >> docs;