gbloader.cpp

来自「ncbi源码」· C++ 代码 · 共 1,566 行 · 第 1/4 页

#endif

    CGBLGuard g(m_Locks,s);
    x_GetRecords(idh, sr_mask);
}


#if !defined(_DEBUG)
inline
#endif
void
CGBDataLoader::x_Check(const STSEinfo* _DEBUG_ARG(me))
{
#if defined(_DEBUG)
    unsigned c = 0;
    bool tse_found=false;
    const STSEinfo* tse2 = m_UseListHead;
    const STSEinfo* t1 = 0;
    while ( tse2 ) {
        c++;
        if( tse2 == me )
            tse_found = true;
        t1 = tse2;
        tse2 = tse2->next;
    }
    _ASSERT(t1 == m_UseListTail);
    _ASSERT(m_Sr2TseInfo.size() == m_TseCount ||
            m_Sr2TseInfo.size() == m_TseCount + 1);
    _ASSERT(c  <= m_TseCount);
    //_ASSERT(m_Tse2TseInfo.size() <= m_TseCount);
    if(me) {
        //GBLOG_POST("check tse " << me << " by " << CThread::GetSelf() );
        _ASSERT(tse_found);
    }
#endif
}


CRef<CGBDataLoader::STSEinfo>
CGBDataLoader::GetTSEinfo(const CTSE_Info& tse_info)
{
    return CRef<STSEinfo>(const_cast<STSEinfo*>
                          (dynamic_cast<const STSEinfo*>
                           (tse_info.GetBlobId().GetPointerOrNull())));
}


void CGBDataLoader::DropTSE(const CTSE_Info& tse_info)
{
    CGBLGuard g(m_Locks,"drop_tse");
    //TTse2TSEinfo::iterator it = m_Tse2TseInfo.find(sep);
    //if (it == m_Tse2TseInfo.end()) // oops - apprently already done;
    //    return true;
    CRef<STSEinfo> tse = GetTSEinfo(tse_info);
    if ( !tse )
        return;
    g.Lock(&*tse);
    _ASSERT(tse);
    x_Check(tse);
  
    //m_Tse2TseInfo.erase(it);
    x_DropTSEinfo(tse);
}


CConstRef<CTSE_Info>
CGBDataLoader::ResolveConflict(const CSeq_id_Handle& handle,
                               const TTSE_LockSet& tse_set)
{
    bool         conflict=false;

#ifdef _DEBUG  
    {{
        CNcbiOstrstream s;
        s << "CGBDataLoader::ResolveConflict("<<handle.AsString();
        ITERATE ( TTSE_LockSet, it, tse_set ) {
            CRef<STSEinfo> ti = GetTSEinfo(**it);
            s << ", " << DUMP(*ti);
        }
        _TRACE(string(CNcbiOstrstreamToString(s)));
    }}
#endif

    GBLOG_POST( "ResolveConflict" );
    CGBLGuard g(m_Locks,"ResolveConflict");
    CRef<SSeqrefs> sr;
    CConstRef<CTSE_Info> best;
    {
        int cnt = 5;
        while ( !(sr = x_ResolveHandle(handle))  && cnt>0 )
            cnt --;
        if ( !sr )
            return best;
    }

    ITERATE(TTSE_LockSet, sit, tse_set) {
        CConstRef<CTSE_Info> ti = *sit;
        CRef<STSEinfo> tse = GetTSEinfo(*ti);
        if ( !tse )
            continue;
      
        x_Check(tse);

        g.Lock(&*tse);

        if(tse->mode.test(STSEinfo::eDead) && !ti->IsDead()) {
            GetDataSource()->x_UpdateTSEStatus(const_cast<CTSE_Info&>(*ti),
                                               true);
        }
        if(tse->m_SeqIds.find(handle)!=tse->m_SeqIds.end()) {
            // listed for given TSE
            if(!best) {
                best=ti; conflict=false;
            }
            else if(!ti->IsDead() && best->IsDead()) {
                best=ti; conflict=false;
            }
            else if(ti->IsDead() && best->IsDead()) {
                conflict=true;
            }
            else if(ti->IsDead() && !best->IsDead()) {
            }
            else {
                conflict=true;
                //_ASSERT(ti->IsDead() || best->IsDead());
            }
        }
        g.Unlock(&*tse);
    }

    if ( !best || conflict ) {
        // try harder
        
        best.Reset();conflict=false;
        ITERATE (SSeqrefs::TSeqrefs, srp, sr->m_Sr) {
            TSr2TSEinfo::iterator tsep =
                m_Sr2TseInfo.find((*srp)->GetKeyByTSE());
            if (tsep == m_Sr2TseInfo.end()) continue;
            ITERATE(TTSE_LockSet, sit, tse_set) {
                CConstRef<CTSE_Info> ti = *sit;
                //TTse2TSEinfo::iterator it =
                //    m_Tse2TseInfo.find(&ti->GetSeq_entry());
                //if(it==m_Tse2TseInfo.end()) continue;
                CRef<STSEinfo> tinfo = GetTSEinfo(*ti);
                if ( !tinfo )
                    continue;

                if(tinfo==tsep->second) {
                    if ( !best )
                        best=ti;
                    else if (ti != best)
                        conflict=true;
                }
            }
        }
        if(conflict)
            best.Reset();
    }
    if ( !best ) {
        _TRACE("CGBDataLoader::ResolveConflict("<<handle.AsString()<<
               ") - conflict");
    }
    return best;
}

//=======================================================================
// GBLoader private interface
// 

void CGBDataLoader::x_ExcludeFromDropList(STSEinfo* tse)
{
    _ASSERT(tse);
    _TRACE("x_ExcludeFromDropList("<<DUMP(*tse)<<")");
    STSEinfo* next = tse->next;
    STSEinfo* prev = tse->prev;
    if ( next || prev ) {
        x_Check(tse);
        if ( next ) {
            _ASSERT(tse != m_UseListTail);
            _ASSERT(tse == next->prev);
            next->prev = prev;
        }
        else {
            _ASSERT(tse == m_UseListTail);
            m_UseListTail = prev;
        }
        if ( prev ) {
            _ASSERT(tse != m_UseListHead);
            _ASSERT(tse == prev->next);
            prev->next = next;
        }
        else {
            _ASSERT(tse == m_UseListHead);
            m_UseListHead = next;
        }
        tse->prev = tse->next = 0;
        --m_TseCount;
    }
    else if ( tse == m_UseListHead ) {
        _ASSERT(tse == m_UseListTail);
        x_Check(tse);
        m_UseListHead = m_UseListTail = 0;
        --m_TseCount;
    }
    else {
        _ASSERT(tse != m_UseListTail);
    }
    x_Check();
}


void CGBDataLoader::x_AppendToDropList(STSEinfo* tse)
{
    _ASSERT(tse);
    _TRACE("x_AppendToDropList("<<DUMP(*tse)<<")");
    _ASSERT(m_Sr2TseInfo[tse->key] == tse);
    x_Check();
    _ASSERT(!tse->next && !tse->prev);
    if ( m_UseListTail ) {
        tse->prev = m_UseListTail;
        m_UseListTail->next = tse;
        m_UseListTail = tse;
    }
    else {
        _ASSERT(!m_UseListHead);
        m_UseListHead = m_UseListTail = tse;
    }
    ++m_TseCount;
    x_Check(tse);
}


void CGBDataLoader::x_UpdateDropList(STSEinfo* tse)
{
    _ASSERT(tse);
    // reset LRU links
    if(tse == m_UseListTail) // already the last one
        return;
  
    // Unlink from current place
    x_ExcludeFromDropList(tse);
    x_AppendToDropList(tse);
}


void CGBDataLoader::x_DropTSEinfo(STSEinfo* tse)
{
    if(!tse) return;
    _TRACE( "DropBlob(" << DUMP(*tse) << ")" );
  
    ITERATE(STSEinfo::TSeqids,sih_it,tse->m_SeqIds) {
        m_Bs2Sr.erase(*sih_it);
    }

    x_ExcludeFromDropList(tse);
    m_Sr2TseInfo.erase(tse->key);
}


void CGBDataLoader::GC(void)
{
    //LOG_POST("X_GC "<<m_TseCount<<","<<m_TseGC_Threshhold<<","<< m_InvokeGC);
    // dirty read - but that ok for garbage collector
    if(!m_InvokeGC || m_TseCount==0) return ;
    if(m_TseCount < m_TseGC_Threshhold) {
        if(m_TseCount < 0.5*m_TseGC_Threshhold)
            m_TseGC_Threshhold = (m_TseCount + 3*m_TseGC_Threshhold)/4;
        return;
    }
    GBLOG_POST( "X_GC " << m_TseCount);
    //GetDataSource()->x_CleanupUnusedEntries();

    CGBLGuard g(m_Locks,"GC");
    x_Check();

    unsigned skip=0;
    // scan 10% of least recently used pile before giving up:
    unsigned skip_max = (int)(0.1*m_TseCount + 1);
    STSEinfo* cur_tse = m_UseListHead;
    while ( cur_tse && skip<skip_max) {
        STSEinfo* tse_to_drop = cur_tse;
        cur_tse = cur_tse->next;
        ++skip;
        // fast checks
        if (tse_to_drop->locked) continue;
        if (tse_to_drop->tseinfop && tse_to_drop->tseinfop->Locked()) continue;
        //CRef<CTSE_Info> tse_info(tse_to_drop->tseinfop);

        if ( !tse_to_drop->tseinfop ) {
            // not loaded yet
            if ( tse_to_drop->m_LoadState != STSEinfo::eLoadStateNone ) {
                // no data
                g.Lock(tse_to_drop);
                GBLOG_POST("X_GC:: drop nonexistent tse " << tse_to_drop);
                CRef<STSEinfo> tse_ref(tse_to_drop);
                x_DropTSEinfo(tse_to_drop);
                g.Unlock(tse_to_drop);
                --skip;
            }
            continue;
        }
        //if(m_Tse2TseInfo.find(sep) == m_Tse2TseInfo.end()) continue;
        
        GBLOG_POST("X_GC::DropTSE("<<TSE(*tse_to_drop)<<")");
        //g.Unlock();
        g.Lock(tse_to_drop);
        if( GetDataSource()->DropTSE(*tse_to_drop->tseinfop) ) {
            --skip;
            m_InvokeGC=false;
        }
        g.Unlock(tse_to_drop);
        //g.Lock();
#if defined(NCBI_THREADS)
        unsigned i=0;
        for(cur_tse = m_UseListHead; cur_tse && i<skip; ++i) {
            cur_tse = cur_tse->next;
        }
#endif
    }
    if(m_InvokeGC) { // nothing has been cleaned up
        //assert(m_TseGC_Threshhold<=m_TseCount); // GC entrance condition
        m_TseGC_Threshhold = m_TseCount+2; // do not even try until next load
    } else if(m_TseCount < 0.5*m_TseGC_Threshhold) {
        m_TseGC_Threshhold = (m_TseCount + m_TseGC_Threshhold)/2;
    }
}


CSeqref::TFlags x_Request2SeqrefMask(CGBDataLoader::EChoice choice)
{
    switch(choice) {
    case CGBDataLoader::eBlob:
    case CGBDataLoader::eBioseq:
    case CGBDataLoader::eAll:
        // whole bioseq
        return CSeqref::fHasAllLocal;
    case CGBDataLoader::eCore:
    case CGBDataLoader::eBioseqCore:
        // everything except bioseqs & annotations
        return CSeqref::fHasCore;
    case CGBDataLoader::eSequence:
        // seq data
        return CSeqref::fHasSeqMap|CSeqref::fHasSeqData;
    case CGBDataLoader::eFeatures:
        // SeqFeatures
        return CSeqref::fHasFeatures;
    case CGBDataLoader::eGraph:
        // SeqGraph
        return CSeqref::fHasGraph;
    case CGBDataLoader::eAlign:
        // SeqGraph
        return CSeqref::fHasAlign;
    case CGBDataLoader::eAnnot:
        // all annotations
        return CSeqref::fHasAlign | CSeqref::fHasGraph |
            CSeqref::fHasFeatures | CSeqref::fHasExternal;
    default:
        return 0;
    }
}


void CGBDataLoader::x_GetRecords(const CSeq_id_Handle& sih,
                                 TMask sr_mask)
{
    int attempt_count = 3, attempt;
    for ( attempt = 0; attempt < attempt_count; ++attempt ) {
        CRef<SSeqrefs> sr = x_ResolveHandle(sih);
        _ASSERT(sr);
        try {
            ITERATE ( SSeqrefs::TSeqrefs, srp, sr->m_Sr ) {
                // skip TSE which doesn't contain requested type of info
                if( ((*srp)->GetFlags() & sr_mask) == 0 )
                    continue;
                
                // find TSE info for each seqref
                TSr2TSEinfo::iterator tsep =
                    m_Sr2TseInfo.find((*srp)->GetKeyByTSE());
                CRef<STSEinfo> tse;
                if (tsep != m_Sr2TseInfo.end()) {
                    tse = tsep->second;
                }
                else {
                    tse.Reset(new STSEinfo());
                    tse->seqref = *srp;
                    tse->key = tse->seqref->GetKeyByTSE();
                    m_Sr2TseInfo[tse->key] = tse;
                    x_AppendToDropList(tse.GetPointer());
                    GBLOG_POST("x_GetRecords-newTSE(" << tse << ")");
                }
                
                CGBLGuard g(m_Locks,CGBLGuard::eMain,"x_GetRecords");