modedecision.c

Nokia H.264/AVC Encoder/Decoder Usage Manual

                            refFrmBuf_s    **refBufList,
                            int            nRefFrames, 
                            int            picType, 
                            int            picWidth,
                            int            rdoLambda,
                            int            ssd0,
                            costMeasure_s  *codResults) 
{
  int recordNumSkipped;
  int recordQp;
  int recordPrevQp;
  int channelDistortion;
  statSlice_s testStat;      

  pecCodeLumaMB(pMb, pMb->predY, 0, picType);
  pedRecoLumaMB(pMb, pMb->predY, 0, picWidth);

  codResults->mse = CalculateSsd
    (& pMb->origY[0][0],     16, pMb->recoY, picWidth, 16, 16);

  pecCodeChromaMB(pMb, pMb->predC, picType);
  pedRecoChromaMB(pMb, pMb->predC, picWidth >> 1);

  codResults->mse += CalculateSsd
    (& pMb->origC[0][0],     16, pMb->recoU, picWidth >> 1, 8, 8);
  codResults->mse += CalculateSsd
    (& pMb->origC[0][0] + 8, 16, pMb->recoV, picWidth >> 1, 8, 8);

  // perform test encoding to get the actual rate

  bibInit(& pMb->mbBs);
  pMb->type = MBK_INTER;

  // do not want to change numSkipped and prevQp
  recordNumSkipped = pMb->numSkipped;
  recordQp         = pMb->qp;
  recordPrevQp     = pMb->prevQp;

  codResults->lumaCoeffBits = mbkSend
    (& pMb->mbBs, pMb, nRefFrames, picType, & testStat);

  pMb->numSkipped = recordNumSkipped;
  pMb->qp         = recordQp;
  pMb->prevQp     = recordPrevQp;

  if (pMb->interMode == MOT_COPY)
    pMb->interMode = MOT_16x16;

  codResults->bits = bibGetNumBits(& pMb->mbBs);
  codResults->cost = codResults->mse + rdoLambda * codResults->bits;
  // channel distortion estimation
  channelDistortion = 0;
  if (pMb->plr > 0)
    getChannelDistortion(pMb, refBufList, ssd0, &channelDistortion);

  codResults->cDistortion = channelDistortion;
}

int mdGetCost(macroblock_s *pMb, costMeasure_s *cost,float scaleFactor )
{
  if (pMb->plr == 0)
    return cost->cost;
  else {
    if ( cost->cost == MAX_COST)
      return MAX_COST;
  else
      return (int)(cost->cost + cost->cDistortion*scaleFactor);
  }
}

static int interMbSyntaxBits(macroblock_s *pMb)
{
  int i, mbType, bitsMBtype;

  // estimate the bits on mbType
  mbType = streamGetMbTypeInter(pMb->interMode, pMb->refIndices);

  bitsMBtype = uvlcBitsUnsigned[mbType];
  if (pMb->interMode == MOT_8x8)
  {
    // need to send sub_mb_type
    for (i = 0; i < 4; i++)
      bitsMBtype += uvlcBitsUnsigned[pMb->interSubMbModes[i]];
  }

  return bitsMBtype;
}


void mdInterModeDecision(macroblock_s   *pMb,
                         refFrmBuf_s    **refBufList,
                         int            nRefFrames, 
                         int            picWidth,
                         int            picType, 
                         int            modeDecision,
                         int            ssd0,
                         costMeasure_s  *interCost)
{
  int i, j;
  blkState_s *mvRefPairs;

#ifndef DISABLE_RDO
  if ((modeDecision & 0x0F) == MODE_DECISION_RDO)
  {
    int mode, bestMode, mode16x16Disabled;
    costMeasure_s modeResults;

    interCost->cost = MAX_COST;
    bestMode = MOT_16x16;
    
    // need skipping RDO only if current best MOT_16x16 can not be skipped

    mode = MOT_16x16;
    mode16x16Disabled = 0;

    if (pMb->skipPredMvValid)
    {
      mode = MOT_COPY;

      for (i = 0; i < 16; i ++)
      {
        pMb->modeMvRef[MOT_COPY][i].ref = 0;
        pMb->modeMvRef[MOT_COPY][i].mv = pMb->skipPredMv;
      }

      if ((pMb->modeMvRef[MOT_16x16][0].ref == 0) &&
        (pMb->modeMvRef[MOT_16x16][0].mv.x == pMb->skipPredMv.x) &&
        (pMb->modeMvRef[MOT_16x16][0].mv.y == pMb->skipPredMv.y))
      {
        // this is handled as a skip case
        mode16x16Disabled = 1;
      }
    }

    for ( ; mode <= MOT_8x8; mode ++)
    {
      if (mode != MOT_COPY)
      {
        if (((mode < MOT_8x8) && (pMb->interModeFlags & (1 << (mode - 1))) == 0) ||
          ((mode == MOT_8x8) && (pMb->interModeFlags >> 3) == 0))
          continue;
      }

      if (mode == MOT_16x16 && mode16x16Disabled)
        continue;

      // the MV and refIdx for this mode is found in motion estimation
      mvRefPairs = pMb->modeMvRef[mode];

      // Copy motion vectors and reference index to get the prediction
      for (j = 0; j < BLK_PER_MB;  j ++) 
        for (i = 0; i < BLK_PER_MB;  i ++) 
          pMb->current[j][i] = mvRefPairs[j * 4 + i];

      // get the cost of this particular inter-mode
      pMb->type = MBK_INTER;

      pMb->interMode = mode;
      if (mode == MOT_COPY)
        pMb->interMode = MOT_16x16;

      pMb->interSyntaxBits = mcpFindDiffMvs(pMb, nRefFrames - 1);

      // get the luma and chroma predictors
      mcpGetPred(pMb, refBufList);

      mdInterModeCost
        (pMb, refBufList, nRefFrames, picType, picWidth, pMb->rdoLambda, ssd0, & modeResults);

      // adjust for the skipping cost
      if (mode == MOT_COPY)
      {
        int skipCountCost;

        // this cost is included in a non-skipped MB
        skipCountCost = pMb->rdoLambda * (vlcuUVLCSize(pMb->numSkipped + 2) - 1);

        if (modeResults.bits)
        {
          int skipCost;

          // let's see if it is worthwhile to skip this MB
          skipCost = CalculateSsd
            (& pMb->origY[0][0], 16, & pMb->predY[0][0], 16, 16, 16);

          skipCost += CalculateSsd
            (& pMb->origC[0][0], 16, & pMb->predC[0][0], 16, 8, 8);
          skipCost += CalculateSsd
            (& pMb->origC[0][8], 16, & pMb->predC[0][8], 16, 8, 8);


          if (pMb->plr == 0) {
            if ((skipCost + skipCountCost < modeResults.cost) &&
              (modeResults.lumaCoeffBits == 0))
            {
              // let's skip this
              modeResults.bits = 0;
              modeResults.mse  = skipCost;
              modeResults.cost = modeResults.mse;
            }
          }
          else {
            if ((skipCost + modeResults.cDistortion + skipCountCost < mdGetCost(pMb, &modeResults, 1.0f)) &&
              (modeResults.lumaCoeffBits == 0))
            {
              // let's skip this
              modeResults.bits = 0;
              modeResults.mse  = skipCost;
              modeResults.cost = modeResults.mse;
            }
          }
        }

        if (modeResults.bits == 0) {
          modeResults.cost   += skipCountCost;
        }
      }

      if (mdGetCost(pMb, &modeResults,1.0f) < mdGetCost(pMb, interCost,1.0f)) // modeResults.cost < interCost->cost
      {
        bestMode = mode;
        *interCost = modeResults;
      }
    }

    pMb->interMode = bestMode;
    
  }
  else
#endif
  {
    int channelDistortion;

    channelDistortion = 0;
    if (pMb->plr > 0)
      getChannelDistortion(pMb, refBufList, ssd0, &channelDistortion);

    interCost->cDistortion = (int) channelDistortion;
  }

  // we got the best mode already if not rdo
  mvRefPairs = pMb->modeMvRef[pMb->interMode];

  for (j = 0; j < BLK_PER_MB;  j ++) 
    for (i = 0; i < BLK_PER_MB;  i ++) 
      pMb->current[j][i] = mvRefPairs[j * 4 + i];

  // do we have MOT_COPY as the best mode
  if ((pMb->interMode == MOT_COPY) && (interCost->bits != 0))
    pMb->interMode = MOT_16x16;

  pMb->type = MBK_INTER;

  // differential motion vectors, must be called before interMbSyntaxBits
  pMb->interSyntaxBits = 0;
  if (pMb->interMode != MOT_COPY)
  {
    pMb->interSyntaxBits  = mcpFindDiffMvs(pMb, nRefFrames - 1);
    pMb->interSyntaxBits += interMbSyntaxBits(pMb);
  }


  // possible to separate an MB/sub-MB unnecessarily, because of greedy search.
  // for example, best mode is 16x8, but 2 parts have the same motion vectors. 
  if (pMb->interMode > MOT_16x16)
  {
    if (MergeTree(mvRefPairs, & pMb->interMode))
    {
      // find the differential motion vectors again
      // mcpFindDiffMvs must be called before interMbSyntaxBits
      pMb->interSyntaxBits = mcpFindDiffMvs(pMb, nRefFrames - 1);
      pMb->interSyntaxBits += interMbSyntaxBits(pMb);
      interCost->sad = pMb->actualInterSad + pMb->interSyntaxBits * pMb->lambda;
    }
  }
}


void mdIntraModeDecision(macroblock_s   *pMb,
                         refFrmBuf_s    **refBufList,
                         int            picWidth,
                         int            picType,
                         int            modeDecision,
                         int            ssd0,
                         costMeasure_s  *intra16x16Cost,
                         costMeasure_s  *intra4x4Cost,
                         costMeasure_s  *intraCost)
{
  int mbAddr;
  
  mbAddr = pMb->idxY * picWidth / MBK_SIZE + pMb->idxX ;
  *intraCost = initCostMeasure;

#ifndef DISABLE_RDO
  if ((modeDecision & 0x0F) == MODE_DECISION_RDO)
  {
    // pick better intra mode between intra16x16 and intra4x4, 
    // and the best chroma intra mode at the same time
    int mode, bestChromaMode, startMode, stopMode, intraType, chromaIntraSad;
    costMeasure_s current, chromaCost;
    int chromaModeAvail[IPR_CHROMA_NUM_MODES];

    
    // Get all possible chroma predictors
    iprGetPredChroma(pMb->predIntraC, chromaModeAvail, pMb->recoU, pMb->recoV, 
      picWidth >> 1, pMb->mbAvailMapIntra);

    if ((modeDecision & RDO_INTRA_CHROMA) == 0)
    {
      chromaIntraSad = mdIntraPredChroma(pMb, chromaModeAvail, 1);
      startMode = pMb->intraModeChroma;
      stopMode  = pMb->intraModeChroma;
    }
    else
    {
      startMode = 0;
      stopMode = IPR_CHROMA_NUM_MODES - 1;
    }

    chromaCost = initCostMeasure;

    bestChromaMode = pMb->intraModeChroma;
    pMb->type = MBK_INTRA;
    intraType = MBK_INTRA_TYPE1;

    for (mode = startMode; mode <= stopMode; mode ++) 
    {
      if (chromaModeAvail[mode]) 
      {
        int otherSyntaxBits;
        statSlice_s testStat;      // dummy

        pecCodeChromaMB(pMb, pMb->predIntraC[mode], picType);
        pedRecoChromaMB(pMb, pMb->predIntraC[mode], picWidth >> 1);

        chromaCost.mse = CalculateSsd
          (& pMb->origC[0][0],     16, pMb->recoU, picWidth >> 1, 8, 8);
        chromaCost.mse += CalculateSsd
          (& pMb->origC[0][0] + 8, 16, pMb->recoV, picWidth >> 1, 8, 8);

        // test encoding
        bibInit(& pMb->mbBs);

        chromaCost.bits  = uvlcBitsUnsigned[mode];
        chromaCost.bits += streamSendChromaCoeffs(& pMb->mbBs, pMb, & testStat);

        if (pMb->intra16x16enabled) {
          // --------------------------------------------------------------------
          // evaluate the cost if chroma is encoded with the best intra16x16

          // number of bits used for mbType and cbp
          // intialize some parameters to encode MB in intra4x4
          pMb->intraType = MBK_INTRA_TYPE2;
          pMb->cbpY = pMb->intra16x16CbpY;

          otherSyntaxBits = streamCountIntraSyntaxBits(pMb, picType);

          current.mse  = intra16x16Cost->mse + chromaCost.mse;
          current.bits = otherSyntaxBits + intra16x16Cost->bits + chromaCost.bits;
          current.cost = current.mse + pMb->rdoLambda * current.bits;
         
          if (refBufList[0] && pMb->plr > 0)
            current.cDistortion = pMb->plr * (refBufList[0]->channelDistortion[mbAddr] + ssd0) / 100;
          else
            current.cDistortion = 0;
          current.sad = intra16x16Cost->sad; 
          if ( mdGetCost(pMb, &current,1.0f) < mdGetCost(pMb, intraCost,1.0f) )
          {
            intraType = MBK_INTRA_TYPE2;
            bestChromaMode = mode;
            *intraCost = current;
          }
        }

        // --------------------------------------------------------------------
        // evaluate the cost if chroma is encoded with the best intra4x4
        // number of bits used for mbType and cbp
        // intialize some parameters to encode MB in intra4x4
        pMb->intraType = MBK_INTRA_TYPE1;
        pMb->cbpY = pMb->intra4x4CbpY;    // calculated in intra4x4 prediction

        otherSyntaxBits = streamCountIntraSyntaxBits(pMb, picType);

        current.mse  = intra4x4Cost->mse + chromaCost.mse;
        current.bits = otherSyntaxBits + intra4x4Cost->bits + chromaCost.bits;
        current.cost = current.mse + pMb->rdoLambda * current.bits;
        // the channel distortion is regarded to be 0.0
        if (refBufList[0] && pMb->plr > 0)
          current.cDistortion = pMb->plr * (refBufList[0]->channelDistortion[mbAddr] + ssd0) / 100;
        else
          current.cDistortion = 0;


        current.sad = intra4x4Cost->sad; 
        if ( mdGetCost(pMb, &current,1.0f) < mdGetCost(pMb, intraCost,1.0f) )
        {
          intraType = MBK_INTRA_TYPE1;
          bestChromaMode = mode;
          *intraCost = current;
        }
      }
    }

    pMb->intraType = intraType;
    pMb->intraModeChroma = bestChromaMode;

   }
  else
#endif
  {
    // pick better intra mode between intra16x16 and intra4x4
    if (intra4x4Cost->cost < intra16x16Cost->cost)
    {
      pMb->intraType = MBK_INTRA_TYPE1;
      *intraCost = *intra4x4Cost;
   pMb->actualIntraSad = pMb->actualIntra4x4Sad;
    }
    else {
      pMb->intraType = MBK_INTRA_TYPE2;
      *intraCost = *intra16x16Cost;
    pMb->actualIntraSad = pMb->actualIntra16x16Sad;
    }

    if (refBufList[0] && pMb->plr > 0)
      intraCost->cDistortion = pMb->plr * (refBufList[0]->channelDistortion[mbAddr] + ssd0) / 100;
    else
      intraCost->cDistortion = 0;


    // the chroma mode is decided later when intra/inter decision is made
  }

}