modedecision.c

Nokia H.264/AVC Encoder/Decoder Usage Manual

  return nonZero;
}


/*
 * mdIntraPredLuma4x4:
 *
 * Parameters:
 *      pMb                   Macroblock coding object
 *      picWidth              Horizontal size of the frame
 *      picType               Type of the frame (intra/inter)
 *      bestMbSad             Best MB sad so far
 *      modeDecision          How mode decision should be performed
 *      intra4x4Cost          Cost for the intra4x4 prediction
 *      low_complex_prof3     low complexity prof 3
 *
 * Function:
 *      Build 4x4 predictions and choose the best mode for each block.
 *
 * Returns:
 *      SAD for the best 4x4 intra prediction modes, if RDO is not enabled,
 *      otherwise, RDO cost of encoding luma is returned.
 *      pMb->cbpY is also modified in the function.
 */
int mdIntraPredLuma4x4(macroblock_s   *pMb, 
                       int            picWidth, 
                       int            picType, 
                       int            bestMbSad, 
                       int            modeDecision,
                       costMeasure_s *intra4x4Cost,
                       int            low_complex_prof3)  
{
  u_int8 predBlk[IPR_NUM_MODES1][BLK_SIZE][MBK_SIZE];
  int modeAvail1[IPR_NUM_MODES1];
  int blk8x8, blkIdx;
  int blkCount;
  int mode;
  int bestMode;
  int nonZero, exitFlag;
  int searchLimit;
  int mostProbableMode;
  int mbModeBits;
  int *mbAvailMap;
  int actualBlkSad=0;
  costMeasure_s bestBlkCost, blkCost;

  pMb->cbpY = 0;
  mbAvailMap = pMb->mbAvailMapIntra;

  intra4x4Cost->mse  = 0;
  intra4x4Cost->sad  = 0;
  pMb->actualIntra4x4Sad = 0;
#ifndef DISABLE_RDO
  if (modeDecision == MODE_DECISION_RDO)
  {
    intra4x4Cost->cost = 0;
    searchLimit = MAX_COST;
  }
  else
#endif
  {
    // used for mbType, 5 bits in P-frame, 1 bit in I-frame
    intra4x4Cost->cost = pMb->lambda * (IS_SLICE_P(picType) ? 5 : 1); 
    searchLimit = bestMbSad;
  }



  mbModeBits = 0;
  blkCount = 0;

  exitFlag = 0;
  if (intra4x4Cost->cost + 16 * pMb->lambda >= searchLimit)
  {
    exitFlag = 1;
    intra4x4Cost->cost += 16 * pMb->lambda;
  }



  for (blk8x8 = 0; (blk8x8 < 16) && (! exitFlag); blk8x8 += 4)
  {
    int numNonZeroBlks, nonZeroBlkFlag;
    int blk8x8ModeBits, blk8x8CoeffBits;

    numNonZeroBlks = 0;
    blk8x8CoeffBits = 0;
    blk8x8ModeBits = 0;

    for (blkIdx = blk8x8; (blkIdx < (blk8x8 + 4)) && (! exitFlag); blkIdx ++)
    {
      int upMode, leftMode, blkAddr;
      u_int8 *origBlk;
      u_int8 *recoY;
      int blk4x4BestModeBits;
      int blk4x4BestNumCoeffs;
      int blkX, blkY;

      blkX = blkRasterOrder[blkIdx] & 3;
      blkY = blkRasterOrder[blkIdx] >> 2;
      blkAddr = blkY * 4 + blkX;

      /* Get ipr modes of the two of the neighbouring blocks */
      leftMode = pMb->blkLeft[blkAddr]->i4x4Mode;
      upMode   = pMb->blkUp[blkAddr]->i4x4Mode;

      recoY = & pMb->recoY[blkY * BLK_SIZE * picWidth + blkX*BLK_SIZE];

      /* Make all possible 4x4 predictions for current block */

      iprGetPredLuma4x4(predBlk, modeAvail1, recoY, 
        picWidth, leftMode, upMode, pMb->i4x4CornersAvail[blkY][blkX]);


      // at least one is IPR_MODE_NA
      if ((upMode + leftMode) >=  IPR_MODE_NA)
        mostProbableMode = IPR_MODE_DC;
      else
        mostProbableMode = min(upMode, leftMode);

      bestMode = 0;
      blk4x4BestModeBits = 0;
      blk4x4BestNumCoeffs = 0;

      origBlk = & pMb->origY[blkY * BLK_SIZE][blkX * BLK_SIZE];

      nonZeroBlkFlag = 0;
      bestBlkCost = initCostMeasure;
      blkCost     = initCostMeasure;

      /* Choose the best mode for the current block */
      
      for (mode = 0; mode < IPR_NUM_MODES1; mode++) {
        

        if (modeAvail1[mode]) {
          int numCoefs;

          blkCost.sad = pMb->lambda * (mode == mostProbableMode ? 1 : 4);

          if (modeDecision == MODE_DECISION_HADAMARD)
            blkCost.sad += traDiffSATD4x4(
            (u_int8 (*)[MBK_SIZE]) origBlk,
            (u_int8 (*)[MBK_SIZE]) predBlk[mode]) >> 1;
          else
            blkCost.sad += traDiffSAD4x4(
            (u_int8 (*)[MBK_SIZE]) origBlk,
            (u_int8 (*)[MBK_SIZE]) predBlk[mode]);

          nonZero  = 1;
          numCoefs = 0;

          /* Compute total SAD for the current mode by adding the start */
          /* 'handicap' and the computed SAD                            */

#ifndef DISABLE_RDO
          if (modeDecision == MODE_DECISION_RDO)
          {
            
            nonZero = encoRecoBlock((u_int8 (*)[MBK_SIZE]) origBlk, predBlk[mode], 
              pMb->coefY[blkIdx], recoY, picWidth, picType, pMb->qp);

            /* If there were non-zero coefficients, decode block */
            // perform entropy encoding
            bibInit(& pMb->mbBs);

            blkCost.bits = streamSend4x4Blk(& pMb->mbBs, pMb, 1,
              COMP_LUMA, blkX, blkY,
              pMb->coefY[blkIdx], BLK_CAT_Y, & numCoefs);

            blkCost.bits += (mode == mostProbableMode) ? 1 : 4;
            blkCost.mse   = CalculateSsd
              (origBlk, MBK_SIZE, recoY, picWidth, BLK_SIZE, BLK_SIZE);

            // calculate the MSE
            blkCost.cost = blkCost.mse + pMb->rdoLambda * blkCost.bits;

          }
          else
#endif
          blkCost.cost = blkCost.sad;

          if (blkCost.cost < bestBlkCost.cost) {
            bestBlkCost   = blkCost;
            bestMode      = mode;
            blk4x4BestNumCoeffs = numCoefs;
            actualBlkSad = blkCost.sad - pMb->lambda * (mode == mostProbableMode ? 1 : 4); 
            blk4x4BestModeBits  = (mode == mostProbableMode) ? 1 : 4; 
            nonZeroBlkFlag = nonZero;
          }

       }

        // early termination in for mode

        if ( (low_complex_prof3) && (bestBlkCost.cost < TH_IPR_MODE  ) )
        {
          break;
        }

      }


      // store number of coeffs of the best mode, for future prediction
      pMb->current[blkY][blkX].i4x4Mode = (int8) bestMode;
      pMb->current[blkY][blkX].numLumaCoefs = 
        (int8) blk4x4BestNumCoeffs;

      // this is the information related i4x4Mode to be stored in stream
      pMb->ipTab[blkIdx] = (int8) ((bestMode == mostProbableMode) ?
        -1 : ((bestMode < mostProbableMode) ? bestMode : bestMode - 1));

      numNonZeroBlks  += nonZeroBlkFlag;
      blk8x8CoeffBits += bestBlkCost.bits - blk4x4BestModeBits;  //will be zero for non-RDO
      blk8x8ModeBits  += blk4x4BestModeBits; 
      blkCount ++;

      intra4x4Cost->cost += bestBlkCost.cost;
      intra4x4Cost->sad  += bestBlkCost.sad;
      intra4x4Cost->mse  += bestBlkCost.mse;
      pMb->actualIntra4x4Sad+=actualBlkSad;

      // a little bit look ahead to have earlier termination
      if ((intra4x4Cost->cost + pMb->lambda * (16 - blkCount)) >= searchLimit)
      {
        // at least this amount
        intra4x4Cost->cost += pMb->lambda * (16 - blkCount);
        exitFlag = 1;
        continue;
      }

      nonZero = encoRecoBlock((u_int8 (*)[MBK_SIZE]) origBlk, predBlk[bestMode], 
        pMb->coefY[blkIdx], recoY, picWidth, picType, pMb->qp);

      /* If there were non-zero coefficients, decode block */
      if (nonZero > 0)
        /* Update luma CBP */
        pMb->cbpY |= 1 << blkIdx;

    }

    mbModeBits += blk8x8ModeBits; 
    if (numNonZeroBlks)
      intra4x4Cost->bits += blk8x8CoeffBits;
    else
      intra4x4Cost->bits += 2;      // do not want to be too biased
  }

  intra4x4Cost->bits += mbModeBits;


  // we re-calculate the sad here
#ifndef DISABLE_RDO
  if (modeDecision == MODE_DECISION_RDO) {
    pMb->intra4x4CbpY = pMb->cbpY;
    intra4x4Cost->cost = intra4x4Cost->mse + intra4x4Cost->bits * pMb->rdoLambda;
  }
  else
#endif
  {
    // sad is used as cost above
    intra4x4Cost->sad  = intra4x4Cost->cost;
  }


  return intra4x4Cost->cost;
}


/*
 *
 * mdIntraPredChroma:
 *
 * Parameters:
 *      pMb                   macroblock information
 *      orig                  Original pixels
 *      pred                  Return pointer for predicted pixels
 *
 * Function:
 *      Build 8x8 chroma predictions and choose the best mode for the
 *      macroblock.
 *
 * Returns:
 *      SAD of the best chroma intra mode
 *
 */
int mdIntraPredChroma(macroblock_s *pMb, 
                      int chromaModeAvail[IPR_CHROMA_NUM_MODES],
                      int hadamard)
{
  int bestMode;
  int bestCost;
  int mode;
  int currentCost;
  int c;
  int i, j;
  int dcCoeffs[2][2];

  /*
   * Build 8x8 intra prediction
   */

  bestMode = -1;
  bestCost  = INT_MAX;

  for (mode = 0; mode < IPR_CHROMA_NUM_MODES; mode++) 
  {
    if (chromaModeAvail[mode]) 
    {
      currentCost = 0;
      for (c = 0; c < MBK_SIZE; c += MBK_SIZE/2) {
        for (j = 0; j < BLK_PER_MB/2; j ++) {
          for (i = 0; i < BLK_PER_MB/2; i ++)
          {
            if (hadamard)
              currentCost += traDiffSATD4x4RetDc(
              (u_int8 (*)[MBK_SIZE]) &pMb->origC[j*BLK_SIZE][c + i*BLK_SIZE],
              (u_int8 (*)[MBK_SIZE]) &pMb->predIntraC[mode][j*BLK_SIZE][c + i*BLK_SIZE],
              & dcCoeffs[j][i]);
            else
              currentCost += traDiffSAD4x4(
              (u_int8 (*)[MBK_SIZE]) &pMb->origC[j*BLK_SIZE][c + i*BLK_SIZE],
              (u_int8 (*)[MBK_SIZE]) &pMb->predIntraC[mode][j*BLK_SIZE][c + i*BLK_SIZE]);
          }
        }

        if (hadamard) {
          traDCT2x2(dcCoeffs);

          // DC (>> 2), 2x2 Hadamard (<< 1) to match 4x4 Hadamard, end up (>> 1)
          currentCost += (abs(dcCoeffs[0][0]) + abs(dcCoeffs[0][1]) +
            abs(dcCoeffs[1][0]) + abs(dcCoeffs[1][1])) >> 1;
        }
      }

      if (hadamard)
        currentCost = currentCost >> 1;

      currentCost = currentCost + uvlcBitsUnsigned[mode] * pMb->lambda;
      if (currentCost < bestCost ) {
        bestCost = currentCost;
        bestMode = mode;
      }
    }
  }

  pMb->intraModeChroma = bestMode;

  return bestCost;
}

extern int8 mcpModePartWidth[5];
extern int8 mcpModePartHeight[5];

static void getChannelDistortion(macroblock_s *pMb,
                               refFrmBuf_s  **refBufList,
                               int          ssd0,
                               int          *distortion)
{
  int blkIdxX, blkIdxY;
  int shapeX, shapeY, mbPartW, mbPartH;
  int i0, j0;
  int i1, j1;
  int i2, j2;
  int k0, l0;
  int picWidth;
  int picHeight;
  int mbIdxRef;
  int mbkPerLine;
  int mbAddr;
  int *pDistortion;

  distortion[0] = 0;

  picWidth = refBufList[0]->picWidth;
  picHeight = refBufList[0]->picHeight;
  mbkPerLine = picWidth / MBK_SIZE;
  mbAddr = pMb->idxY * picWidth / MBK_SIZE + pMb->idxX ;

  shapeX = mcpModePartWidth[pMb->interMode];
  shapeY = mcpModePartHeight[pMb->interMode];
  mbPartW = shapeX * BLK_SIZE;
  mbPartH = shapeY * BLK_SIZE;

  // 1:    (1-p) * Dc(n-1, j)
  for (blkIdxY = 0; blkIdxY < BLK_PER_MB; blkIdxY += shapeY) {
    for (blkIdxX = 0; blkIdxX < BLK_PER_MB; blkIdxX += shapeX) {
      blkState_s *pBlkState;

      // the starting position of current block in pixel: k0, l0
      k0 = (pMb->idxX * BLK_PER_MB + blkIdxX) * BLK_SIZE;
      l0 = (pMb->idxY * BLK_PER_MB + blkIdxY) * BLK_SIZE;

      // Absolute motion vector coordinates of the macroblock
      pBlkState = & pMb->current[blkIdxY][blkIdxX];
      pDistortion = refBufList[pBlkState->ref]->channelDistortion;

      i0 = k0 * 4 + pBlkState->mv.x;
      j0 = l0 * 4 + pBlkState->mv.y;

      // the starting position of the ref block in pixel: i0, j0
      i0 = i0 / 4;
      j0 = j0 / 4;
      if (i0 < 0) i0 = 0;
      if (j0 < 0) j0 = 0;
      if (i0 >= picWidth) i0 = picWidth - 1;
      if (j0 >= picHeight) j0 = picHeight - 1;

      // calculate the distortion here:
      for (j1 = j0; j1 < j0 + mbPartH; j1++) {
        for (i1 = i0; i1 < i0 + mbPartW; i1++) {
          i2 = i1;
          j2 = j1;

          if (i2 >= picWidth) i2 = picWidth - 1;
          if (j2 >= picHeight) j2 = picHeight - 1;

          mbIdxRef  = (j2 / MBK_SIZE) * mbkPerLine + i2 / MBK_SIZE;
          distortion[0] += pDistortion[mbIdxRef];  //  / 256.0
        }
      }

    }
  }
  distortion[0] = distortion[0] >> 8; //  / 256
  distortion[0] = distortion[0] * (100-pMb->plr);

  // 2:    p * Dc(n-1, i)
  distortion[0] += refBufList[0]->channelDistortion[mbAddr] * pMb->plr;

  // 3:    p * ssd
  distortion[0] += ssd0 * pMb->plr;
  distortion[0] = distortion[0] / 100;
}
static void mdInterModeCost(macroblock_s   *pMb,