modedecision.c

Nokia H.264/AVC Encoder/Decoder Usage Manual

/*
COPYRIGHT, LICENSE AND WARRANTY INFORMATION

This software module has been originally developed by Nokia Corporation. 

Provided that a person, entity or a company willing to use the Software 
(hereinafter Licensee) comply with all the terms and conditions of this 
Statement and subject to the limitations set forth in this Statement Nokia 
grants to such Licensee a non-exclusive, sub-licensable, worldwide, 
limited license under copyrights owned by Nokia to use the Software for 
the sole purpose of creating, manufacturing, selling, marketing, or  
distributing (including the right to make modifications to the Software) 
a fully compliant decoder implementation (hereinafter "Decoder") of 
ITU-T Recommendation H.264 / ISO/IEC International Standard 14496-10 and 
an encoder implementation producing output that is decodable with the 
Decoder.

Nokia retains the ownership of copyrights to the Software. There is no 
patent nor other intellectual property right of Nokia licensed under this 
Statement (except the copyright license above). Licensee hereby assumes 
sole responsibility to secure any other intellectual property rights 
needed, if any. For example, if patent licenses  are required, it is 
their responsibility to acquire the license before utilizing the Software.

The license by Nokia is subject to that the Licensee grants to Nokia the 
non-exclusive, worldwide, royalty-free, perpetual and irrevocable 
covenant that the Licensee(s) shall not bring a suit before any court or 
administrative agency or otherwise assert a claim for infringement under 
the Licensee intellectual property rights that, but for a license, would 
be infringed by the Software against  
   (a)  Nokia or Nokia's Affiliate; or  
   (b)  other recipient of a license and covenant not to sue with respect 
        to the Software from Nokia; or 
   (c)  contractor, customer or distributor of a party listed above in a 
        or b,  
which suit or claim is related to the Software or use thereof.

The Licensee(s) further agrees to grant a reciprocal license to Nokia 
(as granted by Nokia to the Licensee(s) on the modifications made by 
Licensee(s) to the Software. 

THE SOFTWARE IS PROVIDED "AS IS" AND THE ORIGINAL DEVELOPER DISCLAIMS 
ANY AND ALL WARRANTIES WHETHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING 
BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS 
FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. THOSE INTENDING TO USE 
THE SOFTWARE ARE EXPRESSLY ADVISED THAT ITS USE MAY INFRINGE EXISTING 
PATENTS AND BE SUBJECT TO ROYALTY PAYMENTS TO PATENT OWNERS. ANYONE 
USING THE SOFTWARE ON THE BASIS OF THIS LICENSE AGREES TO OBTAIN THE 
NECESSARY PERMISSIONS FROM ANY AND ALL APPLICABLE PATENT OWNERS FOR SUCH 
USE.

IN NO EVENT SHALL THE ORIGINAL DEVELOPER BE LIABLE FOR ANY CLAIM, 
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT 
OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 
OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

This copyright, license and warranty information notice must be retained 
in all copies and derivative works of the Software or substantial 
portions thereof.
*/




#include <limits.h>

#include "globals.h"
#include "debug.h"
#include "bitbuffer.h"
#include "macroblock.h"
#include "motest.h"
#include "motcomp.h"
#include "prederrorcod.h"
#include "prederrordec.h"
#include "vlcutility.h"
#include "stream.h"
#include "meutility.h"
#include "transform.h"
#include "intrapred.h"

/*
 * mdBestIntraLuma16x16Plain
 *
 * Parameters:
 *      pMb                   Macroblock information
 *      orig                  Original pixels
 *      predBlk               Storage all 16x16 mode predictions
 *      modeAvail             Which modes are available
 *      bestMbSad             Best MB SAD so far
 *
 * Function:
 *      Build 16x16 predictions and choose the best mode for the macroblock.
 *      The decision is made based on SAD after Hadamard transform.
 *
 * Returns:
 *      SAD for the best 16x16 intra prediction mode.
 *      Best intra16x16 mode is also set in this function.
 */
static int 
mdBestIntraLuma16x16Plain(macroblock_s *pMb, 
                          u_int8 orig[MBK_SIZE][MBK_SIZE],
                          u_int8 predBlk[IPR_NUM_MODES2][MBK_SIZE][MBK_SIZE],
                          int    modeAvail[IPR_NUM_MODES2],
                          int    hadamard,
                          int    bestMbSad)
{
  int mode;
  int currentCost;
  int bestMode;
  int bestCost;
  int exitFlag;
  int i, j;
  int dcCoeffs[4][4];


  bestMode = -1;
  bestCost = MAX_COST;


  for (mode = 0; mode < IPR_NUM_MODES2; mode ++) 
  {

    if (modeAvail[mode]) {

      currentCost = 0;
      exitFlag = 0;

      for (j = 0; (j < MBK_SIZE) && (! exitFlag); j += 4) 
      {
        for (i = 0; (i < MBK_SIZE) && (! exitFlag); i += 4)
        {
          if (hadamard)
            currentCost += traDiffSATD4x4RetDc(
            (u_int8 (*)[MBK_SIZE]) &orig[j][i],
            (u_int8 (*)[MBK_SIZE]) &predBlk[mode][j][i],
            & dcCoeffs[j >> 2][i >> 2]);
          else
            currentCost += traDiffSAD4x4(
            (u_int8 (*)[MBK_SIZE]) &orig[j][i],
            (u_int8 (*)[MBK_SIZE]) &predBlk[mode][j][i]);

          // perform early termination
          exitFlag = ((currentCost >> 1) >= bestMbSad);


        }
      }

      if (! exitFlag)
      {

        // compare with best mode, only if early termination has not happened
        if (hadamard) {
          currentCost += traHadamard4x4(dcCoeffs) >> 2;
          currentCost >>= 1;
        }

        if (currentCost < bestCost ) {
          bestCost  = currentCost;
          bestMode = mode;
        }
      }
    }


  }
  pMb->intra16x16mode = bestMode;

  return bestCost;
}


///// LOW_COMPLEX_PROF3

static int 
mdBestIntraLuma16x16Plain_fast(macroblock_s *pMb, 
                          u_int8 orig[MBK_SIZE][MBK_SIZE],
                          u_int8 predBlk[IPR_NUM_MODES2][MBK_SIZE][MBK_SIZE],
                          int    modeAvail[IPR_NUM_MODES2],
                          int    hadamard,
                          int    bestMbSad)
{
  int mode;
  int currentCost;
  int bestMode;
  int bestCost;
  int exitFlag;
  int i, j;
  int dcCoeffs[4][4];


  int currentCost_thre=MAX_COST;

  bestMode = -1;
  bestCost = MAX_COST;

  for (mode = 0; mode < (IPR_NUM_MODES2-1); mode ++) 

  {

    if (modeAvail[mode]) {

      currentCost = 0;
      exitFlag = 0;

      for (j = 0; (j < MBK_SIZE) && (! exitFlag); j += 4) 
      {
        for (i = 0; (i < MBK_SIZE) && (! exitFlag); i += 4)
        {
          if (hadamard)
            currentCost += traDiffSATD4x4RetDc(
            (u_int8 (*)[MBK_SIZE]) &orig[j][i],
            (u_int8 (*)[MBK_SIZE]) &predBlk[mode][j][i],
            & dcCoeffs[j >> 2][i >> 2]);
          else
            currentCost += traDiffSAD4x4(
            (u_int8 (*)[MBK_SIZE]) &orig[j][i],
            (u_int8 (*)[MBK_SIZE]) &predBlk[mode][j][i]);

          // perform early termination

          exitFlag = (  ((currentCost >> 1) >= bestMbSad) ||
                          (currentCost>currentCost_thre)  );

        }
      }

      if (! exitFlag)
      {
        if (currentCost < currentCost_thre )
          currentCost_thre=currentCost;

        // compare with best mode, only if early termination has not happened
        if (hadamard) {
          currentCost += traHadamard4x4(dcCoeffs) >> 2;
          currentCost >>= 1;
        }

        if (currentCost < bestCost ) {
          bestCost  = currentCost;
          bestMode = mode;
        }
      }
    }


  }
  pMb->intra16x16mode = bestMode;

  return bestCost;
}

///////

/*
 * mdIntraPredLuma16x16:
 *
 * Parameters:
 *      pMb                   Macroblock coding object
 *      picWidth              Horizontal size of the frame
 *      picType               Type of the frame (intra/inter)
 *      bestMbSad             Best MB sad so far
 *      modeDecision          How mode decision should be performed
 *      intra16x16Cost        Cost for the intra4x4 prediction
 *      low_complex_prof3     low complexity profile 3
 *
 * Function:
 *      Build 16x16 predictions and choose the best mode.
 *
 * Returns:
 *      -
 */
void mdIntraPredLuma16x16(macroblock_s  *pMb, 
                          int           picWidth,
                          int           picType,
                          int           bestMbSad,
                          int           modeDecision,
                          costMeasure_s *intra16x16Cost,
                          int           low_complex_prof3)
                            
{
  int modeAvail[IPR_NUM_MODES2];

  if (low_complex_prof3)
    iprGetPredLuma16x16_fast
      (pMb->predBlk2, modeAvail, pMb->recoY, picWidth, pMb->mbAvailMapIntra);
  else
    iprGetPredLuma16x16
      (pMb->predBlk2, modeAvail, pMb->recoY, picWidth, pMb->mbAvailMapIntra);


  // calculate the MSE of intra16x16 and intra4x4
#ifndef DISABLE_RDO
  if ((modeDecision & 0x0F) == MODE_DECISION_RDO)
  {
    int mode, bestMode, startMode, stopMode;
    costMeasure_s current;
    statSlice_s testStat;      // dummy

    pMb->type      = MBK_INTRA;
    pMb->intraType = MBK_INTRA_TYPE2;

    if ((modeDecision & RDO_INTRA_LUMA_16x16) == 0)
    {
      // always try to find the best 16x16 prediction
      if (low_complex_prof3)
        intra16x16Cost->sad = mdBestIntraLuma16x16Plain_fast(pMb, 
          pMb->origY, pMb->predBlk2, modeAvail, 1, MAX_COST);
      else
        intra16x16Cost->sad = mdBestIntraLuma16x16Plain(pMb, 
          pMb->origY, pMb->predBlk2, modeAvail, 1, MAX_COST);

      // only check the best mode just found, to evaluate the RD
      startMode = pMb->intra16x16mode;
      stopMode  = pMb->intra16x16mode;
    }
    else
    {
      startMode = 0;
      stopMode = IPR_NUM_MODES2 - 1;
    }

    // consider the best intra16x16 mode in the context of the entire MB
    // more accurate, but overhead of encoding chroma repeatedly
    bestMode = IPR_MODE2_DC;
    for (mode = startMode; mode <= stopMode; mode ++) 
    {
      if (modeAvail[mode]) 
      {
        pMb->intra16x16mode = mode;

        pecCodeLumaMB(pMb, pMb->predBlk2[mode], 1, picType);
        pedRecoLumaMB(pMb, pMb->predBlk2[mode], 1, picWidth);

        pMb->intra16x16CbpY = pMb->cbpY;

        current.mse = 
          CalculateSsd(& pMb->origY[0][0], 16, pMb->recoY, picWidth, 16, 16);

        // perform test encoding to get the actual rate
        bibInit(& pMb->mbBs);

        // suppose the macroblock type is intra-16x16
        current.bits = streamSendLumaCoeffs(& pMb->mbBs, pMb, & testStat);

        current.cost = current.mse + pMb->rdoLambda * current.bits;
        current.sad = 0;
        current.edgeMse = 0;

        if (current.cost < intra16x16Cost->cost)
        {
          bestMode = mode;
          *intra16x16Cost = current;
        }
      }
    }

    pMb->intra16x16mode = bestMode;

   }
  else
#endif
  {
    int mbType;
    
    // early termination will be applied
    if (low_complex_prof3)
      intra16x16Cost->sad = mdBestIntraLuma16x16Plain_fast(pMb, 
      pMb->origY, pMb->predBlk2, modeAvail, (modeDecision & 0xf) != MODE_DECISION_SIMPLE, bestMbSad);
    else
      intra16x16Cost->sad = mdBestIntraLuma16x16Plain(pMb, 
      pMb->origY, pMb->predBlk2, modeAvail, (modeDecision & 0xf) != MODE_DECISION_SIMPLE, bestMbSad);
    
    //intra16x16Cost->sad holds the SAD between prediction and actual
    pMb->actualIntra16x16Sad = intra16x16Cost->sad;
    
    // at least these many bits will be used for mbType, all CBPs are set to 0
    mbType = ((IS_SLICE_P(picType)) ? 5 : 0) + streamGetMbTypeIntra
      (MBK_INTRA_TYPE2, pMb->intra16x16mode, 0, 0, 0);
    
    
  
    intra16x16Cost->sad += pMb->lambda * uvlcBitsUnsigned[mbType]; 
    intra16x16Cost->cost = intra16x16Cost->sad;

  }

}


/*
 * encoRecoBlock
 *
 * Parameters:
 *      origBlk               Original pixels
 *      predBlk               Prediction block
 *      coeff                 Quantized coefficients
 *      reco                  Which modes are available
 *      recoWidth             Horizontal size reconstructed buffer
 *      picType               Type of the frame (intra/inter)
 *      qp                    Macroblock qp
 *
 * Function:
 *      Perform forward and backward transform/quantization on a 4x4 block.
 *      The reconstructed pixels are put in buffer "reco".
 *
 * Returns:
 *      SAD for the best 16x16 intra prediction mode.
 *      Best intra16x16 mode is also set in this function.
 */
int encoRecoBlock(u_int8 origBlk[][MBK_SIZE], 
                  u_int8 predBlk[][MBK_SIZE], 
                  int    coeff[BLK_SIZE][BLK_SIZE],
                  u_int8 *reco,
                  int    recoWidth, 
                  int    picType,
                  int    qp)
{
  int nonZero;

  /* Code block */
  nonZero = pecCodeBlock(origBlk, predBlk, coeff, 0, qp, picType);

  /* If there were non-zero coefficients, decode block */
  if (nonZero > 0) {
    // this function is never used in intra16x16 mode, so there is no DC
    pedRecoBlock(predBlk, coeff, reco, recoWidth, 0 /* no DC */, 0, qp);
  }
  /* Otherwise, just copy prediction */
  else
  {
    * (int *) & reco[recoWidth * 0] = *(int *) & predBlk[0][0];
    * (int *) & reco[recoWidth * 1] = *(int *) & predBlk[1][0];
    * (int *) & reco[recoWidth * 2] = *(int *) & predBlk[2][0];
    * (int *) & reco[recoWidth * 3] = *(int *) & predBlk[3][0];
  }