mode_decision.c

H264视频编解码程序

        if (enc_mb.valid[mode] && !inter_skip)
        {
          for (cost=0, block=0; block<(mode==1?1:2); block++)
          {
            PartitionMotionSearch (mode, block, enc_mb.lambda_mf);
            //--- set 4x4 block indizes (for getting MV) ---
            j = (block==1 && mode==2 ? 2 : 0);
            i = (block==1 && mode==3 ? 2 : 0);
            
            //--- get cost and reference frame for List 0 prediction ---
            bmcost[LIST_0] = INT_MAX;
            list_prediction_cost(LIST_0, block, mode, enc_mb, bmcost, best_ref);
            
            if (bslice)
            {
              //--- get cost and reference frame for List 1 prediction ---
              bmcost[LIST_1] = INT_MAX;
              list_prediction_cost(LIST_1, block, mode, enc_mb, bmcost, best_ref);
              
              // Compute bipredictive cost between best list 0 and best list 1 references
              list_prediction_cost(BI_PRED, block, mode, enc_mb, bmcost, best_ref);
              
              // Finally, if mode 16x16, compute cost for bipredictive ME vectore
              if (input->BiPredMotionEstimation && mode == 1)
              {                
                list_prediction_cost(BI_PRED_L0, block, mode, enc_mb, bmcost, 0);
                list_prediction_cost(BI_PRED_L1, block, mode, enc_mb, bmcost, 0);
              }
              else
              {
                bmcost[BI_PRED_L0] = INT_MAX;
                bmcost[BI_PRED_L1] = INT_MAX;
              }
              
              // Determine prediction list based on mode cost
              determine_prediction_list(mode, bmcost, best_ref, &best_pdir, &cost, &bi_pred_me);
            }
            else // if (bslice)
            {
              best_pdir  = 0;
              cost      += bmcost[LIST_0];
            }
            
            assign_enc_picture_params(mode, best_pdir, block, enc_mb.list_offset[LIST_0], best_ref[LIST_0], best_ref[LIST_1], bslice);
            
            //----- set reference frame and direction parameters -----
            if (mode==3)
            {
              best8x8fwref [3][block  ] = best8x8fwref [3][  block+2] = best_ref[LIST_0];
              best8x8pdir  [3][block  ] = best8x8pdir  [3][  block+2] = best_pdir;
              best8x8bwref [3][block  ] = best8x8bwref [3][  block+2] = best_ref[LIST_1];
            }
            else if (mode==2)
            {
              best8x8fwref [2][2*block] = best8x8fwref [2][2*block+1] = best_ref[LIST_0];
              best8x8pdir  [2][2*block] = best8x8pdir  [2][2*block+1] = best_pdir;
              best8x8bwref [2][2*block] = best8x8bwref [2][2*block+1] = best_ref[LIST_1];
            }
            else
            {
              best8x8fwref [1][0] = best8x8fwref [1][1] = best8x8fwref [1][2] = best8x8fwref [1][3] = best_ref[LIST_0];
              best8x8pdir  [1][0] = best8x8pdir  [1][1] = best8x8pdir  [1][2] = best8x8pdir  [1][3] = best_pdir;
              best8x8bwref [1][0] = best8x8bwref [1][1] = best8x8bwref [1][2] = best8x8bwref [1][3] = best_ref[LIST_1];
            }
            
            //--- set reference frames and motion vectors ---
            if (mode>1 && block==0)
              SetRefAndMotionVectors (block, mode, best_pdir, best_ref[LIST_0], best_ref[LIST_1]);            
          } // for (block=0; block<(mode==1?1:2); block++)
          
          if(!input->rdopt)
          {
            currMB->luma_transform_size_8x8_flag = 0;
            if (input->Transform8x8Mode) //for inter rd-off, set 8x8 to do 8x8 transform
            {
              SetModesAndRefframeForBlocks(mode);
              currMB->luma_transform_size_8x8_flag = TransformDecision(-1, &cost);
            }
          }          
          
          if(input->rdopt == 2 && mode == 1)
          {
            if(pslice)
              min_rdcost = max_rdcost;
            
            //=====   S T O R E   C O D I N G   S T A T E   =====
            //---------------------------------------------------
            //store_coding_state (cs_cm);

            for (ctr16x16=0, k=0; k<1; k++)
            {
              i16mode = 0; 
              
              //--- for INTER16x16 check all prediction directions ---
              if (bslice)
              {
                best8x8pdir[1][0] = best8x8pdir[1][1] = best8x8pdir[1][2] = best8x8pdir[1][3] = ctr16x16;
                
                if ( (bslice) && (input->BiPredMotionEstimation) 
                  && (ctr16x16 == 2 && img->bi_pred_me[mode] < 2 && mode == 1))
                  ctr16x16--;
                if (ctr16x16 < 2) 
                  index--;
                ctr16x16++;
              }
              
              currMB->c_ipred_mode=DC_PRED_8;
              compute_mode_RD_cost(mode, currMB, enc_mb, &min_rdcost, &min_rate, i16mode, bslice, &inter_skip);

              if ((input->BiPredMotionEstimation) && (bslice) && ctr16x16 == 2 
                && img->bi_pred_me[mode] < 2 && mode == 1 && best8x8pdir[1][0] == 2) 
                img->bi_pred_me[mode] = img->bi_pred_me[mode] + 1;
            } // for (ctr16x16=0, k=0; k<1; k++)
            
            if(pslice)
            {
              // Get SKIP motion vector and compare SKIP_MV with best motion vector of 16x16
              FindSkipModeMotionVector ();
              if(input->EarlySkipEnable)
              {
                //===== check for SKIP mode =====
                if ( currMB->cbp==0 && enc_picture->ref_idx[LIST_0][img->block_y][img->block_x]==0 &&
                  enc_picture->mv[LIST_0][img->block_y][img->block_x][0]==allmvs[0] &&
                  enc_picture->mv[LIST_0][img->block_y][img->block_x][1]==allmvs[1]               )
                {
                  inter_skip = 1;
                  best_mode = 0;
                }
              } // if(input->EarlySkipEnable)
            }
            
            // store variables.
            RDCost16 = min_rdcost;
            mode16 = best_mode;
            cost16 = cost;
          } // if(input->rdopt == 2 && mode == 1)
          
          if ((!inter_skip) && (cost < min_cost))
          {
            best_mode = mode;
            min_cost  = cost;
            best_transform_flag = currMB->luma_transform_size_8x8_flag;
          }
        } // if (enc_mb.valid[mode])
      } // for (mode=1; mode<4; mode++)
      
      if ((!inter_skip) && enc_mb.valid[P8x8])
      {
        giRDOpt_B8OnlyFlag = 1;
		
        tr8x8.cost8x8 = INT_MAX;
        tr4x4.cost8x8 = INT_MAX;
        //===== store coding state of macroblock =====
        store_coding_state (cs_mb);
        
        currMB->all_blk_8x8 = -1;
        
        if (input->Transform8x8Mode)
        {  
          tr8x8.cost8x8 = 0;
          //===========================================================
          // Check 8x8 partition with transform size 8x8 
          //===========================================================
          //=====  LOOP OVER 8x8 SUB-PARTITIONS  (Motion Estimation & Mode Decision) =====
          for (cost_direct=cbp8x8=cbp_blk8x8=cnt_nonz_8x8=0, block=0; block<4; block++)
          {
            submacroblock_mode_decision(enc_mb, &tr8x8, currMB, cofAC_8x8ts[block],
            &have_direct, bslice, block, &cost_direct, &cost, &cost8x8_direct, 1);
            best8x8mode       [block] = tr8x8.part8x8mode [block];
            best8x8pdir [P8x8][block] = tr8x8.part8x8pdir [block];
            best8x8fwref[P8x8][block] = tr8x8.part8x8fwref[block];
            best8x8bwref[P8x8][block] = tr8x8.part8x8bwref[block];
          }
          
          // following params could be added in RD_8x8DATA structure
          cbp8_8x8ts      = cbp8x8;
          cbp_blk8_8x8ts  = cbp_blk8x8;
          cnt_nonz8_8x8ts = cnt_nonz_8x8;
          currMB->luma_transform_size_8x8_flag = 0; //switch to 4x4 transform size

          //--- re-set coding state (as it was before 8x8 block coding) ---
          //reset_coding_state (cs_mb);        
        }// if (input->Transform8x8Mode)
        

        if (input->Transform8x8Mode != 2)  
        {
          tr4x4.cost8x8 = 0;
          //=================================================================
          // Check 8x8, 8x4, 4x8 and 4x4 partitions with transform size 4x4
          //=================================================================
          //=====  LOOP OVER 8x8 SUB-PARTITIONS  (Motion Estimation & Mode Decision) =====
          for (cost_direct=cbp8x8=cbp_blk8x8=cnt_nonz_8x8=0, block=0; block<4; block++)
          {
            submacroblock_mode_decision(enc_mb, &tr4x4, currMB, cofAC8x8[block],
              &have_direct, bslice, block, &cost_direct, &cost, &cost8x8_direct, 0);
            
            best8x8mode       [block] = tr4x4.part8x8mode [block];
            best8x8pdir [P8x8][block] = tr4x4.part8x8pdir [block];
            best8x8fwref[P8x8][block] = tr4x4.part8x8fwref[block];
            best8x8bwref[P8x8][block] = tr4x4.part8x8bwref[block];
          }          
          //--- re-set coding state (as it was before 8x8 block coding) ---
          // reset_coding_state (cs_mb);  
        }// if (input->Transform8x8Mode != 2)
        
        //--- re-set coding state (as it was before 8x8 block coding) ---
        reset_coding_state (cs_mb);
        
        
        // This is not enabled yet since mpr has reverse order.
        if (input->RCEnable)
          rc_store_diff(img->opix_x,img->opix_y,img->mpr);
        
        //check cost for P8x8 for non-rdopt mode
        if (!input->rdopt && (tr4x4.cost8x8 < min_cost || tr8x8.cost8x8 < min_cost))
        {
          best_mode = P8x8;
          if (input->Transform8x8Mode == 2)
          {
              min_cost = tr8x8.cost8x8;
              currMB->luma_transform_size_8x8_flag=1;
          }
          else if (input->Transform8x8Mode)
          {
            if (tr8x8.cost8x8 < tr4x4.cost8x8)
            {
              min_cost = tr8x8.cost8x8;
              currMB->luma_transform_size_8x8_flag=1;
            }
            else if(tr4x4.cost8x8 < tr8x8.cost8x8)
            {
              min_cost = tr4x4.cost8x8;
              currMB->luma_transform_size_8x8_flag=0;
            }
            else
            {
              if (GetBestTransformP8x8() == 0)
              {
                min_cost = tr4x4.cost8x8;
                currMB->luma_transform_size_8x8_flag=0;
              }
              else
              {
                min_cost = tr8x8.cost8x8;
                currMB->luma_transform_size_8x8_flag=1;
              }
            }
          }
          else
          {
            min_cost = tr4x4.cost8x8;
            currMB->luma_transform_size_8x8_flag=0;
          }
        }// if (!input->rdopt && (tr4x4.cost8x8 < min_cost || tr8x8.cost8x8 < min_cost))
        giRDOpt_B8OnlyFlag = 0;
      }
      else // if (enc_mb.valid[P8x8])
      {
        tr4x4.cost8x8 = INT_MAX;
      }
      
      // Find a motion vector for the Skip mode
      if(input->rdopt != 2 && pslice)
        FindSkipModeMotionVector ();
    }   
    else // if (!intra)
    {
      min_cost = INT_MAX;
    }
    
    //========= C H O O S E   B E S T   M A C R O B L O C K   M O D E =========
    //-------------------------------------------------------------------------
   if (input->rdopt)
   {
     // store_coding_state (cs_cm);    
    if (!inter_skip)
    {
        int mb_available_up;
        int mb_available_left;
        int mb_available_up_left;
        
        if(input->rdopt == 2 && img->type!=I_SLICE)
        {
          min_rdcost = RDCost16;
          best_mode  = mode16;
        }
        else
          min_rdcost = max_rdcost;
        
        // if Fast High mode, compute  inter modes separate process for inter/intra
        max_index = (img->residue_transform_flag) ? 11 : ((input->rdopt == 2 && !intra && input->SelectiveIntraEnable ) ? 5 : 9);
        
        if (input->BiPredMotionEstimation)
          img->bi_pred_me[1] =0;  
        
        if (img->yuv_format != YUV400 && max_index != 5)
        {
          // precompute all new chroma intra prediction modes
          IntraChromaPrediction(&mb_available_up, &mb_available_left, &mb_available_up_left);
          max_chroma_pred_mode = PLANE_8;
        }
        else
          max_chroma_pred_mode = DC_PRED_8;
        
        for (currMB->c_ipred_mode=DC_PRED_8; currMB->c_ipred_mode<=max_chroma_pred_mode; currMB->c_ipred_mode++)
        {
          // bypass if c_ipred_mode is not allowed
          if ( (img->yuv_format != YUV400) &&
            (  ((!intra || !input->IntraDisableInterOnly) && input->ChromaIntraDisable == 1 && currMB->c_ipred_mode!=DC_PRED_8) 
            || (currMB->c_ipred_mode == VERT_PRED_8 && !mb_available_up) 
            || (currMB->c_ipred_mode == HOR_PRED_8  && !mb_available_left) 
            || (currMB->c_ipred_mode == PLANE_8     && (!mb_available_left || !mb_available_up || !mb_available_up_left))))
            continue;        
          
          //===== GET BEST MACROBLOCK MODE =====
          for (ctr16x16=0, index=0; index < max_index; index++)
          {
            mode = mb_mode_table[index];
            
            if (img->yuv_format != YUV400)
            {           
              if (input->rdopt == 2)
              {
                i16mode = 0;              
                // RDcost of mode 1 in P-slice and mode 0, 1 in B-slice are already available
                if(((bslice && mode == 0) || (!islice && mode == 1)))
                  continue;
              }
              else
              {
                // Residue Color Transform
                if(img->residue_transform_flag)
                {
                  mode = mb_mode_table_RCT[index];
                  if( mode == I16MB) 
                    i16mode = index -5;
                  // bypass if i16mode is not allowed
                  if (mode == I16MB &&
                    (  (i16mode==VERT_PRED_16 && !mb_available_up) 
                    || (i16mode==HOR_PRED_16  && !mb_available_left) 
                    || (i16mode==PLANE_16    && (!mb_available_left || !mb_available_up || !mb_available_up_left))))
                    continue;
                }
                else
                {
                  mode = mb_mode_table[index];
                  i16mode = 0; 
                }
              }
            }
            //--- for INTER16x16 check all prediction directions ---
            if (mode==1 && bslice)
            {
              best8x8pdir[1][0] = best8x8pdir[1][1] = best8x8pdir[1][2] = best8x8pdir[1][3] = ctr16x16;
              
              if ( (bslice) && (input->BiPredMotionEstimation) 
                && (ctr16x16 == 2 && img->bi_pred_me[mode] < 2 && mode == 1))