mode_decision.c

JM 11.0 KTA 2.1 Source Code

  int maxindex =  (transform8x8) ? 2 : 5;
  int pix_x, pix_y;
  int block_x, block_y;
  
  int fadjust[16][16], fadjustCr[2][16][16];    
#ifdef ADAPTIVE_FD_SD_CODING
  float adjust_adaptive_f_spatial_domain_4x4=0.0;
  float adjust_adaptive_f_spatial_domain_8x8=0.0;
#endif
  
  int (*fadjustTransform)[16][16] = transform8x8? img->fadjust8x8 : img->fadjust4x4;
  int (*fadjustTransformCr)[2][16][16] = transform8x8? img->fadjust8x8Cr : img->fadjust4x4Cr;
  int lumaAdjustIndex = transform8x8? 2 : 3;
  int chromaAdjustIndex = transform8x8? 0 : 2;

#ifdef BEST_NZ_COEFF
  int best_nz_coeff[2][2];
#endif
  
  //--- set coordinates ---
  j0 = ((block/2)<<3);    j1 = (j0>>2);
  i0 = ((block%2)<<3);    i1 = (i0>>2);
  
#ifdef BEST_NZ_COEFF
  for(j = 0; j <= 1; j++)
  {
    for(i = 0; i <= 1; i++)
      best_nz_coeff[i][j] = img->nz_coeff[img->current_mb_nr][i1 + i][j1 + j] = 0;
  }
#endif
  
  if (transform8x8)
    currMB->luma_transform_size_8x8_flag = 1; //switch to transform size 8x8
  
  //--- store coding state before coding ---
  store_coding_state (cs_cm);
  
  //=====  LOOP OVER POSSIBLE CODING MODES FOR 8x8 SUB-PARTITION  =====
  for (min_cost8x8=INT_MAX, min_rdcost=1e30, index=(bslice?0:1); index<maxindex; index++)
  {
    mode = b8_mode_table[index]; 
    *cost = 0;
    if (enc_mb.valid[mode] && (transform8x8 == 0 || mode != 0 || (mode == 0 && active_sps->direct_8x8_inference_flag)))
    {
      curr_cbp_blk = 0;
      
      if (mode==0)
      {
        //--- Direct Mode ---                                      
        if (!input->rdopt )
        {
          direct4x4_tmp = 0; 
          direct8x8_tmp = 0;
          direct4x4_tmp = Get_Direct_Cost8x8 ( block, &direct8x8_tmp);
          
          if ((direct4x4_tmp==INT_MAX)||(*cost_direct==INT_MAX))
          {
            *cost_direct = INT_MAX;
            if (transform8x8) 
              *cost8x8_direct = INT_MAX;
          }
          else
          {
            *cost_direct += direct4x4_tmp;
            if (transform8x8) 
              *cost8x8_direct += direct8x8_tmp;
          }                     
          (*have_direct)++;
          
          if (transform8x8)
          {
            switch(input->Transform8x8Mode)
            {
            case 1: // Mixture of 8x8 & 4x4 transform
              if((direct8x8_tmp < direct4x4_tmp) || !(enc_mb.valid[5] && enc_mb.valid[6] && enc_mb.valid[7]))
                *cost = direct8x8_tmp;
              else
                *cost = direct4x4_tmp;
              break;
            case 2: // 8x8 Transform only
              *cost = direct8x8_tmp;
              break;
            default: // 4x4 Transform only
              *cost = direct4x4_tmp;
              break;
            }            
            if (input->Transform8x8Mode==2) 
              *cost = INT_MAX;                                       
          }
          else
          {
            *cost = direct4x4_tmp;
          }
        }
        
        block_x = img->block_x+(block&1)*2;
        block_y = img->block_y+(block&2);
        best_ref[LIST_0] = direct_ref_idx[LIST_0][block_y][block_x];
        best_ref[LIST_1] = direct_ref_idx[LIST_1][block_y][block_x];
        best_pdir   = direct_pdir[block_y][block_x];
      } // if (mode==0)
      else
      {        
        //======= motion estimation for all reference frames ========
        //-----------------------------------------------------------
#ifdef RDO_Q
        PartitionMotionSearch (mode, block, enc_mb.lambda_mf,transform8x8);
#else
        PartitionMotionSearch (mode, block, enc_mb.lambda_mf);
#endif   
        //--- get cost and reference frame for LIST 0 prediction ---
        bmcost[LIST_0] = INT_MAX;
        list_prediction_cost(LIST_0, block, mode, enc_mb, bmcost, best_ref);
        
        //store LIST 0 reference index for every block
        block_x = img->block_x+(block&1)*2;
        block_y = img->block_y+(block&2);
        for (j = block_y; j< block_y + 2; j++)
        {
          for (i = block_x; i < block_x + 2; i++)
          {
            enc_picture->ref_idx   [LIST_0][j][i] = best_ref[LIST_0];
            enc_picture->ref_pic_id[LIST_0][j][i] = 
              enc_picture->ref_pic_num[enc_mb.list_offset[LIST_0]][(short)best_ref[LIST_0]];
          }
        } 
        
        if (bslice)
        {
          //--- get cost and reference frame for LIST 1 prediction ---
          bmcost[LIST_1] = INT_MAX;
          bmcost[BI_PRED] = INT_MAX;
          list_prediction_cost(LIST_1, block, mode, enc_mb, bmcost, best_ref);
          
          // Compute bipredictive cost between best list 0 and best list 1 references
          list_prediction_cost(BI_PRED, block, mode, enc_mb, bmcost, best_ref);
          
          //--- get prediction direction ----          
          determine_prediction_list(mode, bmcost, best_ref, &best_pdir, cost, &bi_pred_me);
          
          //store backward reference index for every block
          for (j = block_y; j< block_y + 2; j++)
          {
            for (i = block_x; i < block_x + 2; i++)
            {
              enc_picture->ref_idx[LIST_0][j][i] = best_ref[LIST_0];
              enc_picture->ref_idx[LIST_1][j][i] = best_ref[LIST_1];
            }
          }
        } // if (bslice)
        else
        {
          best_pdir = 0;
          *cost     = bmcost[LIST_0];
        }
      } // if (mode!=0)
      
      if (input->rdopt)
      {
        //--- get and check rate-distortion cost ---
        rdcost = RDCost_for_8x8blocks (&cnt_nonz, &curr_cbp_blk, enc_mb.lambda_md,
          block, mode, best_pdir, best_ref[LIST_0], best_ref[LIST_1]);
      }
      else
      {
        if (*cost!=INT_MAX)
          *cost += (REF_COST (enc_mb.lambda_mf, B8Mode2Value (mode, best_pdir),
          enc_mb.list_offset[(best_pdir<1?LIST_0:LIST_1)]) - 1);
      }
      
      //--- set variables if best mode has changed ---
      if ( ( input->rdopt && rdcost < min_rdcost) 
        || (!input->rdopt && *cost < min_cost8x8))
      {
        min_cost8x8                 = *cost;
        min_rdcost                  = rdcost;
        dataTr->part8x8mode [block] = mode;
        dataTr->part8x8pdir [block] = best_pdir;
        dataTr->part8x8fwref[block] = best_ref[LIST_0];
        dataTr->part8x8bwref[block] = best_ref[LIST_1];
        
        img->mb_data[img->current_mb_nr].b8mode[block] = mode;
        
#ifdef BEST_NZ_COEFF
        for(j = 0; j <= 1; j++)
        {
          for(i = 0; i <= 1; i++)
            best_nz_coeff[i][j]= cnt_nonz ? img->nz_coeff[img->current_mb_nr][i1 + i][j1 + j] : 0;
        }
#endif
        
        //--- store number of nonzero coefficients ---
        best_cnt_nonz  = cnt_nonz;
        
        if (input->rdopt)
        {
          //--- store block cbp ---
          cbp_blk8x8    &= (~(0x33 << (((block>>1)<<3)+((block%2)<<1)))); // delete bits for block
          cbp_blk8x8    |= curr_cbp_blk;
          
          //--- store coefficients ---
          for (k=0; k< 4; k++)
          {
            for (j=0; j< 2; j++)
              for (i=0; i<65; i++)  
                cofACtr[k][j][i] = img->cofAC[block][k][j][i]; // 18->65 for ABT
          }   
          //--- store reconstruction and prediction --- 
          if(!img->residue_transform_flag)
          {
            for (j=j0; j<j0+8; j++)    
            {
#ifdef ADAPTIVE_FD_SD_CODING
              memcpy(&dataTr->quantizer_indices8x8[j][i0],&currMB->quantizer_indices[j][i0],8*sizeof(int));
#endif
              pix_y = img->pix_y + j;
              for (i=i0; i<i0+8; i++)
              {
                pix_x = img->pix_x + i;
                dataTr->rec_mbY8x8[j][i] = enc_picture->imgY[pix_y][pix_x];
                dataTr->mpr8x8[j][i] = img->mpr[j][i];
                if(img->type==SP_SLICE && (!si_frame_indicator))
                  dataTr->lrec[j][i]=lrec[pix_y][pix_x]; // store the coefficients for primary SP slice
              }     
            }
#ifdef ADAPTIVE_FD_SD_CODING
            dataTr->SD_Coding_on_off8x8=currMB->SD_Coding_on_off;
            dataTr->SD_or_FD8x8[j0/8][i0/8]=currMB->SD_or_FD[j0/8][i0/8];
            set_bit(&(dataTr->SD_or_FD_t8x88x8), (j0/8*2+i0/8), (currMB->SD_or_FD_t8x8) & (1<<(j0/8*2+i0/8)));
#endif
          }
          else
          {
            for (j=j0; j<j0+8; j++)                
              for (i=i0; i<i0+8; i++)
              {
                dataTr->rec_resG_8x8  [j][i] = rec_resG  [j][i];
                dataTr->resTrans_R_8x8[j][i] = resTrans_R[j][i];
                dataTr->resTrans_B_8x8[j][i] = resTrans_B[j][i];
                dataTr->mprRGB_8x8 [0][j][i] = mprRGB [0][j][i];
                dataTr->mprRGB_8x8 [1][j][i] = mprRGB [1][j][i];
                dataTr->mprRGB_8x8 [2][j][i] = mprRGB [2][j][i];                  
              }     
          }
        }                  
        if (img->AdaptiveRounding)
        {
#ifdef ADAPTIVE_FD_SD_CODING
          if (transform8x8) adjust_adaptive_f_spatial_domain_8x8=img->adjust_adaptive_f_spatial_domain_8x8;
          else              adjust_adaptive_f_spatial_domain_4x4=img->adjust_adaptive_f_spatial_domain_4x4;          
#endif
          for (j=j0; j<j0+8; j++)
            for (i=i0; i<i0+8; i++)                    
            {
              fadjust     [j][i] = fadjustTransform  [0]   [j][i];
              fadjustCr[0][j][i] = fadjustTransformCr[0][0][j][i];
              fadjustCr[1][j][i] = fadjustTransformCr[0][1][j][i];
            }
        }
        //--- store best 8x8 coding state ---
        if (block < 3)
          store_coding_state (cs_b8);
      } // if (rdcost <= min_rdcost)
      
      //--- re-set coding state as it was before coding with current mode was performed ---
      
      reset_coding_state (cs_cm);
    } // if ((enc_mb.valid[mode] && (transform8x8 == 0 || mode != 0 || (mode == 0 && active_sps->direct_8x8_inference_flag)))
  } // for (min_rdcost=1e30, index=(bslice?0:1); index<6; index++)  
  
#ifdef BEST_NZ_COEFF
  for(j = 0; j <= 1; j++)
  {
    for(i = 0; i <= 1; i++)
      img->nz_coeff[img->current_mb_nr][i1 + i][j1 + j] = best_nz_coeff[i][j];
  }
#endif              
  
  if (!transform8x8)
    dataTr->cost8x8 += min_cost8x8;
  
  if (!input->rdopt)
  {
    if (transform8x8)
    {
      dataTr->cost8x8 += min_cost8x8;              
      mode = dataTr->part8x8mode[block];
      pdir = dataTr->part8x8pdir[block];
    }
    else
    {
      mode = dataTr->part8x8mode[block];
      pdir = dataTr->part8x8pdir[block];    
    }
    curr_cbp_blk  = 0;
    best_cnt_nonz = LumaResidualCoding8x8 (&dummy, &curr_cbp_blk, block, pdir,
      (pdir==0||pdir==2?mode:0), (pdir==1||pdir==2?mode:0), dataTr->part8x8fwref[block], dataTr->part8x8bwref[block]);    
    
    cbp_blk8x8   &= (~(0x33 << (((block>>1)<<3)+((block%2)<<1)))); // delete bits for block
    cbp_blk8x8   |= curr_cbp_blk;
    
    //--- store coefficients ---
    for (k=0; k< 4; k++)
    {
      for (j=0; j< 2; j++)
        memcpy(cofACtr[k][j],img->cofAC[block][k][j],65 * sizeof(int));
    } 
    
    //--- store reconstruction and prediction ---
    if(!img->residue_transform_flag) // Residue Color Transform
    {
      for (j=j0; j<j0+2* BLOCK_SIZE; j++)
      {       
        memcpy(&dataTr->rec_mbY8x8[j][i0], &enc_picture->imgY[img->pix_y + j][img->pix_x + i0], 2* BLOCK_SIZE * sizeof (imgpel));
        memcpy(&dataTr->mpr8x8[j][i0], &img->mpr[j][i0], 2* BLOCK_SIZE * sizeof (imgpel));
        if(img->type==SP_SLICE &&(!si_frame_indicator))
          memcpy(&dataTr->lrec[j][i0],&lrec[img->pix_y+j][img->pix_x+i0],2*BLOCK_SIZE*sizeof(int)); // store coefficients for primary SP slice
      }
#ifdef ADAPTIVE_FD_SD_CODING
      dataTr->SD_Coding_on_off8x8=currMB->SD_Coding_on_off;
      for (j=j0; j<j0+2* BLOCK_SIZE; j++)
      {
        memcpy(&dataTr->quantizer_indices8x8[j][i0],&currMB->quantizer_indices[j][i0],2* BLOCK_SIZE*sizeof(int));
      }
      dataTr->SD_or_FD8x8[j0/8][i0/8]=currMB->SD_or_FD[j0/8][i0/8];
      set_bit(&(dataTr->SD_or_FD_t8x88x8), (j0/8*2+i0/8), (currMB->SD_or_FD_t8x8) & (1<<(j0/8*2+i0/8)));
#endif
    }
    else 
    {   
      for (j=j0; j<j0+8; j++)
        for (i=i0; i<i0+8; i++)
        { 
          dataTr->rec_resG_8x8  [j][i] = rec_resG  [j][i];
          dataTr->resTrans_R_8x8[j][i] = resTrans_R[j][i];
          dataTr->resTrans_B_8x8[j][i] = resTrans_B[j][i];          
          dataTr->mprRGB_8x8 [0][j][i] = mprRGB [0][j][i];
          dataTr->mprRGB_8x8 [1][j][i] = mprRGB [1][j][i];
          dataTr->mprRGB_8x8 [2][j][i] = mprRGB [2][j][i];
        }
    }              
  }
  
  //----- set cbp and count of nonzero coefficients ---
  if (best_cnt_nonz)
  {
    cbp8x8       |= (1 << block);
    cnt_nonz_8x8 += best_cnt_nonz;
  }
  
  if (!transform8x8)
  {
    if (block<3)
    {
      //===== re-set reconstructed block =====
      j0   = 8*(block/2);
      i0   = 8*(block%2);
      for (j=j0; j<j0 + 2 * BLOCK_SIZE; j++)