loopfilter.c

This program can encode the YUV vdieo format to H.264 and decode it.

                  (iabs( list0_mv[blk_y][blk_x][1] - list0_mv[blk_y2][blk_x2][1]) >= mvlimit ) |
                  (iabs( list1_mv[blk_y][blk_x][0] - list1_mv[blk_y2][blk_x2][0]) >= 4) |
                  (iabs( list1_mv[blk_y][blk_x][1] - list1_mv[blk_y2][blk_x2][1]) >= mvlimit))
                  &&
                  ((iabs( list0_mv[blk_y][blk_x][0] - list1_mv[blk_y2][blk_x2][0]) >= 4) |
                  (iabs( list0_mv[blk_y][blk_x][1] - list1_mv[blk_y2][blk_x2][1]) >= mvlimit) |
                  (iabs( list1_mv[blk_y][blk_x][0] - list0_mv[blk_y2][blk_x2][0]) >= 4) |
                  (iabs( list1_mv[blk_y][blk_x][1] - list0_mv[blk_y2][blk_x2][1]) >= mvlimit)));
              }
            }
            else
            {
              StrValue = 1;
            }
          }        
        }
        memset(&Strength[idx], (byte) StrValue, BLOCK_SIZE * sizeof(byte));
      }
    }
    else
    {
      // Start with Strength=3. or Strength=4 for Mb-edge
      StrValue = (edge == 0 && ((((img->structure==FRAME))) || ((img->structure != FRAME) && !dir))) ? 4 : 3;
      memset(&Strength[0], (byte) StrValue, MB_BLOCK_SIZE * sizeof(byte));
    }      
  }
}

/*!
 *********************************************************************************************
 * \brief
 *    returns a buffer of 16 Strength values for one stripe in a mb (for MBAFF)
 *********************************************************************************************
 */
void GetStrengthMBAff(byte Strength[16],ImageParameters *img,int MbQAddr,int dir,int edge, int mvlimit)
{
  int    blkP, blkQ, idx;
  int    blk_x, blk_x2, blk_y, blk_y2 ;
  short  ***list0_mv = enc_picture->mv[LIST_0];
  short  ***list1_mv = enc_picture->mv[LIST_1];
  char   **list0_refIdxArr = enc_picture->ref_idx[LIST_0];
  char   **list1_refIdxArr = enc_picture->ref_idx[LIST_1];
  int64  **list0_refPicIdArr = enc_picture->ref_pic_id[LIST_0];
  int64  **list1_refPicIdArr = enc_picture->ref_pic_id[LIST_1];
  int64    ref_p0,ref_p1,ref_q0,ref_q1;
  int      xQ, yQ;
  int      mb_x, mb_y;
  Macroblock *MbQ = &(img->mb_data[MbQAddr]);
  Macroblock *MbP;
  PixelPos pixP;
  int dir_m1 = (1 - dir);

  for( idx=0 ; idx<16 ; idx++ )
  {
    xQ = dir ? idx : edge;
    yQ = dir ? (edge < MB_BLOCK_SIZE ? edge : 1) : idx;
    getNeighbour(MbQAddr, xQ - dir_m1, yQ - dir, IS_LUMA, &pixP);
    blkQ = ((yQ >> 2) << 2) + (xQ >> 2);
    blkP = ((pixP.y >> 2)<<2) + (pixP.x >> 2);

    MbP = &(img->mb_data[pixP.mb_addr]);
    mixedModeEdgeFlag = (byte) (MbQ->mb_field != MbP->mb_field);   

    if ((img->type==SP_SLICE)||(img->type==SI_SLICE) )
    {
      Strength[idx] = (edge == 0 && (((!img->MbaffFrameFlag && (img->structure==FRAME)) ||
      (img->MbaffFrameFlag && !MbP->mb_field && !MbQ->mb_field)) ||
      ((img->MbaffFrameFlag || (img->structure != FRAME)) && !dir))) ? 4 : 3;
    }
    else
    {
      // Start with Strength=3. or Strength=4 for Mb-edge
      Strength[idx] = (edge == 0 && (((!img->MbaffFrameFlag && (img->structure==FRAME)) ||
        (img->MbaffFrameFlag && !MbP->mb_field && !MbQ->mb_field)) ||
        ((img->MbaffFrameFlag || (img->structure!=FRAME)) && !dir))) ? 4 : 3;

      if(  !(MbP->mb_type==I4MB || MbP->mb_type==I16MB || MbP->mb_type==I8MB || MbP->mb_type==IPCM)
        && !(MbQ->mb_type==I4MB || MbQ->mb_type==I16MB || MbQ->mb_type==I8MB || MbQ->mb_type==IPCM) )
      {
        if( ((MbQ->cbp_blk &  ((int64)1 << blkQ )) != 0) || ((MbP->cbp_blk &  ((int64)1 << blkP)) != 0) )
          Strength[idx] = 2 ;
        else
        {
          // if no coefs, but vector difference >= 1 set Strength=1
          // if this is a mixed mode edge then one set of reference pictures will be frame and the
          // other will be field
          if (mixedModeEdgeFlag)
          {
            (Strength[idx] = 1);
          }
          else
          {
            get_mb_block_pos (MbQAddr, &mb_x, &mb_y);
            blk_y  = (mb_y<<2) + (blkQ >> 2) ;
            blk_x  = (mb_x<<2) + (blkQ  & 3) ;
            blk_y2 = pixP.pos_y >> 2;
            blk_x2 = pixP.pos_x >> 2;

            {
              ref_p0 = list0_refIdxArr[blk_y ][blk_x ] < 0 ? INT64_MIN : list0_refPicIdArr[blk_y ][blk_x];
              ref_q0 = list0_refIdxArr[blk_y2][blk_x2] < 0 ? INT64_MIN : list0_refPicIdArr[blk_y2][blk_x2];
              ref_p1 = list1_refIdxArr[blk_y ][blk_x ] < 0 ? INT64_MIN : list1_refPicIdArr[blk_y ][blk_x];
              ref_q1 = list1_refIdxArr[blk_y2][blk_x2]<0 ? INT64_MIN : list1_refPicIdArr[blk_y2][blk_x2];
              if ( ((ref_p0==ref_q0) && (ref_p1==ref_q1)) ||
                ((ref_p0==ref_q1) && (ref_p1==ref_q0)))
              {
                Strength[idx]=0;
                // L0 and L1 reference pictures of p0 are different; q0 as well
                if (ref_p0 != ref_p1)
                {
                  // compare MV for the same reference picture
                  if (ref_p0==ref_q0)
                  {
                    Strength[idx] =  (byte) (
                      (iabs( list0_mv[blk_y][blk_x][0] - list0_mv[blk_y2][blk_x2][0]) >= 4) |
                      (iabs( list0_mv[blk_y][blk_x][1] - list0_mv[blk_y2][blk_x2][1]) >= mvlimit) |
                      (iabs( list1_mv[blk_y][blk_x][0] - list1_mv[blk_y2][blk_x2][0]) >= 4) |
                      (iabs( list1_mv[blk_y][blk_x][1] - list1_mv[blk_y2][blk_x2][1]) >= mvlimit));
                  }
                  else
                  {
                    Strength[idx] =  (byte) (
                      (iabs( list0_mv[blk_y][blk_x][0] - list1_mv[blk_y2][blk_x2][0]) >= 4) |
                      (iabs( list0_mv[blk_y][blk_x][1] - list1_mv[blk_y2][blk_x2][1]) >= mvlimit) |
                      (iabs( list1_mv[blk_y][blk_x][0] - list0_mv[blk_y2][blk_x2][0]) >= 4) |
                      (iabs( list1_mv[blk_y][blk_x][1] - list0_mv[blk_y2][blk_x2][1]) >= mvlimit));
                  }
                }
                else
                { // L0 and L1 reference pictures of p0 are the same; q0 as well

                  Strength[idx] = (byte) (
                    ((iabs( list0_mv[blk_y][blk_x][0] - list0_mv[blk_y2][blk_x2][0]) >= 4) |
                    (iabs( list0_mv[blk_y][blk_x][1] - list0_mv[blk_y2][blk_x2][1]) >= mvlimit ) |
                    (iabs( list1_mv[blk_y][blk_x][0] - list1_mv[blk_y2][blk_x2][0]) >= 4) |
                    (iabs( list1_mv[blk_y][blk_x][1] - list1_mv[blk_y2][blk_x2][1]) >= mvlimit))
                    &&
                    ((iabs( list0_mv[blk_y][blk_x][0] - list1_mv[blk_y2][blk_x2][0]) >= 4) |
                    (iabs( list0_mv[blk_y][blk_x][1] - list1_mv[blk_y2][blk_x2][1]) >= mvlimit) |
                    (iabs( list1_mv[blk_y][blk_x][0] - list0_mv[blk_y2][blk_x2][0]) >= 4) |
                    (iabs( list1_mv[blk_y][blk_x][1] - list0_mv[blk_y2][blk_x2][1]) >= mvlimit)));
                }
              }
              else
              {
                Strength[idx] = 1;
              }
            }
          }
        }
      }
    }
  }
}

/*!
 *****************************************************************************************
 * \brief
 *    Filters 16 pel block edge of Frame or Field coded MBs 
 *****************************************************************************************
 */
void EdgeLoopLumaNormal(imgpel** Img, byte Strength[16], ImageParameters *img, int MbQAddr, int AlphaC0Offset, int BetaOffset,
              int dir, int edge, int width)
{
  int      pel, ap = 0, aq = 0, Strng ;
  int      incQ    = dir ? width : 1;
  int      incP    = dir ? width : 1;

  int      C0, c0, Delta, dif, AbsDelta ;
  imgpel   L2 = 0, L1, L0, R0, R1, R2 = 0, L3, R3;
  int      RL0;
  int      Alpha = 0, Beta = 0 ;
  const byte* ClipTab = NULL;
  int      small_gap;
  int      indexA, indexB;
  imgpel   *SrcPtrP, *SrcPtrQ;
  int      QP;
  int      xQ, yQ;
  Macroblock *MbQ = &(img->mb_data[MbQAddr]);
  Macroblock *MbP;
  PixelPos pixP, pixQ, pixMB1, pixMB2;
  int      bitdepth_scale = img->bitdepth_scale[IS_LUMA];

  xQ = dir ? 0 : edge - 1;
  yQ = dir ? (edge < MB_BLOCK_SIZE ? edge - 1: 0) : 0;

  getNeighbour(MbQAddr, xQ, yQ, IS_LUMA, &pixMB1);

  if (pixMB1.available || (MbQ->LFDisableIdc== 0))
  {
    pixP = pixMB1;
    MbP = &(img->mb_data[pixP.mb_addr]);
    if (dir)
      yQ ++;
    else
      xQ ++;

    getNeighbour(MbQAddr, xQ, yQ, IS_LUMA, &pixMB2);
    pixQ = pixMB2;

    // Average QP of the two blocks
    QP = (MbP->qp + MbQ->qp + 1) >> 1;

    indexA = iClip3(0, MAX_QP, QP + AlphaC0Offset);
    indexB = iClip3(0, MAX_QP, QP + BetaOffset);

    Alpha   = ALPHA_TABLE[indexA] * bitdepth_scale;
    Beta    = BETA_TABLE [indexB] * bitdepth_scale;
    ClipTab = CLIP_TAB[indexA];

    for( pel = 0 ; pel < MB_BLOCK_SIZE ; pel++ )
    {
      if (dir)
      {
        xQ         = pel;
        pixP.x     = pixMB1.x + pel;
        pixP.pos_x = pixMB1.pos_x + pel;
        pixQ.x     = pixMB2.x + pel;
        pixQ.pos_x = pixMB2.pos_x + pel;
      }
      else
      {
        yQ         = pel;
        pixP.y     = pixMB1.y + pel;
        pixP.pos_y = pixMB1.pos_y + pel;
        pixQ.y     = pixMB2.y + pel;
        pixQ.pos_y = pixMB2.pos_y + pel;
      }

      if( (Strng = Strength[pel]) != 0)
      {        
        //MbP = &(img->mb_data[pixP.mb_addr]);

        SrcPtrQ = &(Img[pixQ.pos_y][pixQ.pos_x]);
        SrcPtrP = &(Img[pixP.pos_y][pixP.pos_x]);

        L3  = SrcPtrP[-incP*3];
        L2  = SrcPtrP[-incP*2];
        L1  = SrcPtrP[-incP];
        L0  = SrcPtrP[0] ;
        R0  = SrcPtrQ[0] ;      
        R1  = SrcPtrQ[ incQ];      
        R2  = SrcPtrQ[ incQ*2];
        R3  = SrcPtrQ[ incQ*3];

        AbsDelta  = iabs( Delta = (R0 - L0) );

        if( AbsDelta < Alpha )
        {
          C0  = ClipTab[ Strng ] * bitdepth_scale;
          if( ((iabs( R0 - R1) - Beta )  & (iabs(L0 - L1) - Beta )) < 0  )
          {            
            if(Strng == 4 )    // INTRA strong filtering
            {
              RL0 = L0 + R0;
              small_gap = (AbsDelta < ((Alpha >> 2) + 2));
              aq  = ( iabs( R0 - R2) < Beta ) & small_gap;               
              ap  = ( iabs( L0 - L2) < Beta ) & small_gap;

              if (ap)
              {
                SrcPtrP[-incP*2] = (imgpel) ((((L3 + L2) <<1) + L2 + L1 + RL0 + 4) >> 3);
                SrcPtrP[-incP]   = (imgpel) (( L2 + L1 + L0 + R0 + 2) >> 2);
                SrcPtrP[0]       = (imgpel) (( R1 + ((L1 + RL0) << 1) +  L2 + 4) >> 3);
              }
              else
              {
                SrcPtrP[ -incP * 2 ] = L2;
                SrcPtrP[ -incP     ] = L1;
                SrcPtrP[ 0         ] = (imgpel) (((L1 << 1) + L0 + R1 + 2) >> 2) ;
              }

              if (aq)
              {
                SrcPtrQ[ 0        ] = (imgpel) (( L1 + ((R1 + RL0) << 1) +  R2 + 4) >> 3);
                SrcPtrQ[ incQ     ] = (imgpel) (( R2 + R0 + R1 + L0 + 2) >> 2);
                SrcPtrQ[ incQ * 2 ] = (imgpel) ((((R3 + R2) <<1) + R2 + R1 + RL0 + 4) >> 3);
              }
              else
              {
                SrcPtrQ[ 0        ] = (imgpel) (((R1 << 1) + R0 + L1 + 2) >> 2);
                SrcPtrQ[ incQ     ] = R1;
                SrcPtrQ[ incQ * 2 ] = R2;
              }
            }
            else   // normal filtering
            {
              RL0 = (L0 + R0 + 1) >> 1;
              aq  = (iabs(R0 - R2) < Beta);
              ap  = (iabs(L0 - L2) < Beta);

              c0  = (C0 + ap + aq) ;
              dif = iClip3( -c0, c0, ((Delta << 2) + (L1 - R1) + 4) >> 3 );

              if( ap )
                SrcPtrP[-incP] += iClip3( -C0,  C0, (L2 + RL0 - (L1<<1)) >> 1 );
              SrcPtrP[0]  = (imgpel) iClip3(0, img->max_imgpel_value, L0 + dif);
              SrcPtrQ[0]  = (imgpel) iClip3(0, img->max_imgpel_value, R0 - dif);
              if( aq  )
                SrcPtrQ[ incQ] += iClip3( -C0,  C0, (R2 + RL0 - (R1<<1)) >> 1 );
            }            
          }
        }
      }
    }
  }
}

/*!
 *****************************************************************************************
 * \brief
 *    Filters 16 pel block edge of Super MB Frame coded MBs
 *****************************************************************************************
 */
void EdgeLoopLumaMBAff(imgpel** Img, byte Strength[16], ImageParameters *img, int MbQAddr, int AlphaC0Offset, int BetaOffset,
              int dir, int edge, int width)
{
  int      pel, ap = 0, aq = 0, Strng ;
  int      incP, incQ;
  int      C0, c0, Delta, dif, AbsDelta ;
  imgpel   L2 = 0, L1, L0, R0, R1, R2 = 0, L3, R3;
  int      RL0;
  int      Alpha = 0, Beta = 0 ;
  const byte* ClipTab = NULL;
  int      small_gap;
  int      indexA, indexB;
  int      PelNum  = MB_BLOCK_SIZE;
  imgpel   *SrcPtrP, *SrcPtrQ;
  int      QP;
  int      xQ, yQ;
  Macroblock *MbQ = &(img->mb_data[MbQAddr]);
  Macroblock *MbP;
  PixelPos pixP, pixQ;
  int      dir_m1 = (1 - dir);
  int      bitdepth_scale = img->bitdepth_scale[IS_LUMA];

  for( pel = 0 ; pel < PelNum ; pel++ )
  {
    xQ = dir ? pel : edge;
    yQ = dir ? (edge < 16 ? edge : 1) : pel;    
    getNeighbour(MbQAddr, xQ - (dir_m1), yQ - dir, IS_LUMA, &pixP);
    
    if (pixP.available || (MbQ->LFDisableIdc== 0))
    {
      if( (Strng = Strength[pel]) != 0)
      {
        getNeighbour(MbQAddr, xQ, yQ, IS_LUMA, &pixQ);
        
        MbP = &(img->mb_data[pixP.mb_addr]);
        fieldModeFilteringFlag = (byte) (MbQ->mb_field || MbP->mb_field);

        incQ    = dir ? ((fieldModeFilteringFlag && !MbQ->mb_field) ? 2 * width : width) : 1;
        incP    = dir ? ((fieldModeFilteringFlag && !MbP->mb_field) ? 2 * width : width) : 1;