loopfilter.cpp

avs decoder sources, added a command line sample

/**
 * Copyright (c) 2006
 *      OpenAVS Developers. All Rights Reserved.
 *
 * Copyright (c) 2005-2006
 *      NSCC. All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

/**
 * ! \file "loopfilter.c"
 *   \brief Functions related to loop filter.
 */

#include "global.h"
#include "loopfilter.h"

static void DeblockMb_BsV00_Luma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE * pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsV10_Luma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE * pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsV01_Luma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE * pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsV11_Luma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE* pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsH00_Luma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE* pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsH01_Luma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE * pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsH10_Luma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE* pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsH11_Luma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE * pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsV00_Chroma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset,
		AVS_BYTE * pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsV10_Chroma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE * pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsH00_Chroma(
		AVS_INT Bs, 
		MBINFO* pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE * pImgData, 
		AVS_INT iStride);
static void DeblockMb_BsH01_Chroma(
		AVS_INT Bs, 
		MBINFO * pMbInfo, 
		AVS_DWORD dwMbIndex, 
		AVS_DWORD dwMbWidth, 
		AVS_INT iImgType, 
		AVS_INT iAlphaOffset, AVS_INT iBetaOffset, 
		AVS_BYTE * pImgData, 
		AVS_INT iStride);

/*
 *   BsH00    BsH01
 * ---------------------
 * |        |          |
 * |BsV00   |BsV01     |
 * |        |          |
 * | BsH10  | BsH11    |
 * ---------------------
 * |        |          |
 * |BsV10   |BsV11     |
 * |        |          |
 * |        |          |
 * ---------------------
 */
static void DeblockVBs2_Luma(
		AVS_BYTE* pStart, 
		AVS_INT Alpha, AVS_INT Beta, 
		AVS_INT iStride)
{
	AVS_INT i;
	AVS_BYTE * p = pStart;
	for (i = 0; i < 8; i++) {
		AVS_BYTE p2 = *(p - 3);
		AVS_BYTE p1 = *(p - 2);
		AVS_BYTE p0 = *(p - 1);

		AVS_BYTE q0 = *p;
		AVS_BYTE q1 = *(p + 1);
		AVS_BYTE q2 = *(p + 2);
		if ((abs(p0 - q0) < Alpha) && 
		    (abs(p1 - p0) < Beta) && 
		    (abs(q1 - q0) < Beta)) {
			AVS_INT ap = abs(p2 - p0);
			AVS_INT aq = abs(q2 - q0);

			if ((ap < Beta) && (abs(p0 - q0) <((Alpha >> 2) + 2))) {
				*(p - 1) = (p1 + 2 * p0 + q0 + 2) >> 2;
				*(p - 2) = (2 * p1 + p0 + q0 + 2) >> 2;
			}
			else {
				*(p - 1) = (2 * p1 + p0 + q0 + 2) >> 2;
			}

			if ((aq < Beta) && (abs(p0 - q0) < ((Alpha >> 2) + 2))) {
				*(p + 0) = (q1 + 2 * q0 + p0 + 2) >> 2;
				*(p + 1) = (2 * q1 + q0 + p0 + 2) >> 2;
			}
			else {
				*(p + 0) = (2 * q1 + q0 + p0 + 2) >> 2;
			}
		}

		p += iStride;
	} 
}


static void DeblockVBs1_Luma(
		AVS_BYTE* pStart,
		AVS_INT C, 
		AVS_INT Alpha, AVS_INT Beta, 
		AVS_INT iStride)
{
	AVS_INT i;
	AVS_BYTE * p = pStart;

	for (i = 0; i < 8; i++) {
		AVS_BYTE p2 = *(p - 3);
		AVS_BYTE p1 = *(p - 2);
		AVS_BYTE p0 = *(p - 1);

		AVS_BYTE q0 = *(p + 0);
		AVS_BYTE q1 = *(p + 1);
		AVS_BYTE q2 = *(p + 2);

		if ((abs(p0 - q0) < Alpha) && 
		    (abs(p1 - p0) < Beta) && 
		    (abs(q1 - q0) < Beta)) {
			AVS_INT delta = Clip3(-C, C, 
			                      (((q0 - p0) * 3 + (p1 - q1) + 4) >> 3));

			*(p - 1) = Clip(p0 + delta);
			*(p - 0) = Clip(q0 + delta);

			AVS_INT ap = abs(p2 - p0);
			AVS_INT aq = abs(q2 - q0);

			if (ap < Beta)
				*(p - 2) = Clip(p1 + 
				                Clip3(-C, C, 
				                (((*(p - 1) - p1) * 3 + (p2 - *(p)) + 4) >> 3)));
			if (aq < Beta)
				*(p + 1) = Clip(q1 -
				                Clip3(-C, C, 
				                (((q1 - *(p)) * 3 + (*(p - 1) - q2) + 4) >> 3)));
		}

		p += iStride;
	}  
}

static void DeblockHBs2_Luma(
		AVS_BYTE * pStart, 
		AVS_INT Alpha, AVS_INT Beta, 
		AVS_INT iStride)
{
	AVS_INT i;
	AVS_BYTE* p = pStart;
	for (i = 0; i < 8; i++) {
		AVS_BYTE p2 = *(p - 3 * iStride);
		AVS_BYTE p1 = *(p - 2 * iStride);
		AVS_BYTE p0 = *(p - 1 * iStride);

		AVS_BYTE q0 = *(p + 0 * iStride);
		AVS_BYTE q1 = *(p + 1 * iStride);
		AVS_BYTE q2 = *(p + 2 * iStride);
		if ((abs(p0 - q0) < Alpha) && 
		    (abs(p1 - p0) < Beta) && 
		    (abs(q1 - q0) < Beta)) {
			AVS_INT ap = abs(p2 - p0);
			AVS_INT aq = abs(q2 - q0);

			if ((ap < Beta) && (abs(p0 - q0) < ((Alpha >> 2) + 2))) {
				*(p - 1 * iStride) = (p1 + 2 * p0 + q0 + 2) >> 2;
				*(p - 2 * iStride) = (2 * p1 + p0 + q0 + 2) >> 2;
			}
			else {
				*(p - iStride) = (2 * p1 + p0 + q0 + 2) >> 2;
			}
			if ((aq < Beta) && (abs(p0 - q0) < ((Alpha >> 2) + 2))) {
				*(p + 0 * iStride) = (q1 + 2 * q0 + p0 + 2) >> 2;
				*(p + 1 * iStride) = (2 * q1 + q0 + p0 + 2) >> 2;
			}
			else {
				*(p) = (2 * q1 + q0 + p0 + 2) >> 2;
			}
		}
		p++;
	}
}

static void DeblockHBs1_Luma(
		AVS_BYTE * pStart,
		AVS_INT C, 
		AVS_INT Alpha, AVS_INT Beta, 
		AVS_INT iStride)
{
	AVS_INT i;
	AVS_BYTE * p = pStart;
	for (i = 0; i < 8; i++) {
		AVS_BYTE p2 = *(p - 3 * iStride);
		AVS_BYTE p1 = *(p - 2 * iStride);
		AVS_BYTE p0 = *(p - 1 * iStride);

		AVS_BYTE q0 = *p;
		AVS_BYTE q1 = *(p + 1 * iStride);
		AVS_BYTE q2 = *(p + 2 * iStride);

		if ((abs(p0 - q0) < Alpha) && 
		    (abs(p1 - p0) < Beta) && 
		    (abs(q1 - q0) < Beta)) {
			AVS_INT delta = 
				Clip3(-C, C, (((q0 - p0) * 3 + (p1 - q1) + 4) >> 3));

			*(p - iStride) = Clip(p0 + delta);
			*(p ) = Clip(q0 + delta);

			AVS_INT ap = abs(p2 - p0);
			AVS_INT aq = abs(q2 - q0);

			if (ap < Beta)
				*(p - 2 * iStride) = 
					Clip(p1 + 
					     Clip3(-C, C, 
					           (((*(p - iStride) - p1) * 3 + 
					              (p2-*(p)) + 4) >> 3)));
			if (aq < Beta)
				*(p + iStride) = 
					Clip(q1 -
					     Clip3(-C, C, 
					           (((q1 - *(p)) * 3 + 
					             (*(p - iStride) - q2) + 4) >> 3)));
		}
		p++;
	}
}

static void DeblockVBs2_Chroma(
		AVS_BYTE * pStart, 
		AVS_INT Alpha, AVS_INT Beta, 
		AVS_INT iStride)
{
	AVS_INT i;
	AVS_BYTE * p = pStart;

	for (i = 0; i < 4; i++) {
		AVS_BYTE p2 = *(p - 3);
		AVS_BYTE p1 = *(p - 2);
		AVS_BYTE p0 = *(p - 1);

		AVS_BYTE q0 = *(p + 0);
		AVS_BYTE q1 = *(p + 1);
		AVS_BYTE q2 = *(p + 2);
		if ((abs(p0 - q0) < Alpha) && 
                    (abs(p1 - p0) < Beta) && 
                    (abs(q1 - q0) < Beta)) {
			AVS_INT ap = abs(p2 - p0);
			AVS_INT aq = abs(q2 - q0);

			if ((ap < Beta) && (abs(p0 - q0) < ((Alpha >> 2) + 2))) {
				*(p - 1) = (p1 + 2 * p0 + q0 + 2) >> 2;
			}
			else {
				*(p - 1) = (2 * p1 + p0 + q0 + 2) >> 2;
			}
			if ((aq < Beta) && (abs(p0 - q0) < ((Alpha >> 2) + 2))) {
				*(p ) = (q1 + 2 * q0 + p0 + 2) >> 2;
			}
			else {
				*(p ) = (2 * q1 + q0 + p0 + 2) >> 2;
			}
		}

		p += iStride;
	}
}

static void DeblockVBs1_Chroma(
		AVS_BYTE * pStart, 
		AVS_INT Alpha, AVS_INT Beta, 
		AVS_INT C, 
		AVS_INT iStride)
{
	AVS_INT i;
	AVS_BYTE * p = pStart;
	for (i = 0; i < 4; i++) {
		AVS_BYTE p2 = *(p - 3);
		AVS_BYTE p1 = *(p - 2);
		AVS_BYTE p0 = *(p - 1);

		AVS_BYTE q0 = *p;
		AVS_BYTE q1 = *(p + 1);
		AVS_BYTE q2 = *(p + 2);

		if (abs(p0 - q0) < Alpha && 
		    abs(p1 - p0) < Beta && 
		    abs(q1 - q0) < Beta) {   
			AVS_INT delta = Clip3(-C, C, 
			                      (((q0 - p0) * 3 + 
			                        (p1 - q1) + 4) >> 3));

			*(p - 1) = Clip(p0 + delta);
			*(p  ) = Clip(q0 + delta);
		}    

		p += iStride;
	}
}

static void DeblockHBs2_Chroma(
		AVS_BYTE * pStart,