vncencodetight.cpp

teamviewer source code vc++

																			  \
	for (dy = 0; dy < h; dy++) {											  \
		for (dx = 0; dx < w; dx++) {										  \
			if (colorValue != fbptr[dx])									  \
				return false;												  \
		}																	  \
		fbptr = (CARD##bpp *)((BYTE *)fbptr + m_bytesPerRow);				  \
	}																		  \
																			  \
	*colorPtr = (CARD32)colorValue; 										  \
	return true;															  \
}

DEFINE_CHECK_SOLID_FUNCTION(8)
DEFINE_CHECK_SOLID_FUNCTION(16)
DEFINE_CHECK_SOLID_FUNCTION(32)

UINT
vncEncodeTight::EncodeRectSimple(BYTE *source, VSocket *outConn, BYTE *dest,
								 const RECT &rect)
{
	const int x = rect.left, y = rect.top;
	const int w = rect.right - x, h = rect.bottom - y;

	const int maxRectSize = m_conf[m_compresslevel].maxRectSize;
	const int maxRectWidth = m_conf[m_compresslevel].maxRectWidth;

	int partialSize = 0;

	if (w > maxRectWidth || w * h > maxRectSize) {
		const int subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
		const int subrectMaxHeight = maxRectSize / subrectMaxWidth;
		int dx, dy, rw, rh;

		for (dy = 0; dy < h; dy += subrectMaxHeight) {
			for (dx = 0; dx < w; dx += maxRectWidth) {
				rw = (dx + maxRectWidth < w) ? maxRectWidth : w - dx;
				rh = (dy + subrectMaxHeight < h) ? subrectMaxHeight : h - dy;

				partialSize = EncodeSubrect(source, outConn, dest,
											x+dx, y+dy, rw, rh);
				if (dy + subrectMaxHeight < h || dx + maxRectWidth < w) {
					outConn->SendExactQueue((char *)dest, partialSize);
				}
			}
		}
	} else {
		partialSize = EncodeSubrect(source, outConn, dest, x, y, w, h);
	}

	return partialSize;
}

UINT
vncEncodeTight::EncodeSubrect(BYTE *source, VSocket *outConn, BYTE *dest,
							  int x, int y, int w, int h)
{
	SendTightHeader(x, y, w, h);

	RECT r;
	r.left = x; r.top = y;
	r.right = x + w; r.bottom = y + h;
	Translate(source, m_buffer, r);

	m_paletteMaxColors = w * h / m_conf[m_compresslevel].idxMaxColorsDivisor;
	if ( m_paletteMaxColors < 2 &&
		 w * h >= m_conf[m_compresslevel].monoMinRectSize ) {
		m_paletteMaxColors = 2;
	}
	switch (m_remoteformat.bitsPerPixel) {
	case 8:
		FillPalette8(w * h);
		break;
	case 16:
		FillPalette16(w * h);
		break;
	default:
		FillPalette32(w * h);
	}

	int encDataSize;
	switch (m_paletteNumColors) {
	case 0:
		// Truecolor image
		if (DetectSmoothImage(w, h)) {
			if (m_qualitylevel != -1) {
				encDataSize = SendJpegRect(dest, w, h,
										   m_conf[m_qualitylevel].jpegQuality);
			} else {
				encDataSize = SendGradientRect(dest, w, h);
			}
		} else {
			encDataSize = SendFullColorRect(dest, w, h);
		}
		break;
	case 1:
		// Solid rectangle
		encDataSize = SendSolidRect(dest);
		break;
	case 2:
		// Two-color rectangle
		encDataSize = SendMonoRect(dest, w, h);
		break;
	default:
		// Up to 256 different colors
		if ( m_paletteNumColors > 96 &&
			 m_qualitylevel != -1 && m_qualitylevel <= 3 &&
			 DetectSmoothImage(w, h) ) {
			encDataSize = SendJpegRect(dest, w, h,
									   m_conf[m_qualitylevel].jpegQuality);
		} else {
			encDataSize = SendIndexedRect(dest, w, h);
		}
	}

	if (encDataSize < 0)
		return vncEncoder::EncodeRect(source, dest, r);

	outConn->SendExactQueue((char *)m_hdrBuffer, m_hdrBufferBytes);

	encodedSize += m_hdrBufferBytes - sz_rfbFramebufferUpdateRectHeader + encDataSize;
	transmittedSize += m_hdrBufferBytes + encDataSize;

	return encDataSize;
}

void
vncEncodeTight::SendTightHeader(int x, int y, int w, int h)
{
	rfbFramebufferUpdateRectHeader rect;

	rect.r.x = Swap16IfLE(x-m_SWOffsetx);
	rect.r.y = Swap16IfLE(y-m_SWOffsety);
	rect.r.w = Swap16IfLE(w);
	rect.r.h = Swap16IfLE(h);
	rect.encoding = Swap32IfLE(rfbEncodingTight);

	dataSize += w * h * (m_remoteformat.bitsPerPixel / 8);
	rectangleOverhead += sz_rfbFramebufferUpdateRectHeader;

	memcpy(m_hdrBuffer, (BYTE *)&rect, sz_rfbFramebufferUpdateRectHeader);
	m_hdrBufferBytes = sz_rfbFramebufferUpdateRectHeader;
}

//
// Subencoding implementations.
//

int
vncEncodeTight::SendSolidRect(BYTE *dest)
{
	int len;

	if (m_usePixelFormat24) {
		Pack24(m_buffer, 1);
		len = 3;
	} else
		len = m_remoteformat.bitsPerPixel / 8;

	m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFill << 4;
	memcpy (dest, m_buffer, len);

	return len;
}

int
vncEncodeTight::SendMonoRect(BYTE *dest, int w, int h)
{
	const int streamId = 1;
	int paletteLen, dataLen;
	CARD8 paletteBuf[8];

	// Prepare tight encoding header.
	dataLen = (w + 7) / 8;
	dataLen *= h;

	m_hdrBuffer[m_hdrBufferBytes++] = (streamId | rfbTightExplicitFilter) << 4;
	m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFilterPalette;
	m_hdrBuffer[m_hdrBufferBytes++] = 1;

	// Prepare palette, convert image.
	switch (m_remoteformat.bitsPerPixel) {
	case 32:
		EncodeMonoRect32((CARD8 *)m_buffer, w, h);

		((CARD32 *)paletteBuf)[0] = m_monoBackground;
		((CARD32 *)paletteBuf)[1] = m_monoForeground;

		if (m_usePixelFormat24) {
			Pack24(paletteBuf, 2);
			paletteLen = 6;
		} else
			paletteLen = 8;

		memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf, paletteLen);
		m_hdrBufferBytes += paletteLen;
		break;

	case 16:
		EncodeMonoRect16((CARD8 *)m_buffer, w, h);

		((CARD16 *)paletteBuf)[0] = (CARD16)m_monoBackground;
		((CARD16 *)paletteBuf)[1] = (CARD16)m_monoForeground;

		memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf, 4);
		m_hdrBufferBytes += 4;
		break;

	default:
		EncodeMonoRect8((CARD8 *)m_buffer, w, h);

		m_hdrBuffer[m_hdrBufferBytes++] = (BYTE)m_monoBackground;
		m_hdrBuffer[m_hdrBufferBytes++] = (BYTE)m_monoForeground;
	}

	return CompressData(dest, streamId, dataLen,
						m_conf[m_compresslevel].monoZlibLevel,
						Z_DEFAULT_STRATEGY);
}

int
vncEncodeTight::SendIndexedRect(BYTE *dest, int w, int h)
{
	const int streamId = 2;
	int i, entryLen;
	CARD8 paletteBuf[256*4];

	// Prepare tight encoding header.
	m_hdrBuffer[m_hdrBufferBytes++] = (streamId | rfbTightExplicitFilter) << 4;
	m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFilterPalette;
	m_hdrBuffer[m_hdrBufferBytes++] = (BYTE)(m_paletteNumColors - 1);

	// Prepare palette, convert image.
	switch (m_remoteformat.bitsPerPixel) {
	case 32:
		EncodeIndexedRect32((CARD8 *)m_buffer, w * h);

		for (i = 0; i < m_paletteNumColors; i++) {
			((CARD32 *)paletteBuf)[i] =
				m_palette.entry[i].listNode->rgb;
		}
		if (m_usePixelFormat24) {
			Pack24(paletteBuf, m_paletteNumColors);
			entryLen = 3;
		} else
			entryLen = 4;

		memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf,
			   m_paletteNumColors * entryLen);
		m_hdrBufferBytes += m_paletteNumColors * entryLen;
		break;

	case 16:
		EncodeIndexedRect16((CARD8 *)m_buffer, w * h);

		for (i = 0; i < m_paletteNumColors; i++) {
			((CARD16 *)paletteBuf)[i] =
				(CARD16)m_palette.entry[i].listNode->rgb;
		}

		memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf,
			   m_paletteNumColors * 2);
		m_hdrBufferBytes += m_paletteNumColors * 2;
		break;

	default:
		return -1;				// Should never happen.
	}

	return CompressData(dest, streamId, w * h,
						m_conf[m_compresslevel].idxZlibLevel,
						Z_DEFAULT_STRATEGY);
}

int
vncEncodeTight::SendFullColorRect(BYTE *dest, int w, int h)
{
	const int streamId = 0;
	int len;

	m_hdrBuffer[m_hdrBufferBytes++] = 0x00;

	if (m_usePixelFormat24) {
		Pack24(m_buffer, w * h);
		len = 3;
	} else
		len = m_remoteformat.bitsPerPixel / 8;

	return CompressData(dest, streamId, w * h * len,
						m_conf[m_compresslevel].rawZlibLevel,
						Z_DEFAULT_STRATEGY);
}

int
vncEncodeTight::SendGradientRect(BYTE *dest, int w, int h)
{
	const int streamId = 3;
	int len;

	if (m_remoteformat.bitsPerPixel == 8)
		return SendFullColorRect(dest, w, h);

	if (m_prevRowBuf == NULL)
		m_prevRowBuf = new int [2048*3];

	m_hdrBuffer[m_hdrBufferBytes++] = (streamId | rfbTightExplicitFilter) << 4;
	m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFilterGradient;

	if (m_usePixelFormat24) {
		FilterGradient24(m_buffer, w, h);
		len = 3;
	} else if (m_remoteformat.bitsPerPixel == 32) {
		FilterGradient32((CARD32 *)m_buffer, w, h);
		len = 4;
	} else {
		FilterGradient16((CARD16 *)m_buffer, w, h);
		len = 2;
	}

	return CompressData(dest, streamId, w * h * len,
						m_conf[m_compresslevel].gradientZlibLevel,
						Z_FILTERED);
}

int
vncEncodeTight::CompressData(BYTE *dest, int streamId, int dataLen,
							 int zlibLevel, int zlibStrategy)
{
	if (dataLen < TIGHT_MIN_TO_COMPRESS) {
		memcpy(dest, m_buffer, dataLen);
		return dataLen;
	}

	z_streamp pz = &m_zsStruct[streamId];

	// Initialize compression stream if needed.
	if (!m_zsActive[streamId]) {
		pz->zalloc = Z_NULL;
		pz->zfree = Z_NULL;
		pz->opaque = Z_NULL;

		vnclog.Print(LL_INTINFO,
					 VNCLOG("calling deflateInit2 with zlib level:%d"),
					 zlibLevel);
		int err = deflateInit2 (pz, zlibLevel, Z_DEFLATED, MAX_WBITS,
								MAX_MEM_LEVEL, zlibStrategy);
		if (err != Z_OK) {
			vnclog.Print(LL_INTINFO,
						 VNCLOG("deflateInit2 returned error:%d:%s"),
						 err, pz->msg);
			return -1;
		}

		m_zsActive[streamId] = true;
		m_zsLevel[streamId] = zlibLevel;
	}

	int outBufferSize = dataLen + dataLen / 100 + 16;

	// Prepare buffer pointers.
	pz->next_in = (Bytef *)m_buffer;
	pz->avail_in = dataLen;
	pz->next_out = (Bytef *)dest;
	pz->avail_out = outBufferSize;

	// Change compression parameters if needed.
	if (zlibLevel != m_zsLevel[streamId]) {
		vnclog.Print(LL_INTINFO,
					 VNCLOG("calling deflateParams with zlib level:%d"),
					 zlibLevel);
		int err = deflateParams (pz, zlibLevel, zlibStrategy);
		if (err != Z_OK) {
			vnclog.Print(LL_INTINFO,
						 VNCLOG("deflateParams returned error:%d:%s"),
						 err, pz->msg);
			return -1;
		}
		m_zsLevel[streamId] = zlibLevel;
	}

	// Actual compression.
	if ( deflate (pz, Z_SYNC_FLUSH) != Z_OK ||
		 pz->avail_in != 0 || pz->avail_out == 0 ) {
		vnclog.Print(LL_INTINFO, VNCLOG("deflate() call failed."));
		return -1;
	}

	return SendCompressedData(outBufferSize - pz->avail_out);
}

int
vncEncodeTight::SendCompressedData(int compressedLen)
{
	// Prepare compressed data size for sending.
	m_hdrBuffer[m_hdrBufferBytes++] = compressedLen & 0x7F;
	if (compressedLen > 0x7F) {
		m_hdrBuffer[m_hdrBufferBytes-1] |= 0x80;
		m_hdrBuffer[m_hdrBufferBytes++] = compressedLen >> 7 & 0x7F;
		if (compressedLen > 0x3FFF) {
			m_hdrBuffer[m_hdrBufferBytes-1] |= 0x80;
			m_hdrBuffer[m_hdrBufferBytes++] = compressedLen >> 14 & 0xFF;
		}
	}
	return compressedLen;