media_source.cpp

MPEG-4编解码的实现(包括MPEG4视音频编解码)

		imgcpy(m_videoDstPrevImage + m_videoDstYSize,
			uImage, 
			m_videoDstWidth / 2,
			m_videoDstHeight / 2,
			uvStride);
		imgcpy(m_videoDstPrevImage + m_videoDstYSize + m_videoDstUVSize,
			vImage, 
			m_videoDstWidth / 2,
			m_videoDstHeight / 2,
			uvStride);
	}

	// forward reconstructed video to sinks
	if (m_pConfig->m_videoEncode
	  && m_pConfig->GetBoolValue(CONFIG_VIDEO_ENCODED_PREVIEW)) {

		if (m_videoDstPrevReconstructImage) {
			CMediaFrame* pFrame =
				new CMediaFrame(CMediaFrame::ReconstructYuvVideoFrame, 
					m_videoDstPrevReconstructImage, 
					m_videoDstYUVSize,
					dstPrevFrameTimestamp, 
					dstPrevFrameDuration);
			ForwardFrame(pFrame);
			delete pFrame;
		}

		m_videoDstPrevReconstructImage = 
			(u_int8_t*)Malloc(m_videoDstYUVSize);

		m_videoEncoder->GetReconstructedImage(
			m_videoDstPrevReconstructImage,
			m_videoDstPrevReconstructImage 
				+ m_videoDstYSize,
			m_videoDstPrevReconstructImage
				+ m_videoDstYSize + m_videoDstUVSize);
	}

	m_videoDstFrameNumber++;

	// calculate how we're doing versus target frame rate
	// this is used to decide if we need to drop frames
	if (m_sourceRealTime) {
		// reset skipped frames
		m_videoSkippedFrames = 0;

		Duration drift =
			(GetTimestamp() - encodingStartTimestamp) 
			- m_videoDstFrameDuration;

		if (drift > 0) {
			m_videoEncodingDrift += drift;
		}
	}

	free(mallocedYuvImage);
	return;
}

void CMediaSource::DoStopVideo()
{
	DestroyVideoResizer();

	if (m_videoEncoder) {
		m_videoEncoder->Stop();
		delete m_videoEncoder;
		m_videoEncoder = NULL;
	}

	m_sourceVideo = false;
}

void CMediaSource::DestroyVideoResizer()
{
	if (m_videoSrcYImage) {
		scale_free_image(m_videoSrcYImage);
		m_videoSrcYImage = NULL;
	}
	if (m_videoDstYImage) {
		scale_free_image(m_videoDstYImage);
		m_videoDstYImage = NULL;
	}
	if (m_videoYResizer) {
		scale_image_done(m_videoYResizer);
		m_videoYResizer = NULL;
	}
	if (m_videoSrcUVImage) {
		scale_free_image(m_videoSrcUVImage);
		m_videoSrcUVImage = NULL;
	}
	if (m_videoDstUVImage) {
		scale_free_image(m_videoDstUVImage);
		m_videoDstUVImage = NULL;
	}
	if (m_videoUVResizer) {
		scale_image_done(m_videoUVResizer);
		m_videoUVResizer = NULL;
	}
}

bool CMediaSource::InitAudio(
	bool realTime)
{
	m_sourceRealTime = realTime;
	m_sinkRealTime = m_pConfig->GetBoolValue(CONFIG_RTP_ENABLE);
	m_audioSrcSampleNumber = 0;
	m_audioSrcFrameNumber = 0;
	m_videoSrcFrameNumber = 0;	// ensure video is also at zero

	// audio destination info
	char* dstEncoding =
		m_pConfig->GetStringValue(CONFIG_AUDIO_ENCODING);

	if (!strcasecmp(dstEncoding, AUDIO_ENCODING_MP3)) {
		m_audioDstType = CMediaFrame::Mp3AudioFrame;
	} else if (!strcasecmp(dstEncoding, AUDIO_ENCODING_AAC)) {
		m_audioDstType = CMediaFrame::AacAudioFrame;
	} else {
		debug_message("unknown dest audio encoding");
		return false;
	}
	m_audioDstChannels =
		m_pConfig->GetIntegerValue(CONFIG_AUDIO_CHANNELS);
	m_audioDstSampleRate =
		m_pConfig->GetIntegerValue(CONFIG_AUDIO_SAMPLE_RATE);
	m_audioDstSampleNumber = 0;
	m_audioDstFrameNumber = 0;
	m_audioDstRawSampleNumber = 0;
	m_audioDstRawFrameNumber = 0;

	m_audioSrcElapsedDuration = 0;
	m_audioDstElapsedDuration = 0;

	return true;
}

bool CMediaSource::SetAudioSrc(
	MediaType srcType,
	u_int8_t srcChannels,
	u_int32_t srcSampleRate)
{
	// audio source info 
	m_audioSrcType = srcType;
	m_audioSrcChannels = srcChannels;
	m_audioSrcSampleRate = srcSampleRate;
	m_audioSrcSamplesPerFrame = 0;	// unknown, presumed variable

	// init audio encoder
	delete m_audioEncoder;

	m_audioEncoder = AudioEncoderCreate(
		m_pConfig->GetStringValue(CONFIG_AUDIO_ENCODER));

	if (m_audioEncoder == NULL) {
		return false;
	}

	if (!m_audioEncoder->Init(m_pConfig, m_sourceRealTime)) {
		delete m_audioEncoder;
		m_audioEncoder = NULL;
		return false;
	}

	m_audioDstSamplesPerFrame = 
		m_audioEncoder->GetSamplesPerFrame();

	// if we need to resample
	if (m_audioDstSampleRate != m_audioSrcSampleRate) {

		// if sampling rates are integral multiples
		// we can use simpler, faster linear interpolation
		// else we use quadratic interpolation
		if (m_audioDstSampleRate < m_audioSrcSampleRate) {
			m_audioResampleUseLinear =
				((m_audioSrcSampleRate % m_audioDstSampleRate) == 0);

		} else if (m_audioDstSampleRate >= m_audioSrcSampleRate) {
			m_audioResampleUseLinear =
				((m_audioDstSampleRate % m_audioSrcSampleRate) == 0);
		}

		m_audioResampleInputNumber = 0;
		free(m_audioResampleInputBuffer);
		m_audioResampleInputBuffer =
			(int16_t*)calloc(16 * m_audioSrcChannels, 2);
	}

	m_audioPreEncodingBufferLength = 0;
	m_audioPreEncodingBufferMaxLength =
		4 * DstSamplesToBytes(m_audioDstSamplesPerFrame);

	m_audioPreEncodingBuffer = (u_int8_t*)realloc(
		m_audioPreEncodingBuffer,
		m_audioPreEncodingBufferMaxLength);
		
	if (m_audioPreEncodingBuffer == NULL) {
		delete m_audioEncoder;
		m_audioEncoder = NULL;
		return false;
	}

	return true;
}

void CMediaSource::ProcessAudioFrame(
	u_int8_t* frameData,
	u_int32_t frameDataLength,
	Timestamp srcFrameTimestamp,
	bool resync)
{
	if (m_videoSrcFrameNumber == 0 && m_audioSrcFrameNumber == 0) {
		m_startTimestamp = srcFrameTimestamp;
	}

	m_audioSrcFrameNumber++;
	m_audioSrcElapsedDuration = srcFrameTimestamp - m_startTimestamp;

	if (resync) {
		// flush preEncodingBuffer
		m_audioPreEncodingBufferLength = 0;

		// change dst sample numbers to account for gap
		m_audioDstSampleNumber =
		m_audioDstRawSampleNumber =
			DstTicksToSamples(m_audioSrcElapsedDuration);
	}

	bool pcmMalloced = false;
	bool pcmBuffered;
	u_int8_t* pcmData = frameData;
	u_int32_t pcmDataLength = frameDataLength;

	// resample audio, if necessary
	if (m_audioSrcSampleRate != m_audioDstSampleRate) {
		ResampleAudio(pcmData, pcmDataLength);

		// resampled data is now available in m_audioPreEncodingBuffer
		pcmBuffered = true;

	} else if (m_audioSrcSamplesPerFrame != m_audioDstSamplesPerFrame) {
		// reframe audio, if necessary
		// e.g. MP3 is 1152 samples/frame, AAC is 1024 samples/frame

		// add samples to end of m_audioBuffer
		// InitAudio() ensures that buffer is large enough
		memcpy(
			&m_audioPreEncodingBuffer[m_audioPreEncodingBufferLength],
			pcmData,
			pcmDataLength);

		m_audioPreEncodingBufferLength += pcmDataLength;

		pcmBuffered = true;

	} else {
		pcmBuffered = false;
	}

// LATER restructure so as get rid of this label, and goto below
pcmBufferCheck:

	if (pcmBuffered) {
		u_int32_t samplesAvailable =
			DstBytesToSamples(m_audioPreEncodingBufferLength);

		// not enough samples collected yet to call encode or forward
		if (samplesAvailable < m_audioDstSamplesPerFrame) {
			return;
		}

		// setup for encode/forward
		pcmData = 
			&m_audioPreEncodingBuffer[0];
		pcmDataLength = 
			DstSamplesToBytes(m_audioDstSamplesPerFrame);
	}

	// encode audio frame
	if (m_pConfig->m_audioEncode) {

		Timestamp encodingStartTimestamp = GetTimestamp();

		bool rc = m_audioEncoder->EncodeSamples(
			(u_int16_t*)pcmData, pcmDataLength);

		if (!rc) {
			debug_message("failed to encode audio");
			return;
		}

		Duration encodingTime =
			(GetTimestamp() - encodingStartTimestamp);

		if (m_sourceRealTime) {
			Duration drift = encodingTime 
				- DstSamplesToTicks(DstBytesToSamples(pcmDataLength));

			if (drift > 0) {
				m_videoSource->AddEncodingDrift(drift);
			}
		}

		u_int32_t forwardedSamples;
		u_int32_t forwardedFrames;

		ForwardEncodedAudioFrames(
			m_startTimestamp 
				+ DstSamplesToTicks(m_audioDstSampleNumber),
			&forwardedSamples,
			&forwardedFrames);

		m_audioDstSampleNumber += forwardedSamples;
		m_audioDstFrameNumber += forwardedFrames;
	}

	// if desired, forward raw audio to sinks
	if (m_pConfig->SourceRawAudio()) {

		// make a copy of the pcm data if needed
		u_int8_t* pcmForwardedData;

		if (!pcmMalloced) {
			pcmForwardedData = (u_int8_t*)Malloc(pcmDataLength);
			memcpy(pcmForwardedData, pcmData, pcmDataLength);
		} else {
			pcmForwardedData = pcmData;
			pcmMalloced = false;
		}

		CMediaFrame* pFrame =
			new CMediaFrame(
				CMediaFrame::PcmAudioFrame, 
				pcmForwardedData, 
				pcmDataLength,
				m_startTimestamp 
					+ DstSamplesToTicks(m_audioDstRawSampleNumber),
				DstBytesToSamples(pcmDataLength),
				m_audioDstSampleRate);
		ForwardFrame(pFrame);
		delete pFrame;

		m_audioDstRawSampleNumber += DstBytesToSamples(pcmDataLength);
		m_audioDstRawFrameNumber++;
	}

	if (pcmMalloced) {
		free(pcmData);
	}

	if (pcmBuffered) {
		m_audioPreEncodingBufferLength -= pcmDataLength;

		memcpy(
			&m_audioPreEncodingBuffer[0],
			&m_audioPreEncodingBuffer[pcmDataLength],
			m_audioPreEncodingBufferLength);

		goto pcmBufferCheck;
	}
}

void CMediaSource::ResampleAudio(
	u_int8_t* frameData,
	u_int32_t frameDataLength)
{
	int16_t* pIn =
		(int16_t*)frameData;
	int16_t* pOut = 
		(int16_t*)&m_audioPreEncodingBuffer[m_audioPreEncodingBufferLength];

	// compute how many input samples are available
	u_int32_t numIn =
		m_audioResampleInputNumber + SrcBytesToSamples(frameDataLength);

	// compute how many output samples 
	// can be generated from the available input samples
	u_int32_t numOut =
		(numIn * m_audioDstSampleRate) / m_audioSrcSampleRate;

	float inTime0 = 
		(float)(m_audioResampleInputNumber * m_audioSrcSampleRate)
		/ (float)m_audioDstSampleRate;

	int32_t inIndex;
	u_int32_t outIndex;

	// for all output samples
	for (outIndex = 0; true; outIndex++) {

		float outTime0 = 
			(outIndex * m_audioSrcSampleRate) / (float)m_audioDstSampleRate;

		inIndex = (int32_t)(outTime0 - inTime0 + 0.5);

		// DEBUG printf("resample out %d %f in %d %f\n",
		// DEBUG outIndex, outTime0, inIndex, inTime0);

		// the unusual location of the loop exit condition
		// is because we need the initial inIndex for the next call
		// see the end of this function for how this is used
		if (outIndex >= numOut) {
			break;
		}

		float x1 = outTime0 - (inTime0 + inIndex);
		float x0 = x1 + 1;
		float x2 = x1 - 1;
		float x3 = x1 - 2;

		for (u_int8_t ch = 0; ch < m_audioSrcChannels; ch++) {
			int16_t y0, y1, y2, y3;

			if (inIndex < 0) {
				y1 = m_audioResampleInputBuffer
					[(-(inIndex) - 1) * m_audioSrcChannels + ch];
			} else {
				y1 = pIn[inIndex * m_audioSrcChannels + ch];
			}
			if (inIndex + 1 < 0) {
				y2 = m_audioResampleInputBuffer
					[(-(inIndex + 1) - 1) * m_audioSrcChannels + ch];
			} else {
				y2 = pIn[(inIndex + 1) * m_audioSrcChannels + ch];
			}

			if (m_audioResampleUseLinear) {
				*pOut++ = (int16_t)
					(((y2 * x1) - (y1 * x2)) + 0.5);

			} else { // use quadratic resampling
				if (inIndex - 1 < 0) {
					y0 = m_audioResampleInputBuffer
						[(-(inIndex - 1) - 1) * m_audioSrcChannels + ch];
				} else {
					y0 = pIn[(inIndex - 1) * m_audioSrcChannels + ch];
				}
				if (inIndex + 2 < 0) {
					y3 = m_audioResampleInputBuffer
						[(-(inIndex + 2) - 1) * m_audioSrcChannels + ch];
				} else {
					y3 = pIn[(inIndex + 2) * m_audioSrcChannels + ch];
				}

				int32_t outValue = (int32_t)(
					- (y0 * x1 * x2 * x3 / 6) 
					+ (y1 * x0 * x2 * x3 / 2) 
					- (y2 * x0 * x1 * x3 / 2) 
					+ (y3 * x0 * x1 * x2 / 6)
					+ 0.5);

				// DEBUG printf("x0 %f y %d %d %d %d -> %d (delta %d)\n",
				// DEBUG x0, y0, y1, y2, y3, 
				// DEBUG outValue, outValue - ((y0 + y1) / 2));

				if (outValue > 32767) {
					outValue = 32767;
				} else if (outValue < -32767) {
					outValue = -32767;
				}

				*pOut++ = (int16_t)outValue;
			}
		}
	}

	// since resampling inputs can span input frame boundaries
	// we may need to save up to 3 samples for the next call
	m_audioResampleInputNumber = 0;
	for (int32_t i = SrcBytesToSamples(frameDataLength) - 1; 
	  i >= inIndex; i--) {

		// DEBUG printf("resample save [%d] <- [%d]\n",
		// DEBUG m_audioResampleInputNumber, i);

		memcpy(
			&m_audioResampleInputBuffer[m_audioResampleInputNumber],
			&pIn[i * m_audioSrcChannels],
			m_audioSrcChannels * sizeof(int16_t));

		m_audioResampleInputNumber++;
	}

	m_audioPreEncodingBufferLength += 
		numOut * m_audioSrcChannels * sizeof(int16_t);

	return;
}

void CMediaSource::ForwardEncodedAudioFrames(
	Timestamp baseTimestamp,
	u_int32_t* pNumSamples,
	u_int32_t* pNumFrames)
{
	u_int8_t* pFrame;
	u_int32_t frameLength;
	u_int32_t frameNumSamples;

	(*pNumSamples) = 0;
	(*pNumFrames) = 0;

	while (m_audioEncoder->GetEncodedSamples(
	  &pFrame, &frameLength, &frameNumSamples)) {

		// sanity check
		if (pFrame == NULL || frameLength == 0) {
			break;
		}

		(*pNumSamples) += frameNumSamples;
		(*pNumFrames)++;

		// forward the encoded frame to sinks
		CMediaFrame* pMediaFrame =
			new CMediaFrame(
				m_audioEncoder->GetFrameType(),
				pFrame, 
				frameLength,
				baseTimestamp 
					+ DstSamplesToTicks((*pNumSamples)),
				frameNumSamples,
				m_audioDstSampleRate);
		ForwardFrame(pMediaFrame);
		delete pMediaFrame;
	}
}

void CMediaSource::DoStopAudio()
{
	if (m_audioEncoder) {
		// flush remaining output from audio encoder
		// and forward it to sinks

		m_audioEncoder->EncodeSamples(NULL, 0);

		u_int32_t forwardedSamples;
		u_int32_t forwardedFrames;

		ForwardEncodedAudioFrames(
			m_startTimestamp
				+ DstSamplesToTicks(m_audioDstSampleNumber),
			&forwardedSamples,
			&forwardedFrames);

		m_audioDstSampleNumber += forwardedSamples;
		m_audioDstFrameNumber += forwardedFrames;

		m_audioEncoder->Stop();
		delete m_audioEncoder;
		m_audioEncoder = NULL;
	}

	free(m_audioResampleInputBuffer);
	m_audioResampleInputBuffer = NULL;

	free(m_audioPreEncodingBuffer);
	m_audioPreEncodingBuffer = NULL;

	m_sourceAudio = false;
}