file_mp4_recorder.cpp

完整的RTP RTSP代码库

}

void CMp4Recorder::ProcessEncodedAudioFrame (CMediaFrame *pFrame)
{
  if (m_audioFrameType == AMRNBAUDIOFRAME ||
      m_audioFrameType == AMRWBAUDIOFRAME) {
    uint8_t decMode = *(uint8_t *)pFrame->GetData();
    m_amrMode |= 1 << ((decMode >> 3) & 0xf);
  }

    if (m_audioFrameNumber == 1) {
      m_audioStartTimestamp = pFrame->GetTimestamp();
      debug_message("record audio start "U64, m_audioStartTimestamp);
      m_canRecordVideo = true;
      m_prevAudioFrame = pFrame;
      m_audioSamples = 0;
      m_audioDiffTicks = 0;
      m_audioDiffTicksTotal = 0;
      m_audioFrameNumber++;
      return; // wait until the next audio frame
    }

    Duration thisFrameDurationInTicks = 
      pFrame->GetTimestamp() - m_prevAudioFrame->GetTimestamp();

    Duration thisFrameDurationInSamples =
      GetTimescaleFromTicks(thisFrameDurationInTicks, m_audioTimeScale);

    Duration elapsedTimeFromTimestamp;
    elapsedTimeFromTimestamp = pFrame->GetTimestamp() - m_audioStartTimestamp;

    if (thisFrameDurationInSamples > pFrame->GetDuration()) {
      // we have a gap frame
      // add the number of extra samples
      m_audioSamples += 
	(thisFrameDurationInSamples - pFrame->GetDuration());
      error_message("adding encoded "D64" samples", 
		    thisFrameDurationInSamples - pFrame->GetDuration());
    }
    // we have a consecutive frame
    // Timestamp of pFrame should reflect encoded audio samples
    // take difference
    m_audioSamples += pFrame->GetDuration();
    /*
     * we don't convert the audio duration any more
     *  m_prevEncodedAudioFrame->SetDuration(audioDurationInSamples);
     * Instead, we'll convert the video duration
     */
    MP4WriteSample(
                   m_mp4File,
                   m_audioTrackId,
                   (u_int8_t*)m_prevAudioFrame->GetData(), 
                   m_prevAudioFrame->GetDataLength(),
                   m_prevAudioFrame->ConvertDuration(m_audioTimeScale));

    m_audioFrameNumber++;
    if (m_prevAudioFrame->RemoveReference()) {
      delete m_prevAudioFrame;
    }
    m_prevAudioFrame = pFrame;
}

/******************************************************************************
 * Process encoded video frame
 ******************************************************************************/
static uint32_t write_sei (uint8_t *to, const uint8_t *from, uint32_t len)
{
  uint8_t *nal_write = to + 4;
  uint32_t written_bytes = 1;
#ifdef DEBUG_H264
  debug_message("sei start %u", len);
#endif
  *nal_write++ = *from++;
  len--;
  
  uint32_t payload_type, payload_size, typesize, sizesize;
  while (len >= 2 && *from != 0x80) {
    payload_type = h264_read_sei_value(from, &typesize);
    payload_size = h264_read_sei_value(from + typesize, &sizesize);
#ifdef DEBUG_H264
    debug_message("type %u sizes %u", payload_type, payload_size);
#endif
    uint32_t size = typesize + sizesize + payload_size;
    if (size <= len) {
      switch (payload_type) {
      case 3:
      case 10:
      case 11:
      case 12:
	// need to skip these
	break;
      default:
	memmove(nal_write, from, size);
	nal_write += size;
	written_bytes += size;
      }
      from += size;
      len -= size;
    } else {
      len = 0;
    }
  }
  if (written_bytes <= 1) {
    return 0;
  } 
  to[0] = (written_bytes >> 24) & 0xff;
  to[1] = (written_bytes >> 16) & 0xff;
  to[2] = (written_bytes >> 8) & 0xff;
  to[3] = written_bytes & 0xff;
#ifdef DEBUG_H264
  debug_message("sei - %u bytes", written_bytes + 4);
#endif
  return written_bytes + 4;
}

void CMp4Recorder::WriteH264Frame (CMediaFrame *pFrame,
				   Duration dur)
{
  bool isIFrame = false;
  Duration rend_offset = 0;
  rend_offset = pFrame->GetPtsTimestamp() - 
    pFrame->GetTimestamp();
  rend_offset = GetTimescaleFromTicks(rend_offset, m_movieTimeScale);

  h264_media_frame_t *mf = (h264_media_frame_t *)pFrame->GetData();
  
  uint32_t size = mf->buffer_len + (4 * mf->nal_number);
  if (size > m_videoTempBufferSize) {
    m_videoTempBuffer = (uint8_t *)realloc(m_videoTempBuffer, size);
    m_videoTempBufferSize = size;
  }
  uint32_t len_written = 0;
  for (uint32_t ix = 0; ix < mf->nal_number; ix++) {
    bool write_it = false;
    switch (mf->nal_bufs[ix].nal_type) {
    case H264_NAL_TYPE_SEI:
      len_written += write_sei(m_videoTempBuffer + len_written,
			       mf->buffer + mf->nal_bufs[ix].nal_offset,
			       mf->nal_bufs[ix].nal_length);
      break;
    case H264_NAL_TYPE_SEQ_PARAM: 
      if (mf->nal_bufs[ix].nal_length != m_videoH264SeqSize ||
	  (m_videoH264Seq != NULL &&
	   memcmp(m_videoH264Seq, 
		  mf->buffer + mf->nal_bufs[ix].nal_offset,
		  m_videoH264SeqSize) != 0)) {
	m_videoH264SeqSize = mf->nal_bufs[ix].nal_length;
	m_videoH264Seq = 
	  (uint8_t *)realloc(m_videoH264Seq, m_videoH264SeqSize);
	memcpy(m_videoH264Seq, 
	       mf->buffer + mf->nal_bufs[ix].nal_offset,
	       m_videoH264SeqSize);
	MP4AddH264SequenceParameterSet(m_mp4File,
				       m_videoTrackId,
				       m_videoH264Seq,
				       m_videoH264SeqSize);
	debug_message("writing seq parameter %u",mf->nal_bufs[ix].nal_length);
      }
      break;
    case H264_NAL_TYPE_PIC_PARAM:
      if (mf->nal_bufs[ix].nal_length != m_videoH264PicSize ||
	  (m_videoH264Pic != NULL &&
	   memcmp(m_videoH264Pic, 
		  mf->buffer + mf->nal_bufs[ix].nal_offset,
		  m_videoH264PicSize) != 0)) {
	m_videoH264PicSize = mf->nal_bufs[ix].nal_length;
	m_videoH264Pic = 
	  (uint8_t *)realloc(m_videoH264Pic, m_videoH264PicSize);
	memcpy(m_videoH264Pic, 
	       mf->buffer + mf->nal_bufs[ix].nal_offset,
	       m_videoH264PicSize);
	MP4AddH264PictureParameterSet(m_mp4File,
				      m_videoTrackId, 
				      m_videoH264Pic,
				      m_videoH264PicSize);
	debug_message("writing pic parameter %u", mf->nal_bufs[ix].nal_length);
      }
      break;
    case H264_NAL_TYPE_IDR_SLICE:
      isIFrame = true;
      // fall through
    case H264_NAL_TYPE_NON_IDR_SLICE:
    case H264_NAL_TYPE_DP_A_SLICE:
    case H264_NAL_TYPE_DP_B_SLICE:
    case H264_NAL_TYPE_DP_C_SLICE:
#if 0
      if (m_videoFrameNumber % 7 == 0) {
	debug_message("drop frame");
	return;
      }
#endif
      write_it = true;
      break;
    case H264_NAL_TYPE_FILLER_DATA:
      write_it = false;
    default:
      write_it = true;
      break;
    }
#ifdef DEBUG_H264
    debug_message("%u h264 nal %d type %d %u write %d", 
		  m_videoFrameNumber, ix, mf->nal_bufs[ix].nal_type, 
		  mf->nal_bufs[ix].nal_length,
		  write_it);
#endif
    if (write_it) {
      // write length.
      uint32_t to_write;
      to_write = mf->nal_bufs[ix].nal_length;
      m_videoTempBuffer[len_written] = (to_write >> 24) & 0xff;
      m_videoTempBuffer[len_written + 1] = (to_write >> 16) & 0xff;
      m_videoTempBuffer[len_written + 2] = (to_write >> 8) & 0xff;
      m_videoTempBuffer[len_written + 3] = to_write & 0xff;
      len_written += 4;
      memcpy(m_videoTempBuffer + len_written,
	     mf->buffer + mf->nal_bufs[ix].nal_offset,
	     to_write);
      len_written += to_write;
    }
  }
#ifdef DEBUG_H264
  debug_message("%u h264 write %u", m_videoFrameNumber, len_written);
#endif
  MP4WriteSample(m_mp4File, 
		 m_videoTrackId,
		 m_videoTempBuffer, 
		 len_written,
		 dur, 
		 rend_offset, 
		 isIFrame);
}

void CMp4Recorder::ProcessEncodedVideoFrame (CMediaFrame *pFrame)
{
    // we drop encoded video frames until we get the first encoded audio frame
    // after that we drop encoded video frames until we get the first I frame.
    // we then stretch this I frame to the start of the first encoded audio
    // frame and write it to the encoded video track
    bool isIFrame = false;
    uint8_t *pData, *pDataStart;
    uint32_t dataLen;

    if (m_videoFrameNumber == 1) {
      // wait until first audio frame before looking for the next I frame
      if (!m_canRecordVideo) {
        if (pFrame->RemoveReference()) delete pFrame;
        return;
      }

      // make sure this frame was captured after the first audio frame
      if (m_recordAudio &&
	  pFrame->GetTimestamp() < m_audioStartTimestamp) {
        if (pFrame->RemoveReference()) delete pFrame;
        return;
      }

      dataLen = pFrame->GetDataLength();
      pDataStart = (uint8_t *)pFrame->GetData();
      if (pFrame->GetType() == MPEG4VIDEOFRAME) {
	pData = MP4AV_Mpeg4FindVop(pDataStart, dataLen);
	if (pData == NULL) {
	  error_message("Couldn't find vop header");
	  if (pFrame->RemoveReference()) delete pFrame;
	  return;
	}
	int voptype =
	    MP4AV_Mpeg4GetVopType(pData,
				  dataLen - (pData - pDataStart));
	if (voptype != VOP_TYPE_I) {
	  debug_message(U64" wrong vop type %d %02x %02x %02x %02x %02x", 
			pFrame->GetTimestamp(),
			voptype,
			pData[0],
			pData[1],
			pData[2],
			pData[3],
			pData[4]);
	  if (pFrame->RemoveReference()) delete pFrame;
	  return;
	}
      } else if (pFrame->GetType() == H263VIDEOFRAME) {
	// wait for an i frame
	if ((pDataStart[4] & 0x02) != 0) {
	  if (pFrame->RemoveReference()) delete pFrame;
	  return;
	}
      } else if (pFrame->GetType() == H264VIDEOFRAME) {
	h264_media_frame_t *mf = (h264_media_frame_t *)pFrame->GetData();
	bool found_idr = false;
	for (uint32_t ix = 0;
	     found_idr == false && ix < mf->nal_number;
	     ix++) {
	  found_idr = mf->nal_bufs[ix].nal_type == H264_NAL_TYPE_IDR_SLICE;
	}
#ifdef DEBUG_H264
	debug_message("h264 nals %d found %d", 
		      mf->nal_number, found_idr);
#endif
	if (found_idr == false) {
	  if (pFrame->RemoveReference()) delete pFrame;
	  return;
	}
      } else {
	// MPEG2 video
	int ret, ftype;
	ret = MP4AV_Mpeg3FindPictHdr(pDataStart, dataLen, &ftype);
	if (ret < 0 || ftype != 1) {
	  if (pFrame->RemoveReference()) delete pFrame;
	  return;
	}
      }

      debug_message("Video start ts "U64, pFrame->GetTimestamp());
	if (m_recordAudio) {
	// reset this timestamp to video's beginning
	  pFrame->SetTimestamp(m_audioStartTimestamp);
      } 
      m_videoStartTimestamp = pFrame->GetTimestamp();
      m_prevVideoFrame = pFrame;
      m_videoFrameNumber++;
      m_videoDurationTimescale = 0;
      return; // wait until the next video frame
    }

    Duration videoDurationInTicks;

    videoDurationInTicks = 
      pFrame->GetTimestamp() - m_videoStartTimestamp;

    // at this point, we'll probably want to add in the audio drift values, 
    // if there are any
    Duration videoDurationInTimescaleTotal;
    videoDurationInTimescaleTotal = 
      GetTimescaleFromTicks(videoDurationInTicks, m_videoTimeScale);

    Duration videoDurationInTimescaleFrame;
    videoDurationInTimescaleFrame = 
      videoDurationInTimescaleTotal - m_videoDurationTimescale;

    m_videoDurationTimescale += videoDurationInTimescaleFrame;
#if 0
    debug_message("vdit "D64" vdits "D64" frame "D64" total "D64" "D64,
		  videoDurationInTicks, videoDurationInTimescaleTotal,
		  videoDurationInTimescaleFrame, m_videoDurationTimescale,
		  GetTicksFromTimescale(m_videoDurationTimescale, 0, 0, m_videoTimeScale));
#endif
    dataLen = m_prevVideoFrame->GetDataLength();

    
    if (pFrame->GetType() == H264VIDEOFRAME) {
      WriteH264Frame(m_prevVideoFrame, videoDurationInTimescaleFrame);
    } else {
      Duration rend_offset = 0;
      if (pFrame->GetType() == MPEG4VIDEOFRAME) {
	pData = MP4AV_Mpeg4FindVop((uint8_t *)m_prevVideoFrame->GetData(),
				   dataLen);
	if (pData) {
	  dataLen -= (pData - (uint8_t *)m_prevVideoFrame->GetData());
	  int vop_type = MP4AV_Mpeg4GetVopType(pData,dataLen);
	  isIFrame = (vop_type == VOP_TYPE_I);
	  rend_offset = m_prevVideoFrame->GetPtsTimestamp() - 
	    m_prevVideoFrame->GetTimestamp();
	  if (rend_offset != 0 && vop_type != VOP_TYPE_B) {
	    rend_offset = GetTimescaleFromTicks(rend_offset, m_movieTimeScale);
	  }
#if 0
	  debug_message("record type %d %02x %02x %02x %02x",
			MP4AV_Mpeg4GetVopType(pData, dataLen),
			pData[0],
			pData[1],
			pData[2],
			pData[3]);
#endif
	} else {
	  pData = (uint8_t *)m_prevVideoFrame->GetData();
	}
      } else if (pFrame->GetType() == H263VIDEOFRAME) {
	pData = (uint8_t *)m_prevVideoFrame->GetData();
	isIFrame = ((pData[4] & 0x02) == 0);
#if 0
	isIFrame =
	  (MP4AV_Mpeg4GetVopType(pData,dataLen) == VOP_TYPE_I);
	debug_message("frame %02x %02x %02x %d", pData[2], pData[3], pData[4],
		      isIFrame);
#endif
      } else {
	// mpeg2
	int ret, ftype;
	pData = (uint8_t *)m_prevVideoFrame->GetData();
	ret = MP4AV_Mpeg3FindPictHdr(pData, dataLen, &ftype);
	isIFrame = false;
	if (ret >= 0) {
	  if (ftype == 1) isIFrame = true;
	  if (ftype != 3) {
#if 1
	    rend_offset = m_prevVideoFrame->GetPtsTimestamp() - 
	      m_prevVideoFrame->GetTimestamp();
	    rend_offset = GetTimescaleFromTicks(rend_offset, m_movieTimeScale);
#else
	    rend_offset = 
	      GetTimescaleFromTicks(m_prevVideoFrame->GetPtsTimestamp(),
				    m_movieTimeScale);
	    rend_offset -= 
	      GetTimescaleFromTicks(m_prevVideoFrame->GetTimestamp(),
				    m_movieTimeScale);
#endif
	  }
	}
      }

      MP4WriteSample(
		     m_mp4File,
		     m_videoTrackId,
		     pData,
		     dataLen,
		     videoDurationInTimescaleFrame,
		     rend_offset,
		     isIFrame);
    }
		
    m_videoFrameNumber++;
    if (m_prevVideoFrame->RemoveReference()) {
      delete m_prevVideoFrame;
    }
    m_prevVideoFrame = pFrame;
}

void CMp4Recorder::ProcessEncodedTextFrame (CMediaFrame *pFrame)
{
  if (m_textFrameNumber == 1) {
    if (m_prevTextFrame != NULL) {