I'm writing a screen recorder that works on different platform and I stuck on the MacOs version.

Here there is the snippet of code that does the video encoding

void ScreenRecorder::StartEncode()
{


  int ret;
  AVFrame *inputFrame =  av_frame_alloc();

  AVPacket *inputPacket = av_packet_alloc();
  
  AVFrame *outputFrame =  av_frame_alloc();
  AVPacket *outputPacket = av_packet_alloc();

  uint64_t audioFrameCount = 0;
  uint64_t videoFrameCount = 0;

  int64_t nextVideoPTS = 0, nextAudioPTS = 0;
  auto src_pix_fmt = correct_for_deprecated_pixel_format(videoInCodecCtx->pix_fmt);
  auto swsContext = sws_getContext(width, height,src_pix_fmt, width, height, PIX_SWS_CONTEXT, SWS_BICUBIC, nullptr,
                                   nullptr, nullptr);


  while (isRun)
  {


    int choice = 0;
    if (videoInFormatCtx && audioInFormatCtx)
      choice = av_compare_ts(nextVideoPTS, videoOutStream->time_base, nextAudioPTS, audioOutStream->time_base);
    else if (videoInFormatCtx)
      choice = -1;
    else if (audioInFormatCtx)
      choice = 1;

    if (choice == -1)
    {
      // Video packet
      av_read_frame(videoInFormatCtx, inputPacket);
      avcodec_send_packet(videoInCodecCtx, inputPacket);
      avcodec_receive_frame(videoInCodecCtx, inputFrame);

      outputFrame->format = PIX_OUTPUT_FMT;
      outputFrame->width = width;
      outputFrame->height = height;
      ret = av_frame_get_buffer(outputFrame, 0);
      if (ret < 0)
      {
        throw std::runtime_error("Unable to allocate video frame");
      }

      sws_scale(swsContext, inputFrame->data, inputFrame->linesize, 0, height, outputFrame->data, outputFrame->linesize);
      outputFrame->pts = videoFrameCount   * videoOutStream->time_base.den / framerate;

      avcodec_send_frame(videoOutCodecCtx, outputFrame);
      if (avcodec_receive_packet(videoOutCodecCtx, outputPacket) == AVERROR(EAGAIN))
        continue;

      outputPacket->stream_index = videoOutStream->index;
      outputPacket->duration = 0; //videoOutStream->time_base.den / 30;
      outputPacket->dts = outputPacket->pts = videoFrameCount * videoOutStream->time_base.den / framerate;
      // std::cerr << "\tVideo::PTS (" << outputFrame->pts << ") timebase " << videoOutStream->time_base.num << "/" << videoOutStream->time_base.den << " real: " << (outputPacket->pts / (double)videoOutStream->time_base.den) << std::endl;
      nextVideoPTS = outputFrame->pts;
      av_interleaved_write_frame(outFormatCtx, outputPacket);
      //av_write_frame(outFormatCtx, outputPacket);

      av_frame_unref(inputFrame);
      av_packet_unref(inputPacket);

      av_frame_unref(outputFrame);
      av_packet_unref(outputPacket);
    }
    else
    {
      //  decoding
      ret = av_read_frame(audioInFormatCtx, inputPacket);
      if (ret < 0)
      {
        throw std::runtime_error("can not read frame");
      }
      ret = avcodec_send_packet(audioInCodecCtx, inputPacket);
      if (ret < 0)
      {
        throw std::runtime_error("can not send pkt in decoding");
      }
      ret = avcodec_receive_frame(audioInCodecCtx, inputFrame);
      if (ret < 0)
      {
        throw std::runtime_error("can not receive frame in decoding");
      }
      //--------------------------------
      // encoding

      uint8_t **cSamples = nullptr;
      ret = av_samples_alloc_array_and_samples(&cSamples, NULL, audioOutCodecCtx->channels, inputFrame->nb_samples, requireAudioFmt, 0);
      if (ret < 0)
      {
        throw std::runtime_error("Fail to alloc samples by av_samples_alloc_array_and_samples.");
      }
      ret = swr_convert(audioConverter, cSamples, inputFrame->nb_samples, (const uint8_t **)inputFrame->extended_data, inputFrame->nb_samples);
      if (ret < 0)
      {
        throw std::runtime_error("Fail to swr_convert.");
      }
      if (av_audio_fifo_space(audioFifo) < inputFrame->nb_samples)
      {
        throw std::runtime_error("audio buffer is too small.");
      }

      ret = av_audio_fifo_write(audioFifo, (void **)cSamples, inputFrame->nb_samples);
      if (ret < 0)
      {
        throw std::runtime_error("Fail to write fifo");
      }

      av_freep(&cSamples[0]);

      av_frame_unref(inputFrame);
      av_packet_unref(inputPacket);

      while (av_audio_fifo_size(audioFifo) >= audioOutCodecCtx->frame_size)
      {
        AVFrame *outputFrame = av_frame_alloc();
        outputFrame->nb_samples = audioOutCodecCtx->frame_size;
        outputFrame->channels = audioInCodecCtx->channels;
        outputFrame->channel_layout = av_get_default_channel_layout(audioInCodecCtx->channels);
        outputFrame->format = requireAudioFmt;
        outputFrame->sample_rate = audioOutCodecCtx->sample_rate;

        ret = av_frame_get_buffer(outputFrame, 0);
        assert(ret >= 0);
        ret = av_audio_fifo_read(audioFifo, (void **)outputFrame->data, audioOutCodecCtx->frame_size);
        assert(ret >= 0);

        outputFrame->pts = audioFrameCount * audioOutStream->time_base.den * audioOutCodecCtx->frame_size / audioOutCodecCtx->sample_rate;

        ret = avcodec_send_frame(audioOutCodecCtx, outputFrame);
        if (ret < 0)
        {
          throw std::runtime_error("Fail to send frame in encoding");
        }
        av_frame_free(&outputFrame);
        ret = avcodec_receive_packet(audioOutCodecCtx, outputPacket);
        if (ret == AVERROR(EAGAIN))
        {
          continue;
        }
        else if (ret < 0)
        {
          throw std::runtime_error("Fail to receive packet in encoding");
        }

        outputPacket->stream_index = audioOutStream->index;
        outputPacket->duration = audioOutStream->time_base.den * audioOutCodecCtx->frame_size / audioOutCodecCtx->sample_rate;
        outputPacket->dts = outputPacket->pts = audioFrameCount * audioOutStream->time_base.den * audioOutCodecCtx->frame_size / audioOutCodecCtx->sample_rate;

        audioFrameCount  ;
        nextAudioPTS = outputPacket->pts;
        // std::cerr << "Audio::PTS (" << nextAudioPTS << ") timebase " << audioOutStream->time_base.num << "/" << audioOutStream->time_base.den << " real: " << (outputPacket->pts / (double)videoOutStream->time_base.den) << std::endl;

        ret = av_write_frame(outFormatCtx, outputPacket);
        av_packet_unref(outputPacket);
        av_frame_unref(outputFrame);
      }
    }
  }

  av_packet_free(&inputPacket);
  av_packet_free(&outputPacket);
  av_frame_free(&inputFrame);
  av_frame_free(&outputFrame);
  sws_freeContext(swsContext);

  printf("encode %lu audio packets in total.\n", audioFrameCount);
}

When I start recording I get the error * [swscaler] bad src pointers * and the result is an all green video except for the first frame ever which is the window I was recording. After some debugging I noticed that the problem is in avcodec_receive_frame (videoInCodecCtx, inputFrame) which doesn't correctly set the inputFrame-> data pointer which is all NULL, but I don't know how to fix it, any idea?

CodePudding user response：

I thought the problem was initially in the avcodec_receive_frame but instead was in av_read_frame which returned EAGAIN.

Now I fix by checking this error with an if-statement and when EAGAIN is returned I do a simple continue. Now the recording is smoothly!