Newest 'ffmpeg' Questions - Stack Overflow

Les articles publiés sur le site

• Problems using FFmpeg / libavfilter for adding overlay to grabbed frames

  21 novembre, par Michael

  On Windows with latest FFmpeg / libav (full build, non-free) a C/C++ app reads YUV420P frames from a frame grabber card.

  A bitmap (BGR24) overlay image from file should be drawn on every frame for the first 20 seconds via libavfilter. First, the BGR24 overlay image becomes converted via format filter to YUV420P. Then the YUV420P frame from frame grabber and the YUV420P overlay frame are pushed into the overlay filter.

  FFmpeg / libavfilter does not report any errors or warnings in console / log. Trying to get the filtered frame out of the graph via av_buffersink_get_frame results in an EAGAIN return code.

  The frames from the frame grabber card are fine, they could become encoded or written to a .yuv file. The overlay frame itself is fine too.

  This is the complete private code (prototype - no style, memory leaks, ...):

  #define __STDC_LIMIT_MACROS
  #define __STDC_CONSTANT_MACROS
  
  #include 
  #include 
  #include 
  
  #include "../fgproto/include/SDL/SDL_video.h"
  #include 
  
  using namespace _DSHOWLIB_NAMESPACE;
  
  #ifdef _WIN32
  //Windows
  extern "C" {
  #include "libavcodec/avcodec.h"
  #include "libavformat/avformat.h"
  #include "libswscale/swscale.h"
  #include "libavdevice/avdevice.h"
  #include "libavfilter/avfilter.h"
  #include log.h>
  #include mem.h>
  #include "libavfilter/buffersink.h"
  #include "libavfilter/buffersrc.h"
  #include "libavutil/opt.h"
  #include "libavutil/hwcontext_qsv.h"
  #include "SDL/SDL.h"
  };
  #endif
  #include 
  #include 
  
  void uSleep(double waitTimeInUs, LARGE_INTEGER frequency)
  {
      LARGE_INTEGER startTime, currentTime;
  
      QueryPerformanceCounter(&startTime);
  
      if (waitTimeInUs > 16500.0)
          Sleep(1);
  
      do
      {
          YieldProcessor();
          //Sleep(0);
          QueryPerformanceCounter(&currentTime);
      }
      while (waitTimeInUs > (currentTime.QuadPart - startTime.QuadPart) * 1000000.0 / frequency.QuadPart);
  }
  
  void check_error(int ret)
  {
      if (ret < 0)
      {
          char errbuf[128];
          int tmp = errno;
          av_strerror(ret, errbuf, sizeof(errbuf));
          std::cerr << "Error: " << errbuf << '\n';
          //exit(1);
      }
  }
  
  bool _isRunning = true;
  
  void swap_uv_planes(AVFrame* frame)
  {
      uint8_t* temp_plane = frame->data[1]; 
      frame->data[1] = frame->data[2]; 
      frame->data[2] = temp_plane; 
  }
  
  typedef struct
  {
      const AVClass* avclass;
  } MyFilterGraphContext;
  
  static constexpr AVClass my_filter_graph_class = 
  {
      .class_name = "MyFilterGraphContext",
      .item_name = av_default_item_name,
      .option = NULL,
      .version = LIBAVUTIL_VERSION_INT,
  };
  
  MyFilterGraphContext* init_log_context()
  {
      MyFilterGraphContext* ctx = static_cast(av_mallocz(sizeof(*ctx)));
  
      if (!ctx)
      {
          av_log(nullptr, AV_LOG_ERROR, "Unable to allocate MyFilterGraphContext\n");
          return nullptr;
      }
  
      ctx->avclass = &my_filter_graph_class;
      return ctx;
  }
  
  int init_overlay_filter(AVFilterGraph** graph, AVFilterContext** src_ctx, AVFilterContext** overlay_src_ctx,
                          AVFilterContext** sink_ctx)
  {
      AVFilterGraph* filter_graph;
      AVFilterContext* buffersrc_ctx;
      AVFilterContext* overlay_buffersrc_ctx;
      AVFilterContext* buffersink_ctx;
      AVFilterContext* overlay_ctx;
      AVFilterContext* format_ctx;
  
      const AVFilter* buffersrc, * buffersink, * overlay_buffersrc, * overlay_filter, * format_filter;
      int ret;
  
      // Create the filter graph
      filter_graph = avfilter_graph_alloc();
      if (!filter_graph)
      {
          fprintf(stderr, "Unable to create filter graph.\n");
          return AVERROR(ENOMEM);
      }
  
      // Create buffer source filter for main video
      buffersrc = avfilter_get_by_name("buffer");
      if (!buffersrc)
      {
          fprintf(stderr, "Unable to find buffer filter.\n");
          return AVERROR_FILTER_NOT_FOUND;
      }
  
      // Create buffer source filter for overlay image
      overlay_buffersrc = avfilter_get_by_name("buffer");
      if (!overlay_buffersrc)
      {
          fprintf(stderr, "Unable to find buffer filter.\n");
          return AVERROR_FILTER_NOT_FOUND;
      }
  
      // Create buffer sink filter
      buffersink = avfilter_get_by_name("buffersink");
      if (!buffersink)
      {
          fprintf(stderr, "Unable to find buffersink filter.\n");
          return AVERROR_FILTER_NOT_FOUND;
      }
  
      // Create overlay filter
      overlay_filter = avfilter_get_by_name("overlay");
      if (!overlay_filter)
      {
          fprintf(stderr, "Unable to find overlay filter.\n");
          return AVERROR_FILTER_NOT_FOUND;
      }
  
      // Create format filter
      format_filter = avfilter_get_by_name("format");
      if (!format_filter)
      {
          fprintf(stderr, "Unable to find format filter.\n");
          return AVERROR_FILTER_NOT_FOUND;
      }
  
      // Initialize the main video buffer source
      char args[512];
  
      // Initialize the overlay buffer source
      snprintf(args, sizeof(args), "video_size=165x165:pix_fmt=bgr24:time_base=1/25:pixel_aspect=1/1"); 
  
      ret = avfilter_graph_create_filter(&overlay_buffersrc_ctx, overlay_buffersrc, nullptr, args, nullptr,
          filter_graph);
  
      if (ret < 0)
      {
          fprintf(stderr, "Unable to create buffer source filter for overlay.\n");
          return ret;
      }
  
      snprintf(args, sizeof(args), "video_size=1920x1080:pix_fmt=yuv420p:time_base=1/25:pixel_aspect=1/1");
  
      ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, nullptr, args, nullptr, filter_graph);
  
      if (ret < 0)
      {
          fprintf(stderr, "Unable to create buffer source filter for main video.\n");
          return ret;
      }
  
      // Initialize the format filter to convert overlay image to yuv420p
      snprintf(args, sizeof(args), "pix_fmts=yuv420p");
  
      ret = avfilter_graph_create_filter(&format_ctx, format_filter, nullptr, args, nullptr, filter_graph);
  
      if (ret < 0)
      {
          fprintf(stderr, "Unable to create format filter.\n");
          return ret;
      }
  
      // Initialize the overlay filter
      ret = avfilter_graph_create_filter(&overlay_ctx, overlay_filter, nullptr, "W-w:H-h:enable='between(t,0,20)':format=yuv420", nullptr, filter_graph);
      if (ret < 0)
      {
          fprintf(stderr, "Unable to create overlay filter.\n");
          return ret;
      }
  
      // Initialize the buffer sink
      ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, nullptr, nullptr, nullptr, filter_graph);
      if (ret < 0)
      {
          fprintf(stderr, "Unable to create buffer sink filter.\n");
          return ret;
      }
  
      // Connect the filters
      ret = avfilter_link(overlay_buffersrc_ctx, 0, format_ctx, 0);
  
      if (ret >= 0)
      {
          ret = avfilter_link(buffersrc_ctx, 0, overlay_ctx, 0);
      }
      else
      {
          fprintf(stderr, "Unable to configure filter graph.\n");
          return ret;
      }
  
  
      if (ret >= 0)
      {
          ret = avfilter_link(format_ctx, 0, overlay_ctx, 1);
      }
      else
      {
          fprintf(stderr, "Unable to configure filter graph.\n");
          return ret;
      }
  
      if (ret >= 0)
      {
          if ((ret = avfilter_link(overlay_ctx, 0, buffersink_ctx, 0)) < 0)
          {
              fprintf(stderr, "Unable to link filter graph.\n");
              return ret;
          }
      }
      else
      {
          fprintf(stderr, "Unable to configure filter graph.\n");
          return ret;
      }
  
      MyFilterGraphContext* log_ctx = init_log_context();
  
      // Configure the filter graph
      if ((ret = avfilter_graph_config(filter_graph, log_ctx)) < 0)
      {
          fprintf(stderr, "Unable to configure filter graph.\n");
          return ret;
      }
  
      *graph = filter_graph;
      *src_ctx = buffersrc_ctx;
      *overlay_src_ctx = overlay_buffersrc_ctx;
      *sink_ctx = buffersink_ctx;
  
      return 0;
  }
  
  int main(int argc, char* argv[])
  {
      unsigned int videoIndex = 0;
  
      avdevice_register_all();
  
      av_log_set_level(AV_LOG_TRACE);
  
      const AVInputFormat* pFrameGrabberInputFormat = av_find_input_format("dshow");
  
      constexpr int frameGrabberPixelWidth = 1920;
      constexpr int frameGrabberPixelHeight = 1080;
      constexpr int frameGrabberFrameRate = 25;
      constexpr AVPixelFormat frameGrabberPixelFormat = AV_PIX_FMT_YUV420P;
  
      char shortStringBuffer[32];
  
      AVDictionary* pFrameGrabberOptions = nullptr;
  
      _snprintf_s(shortStringBuffer, sizeof(shortStringBuffer), "%dx%d", frameGrabberPixelWidth, frameGrabberPixelHeight);
      av_dict_set(&pFrameGrabberOptions, "video_size", shortStringBuffer, 0);
  
      _snprintf_s(shortStringBuffer, sizeof(shortStringBuffer), "%d", frameGrabberFrameRate);
  
      av_dict_set(&pFrameGrabberOptions, "framerate", shortStringBuffer, 0);
      av_dict_set(&pFrameGrabberOptions, "pixel_format", "yuv420p", 0);
      av_dict_set(&pFrameGrabberOptions, "rtbufsize", "128M", 0);
  
      AVFormatContext* pFrameGrabberFormatContext = avformat_alloc_context();
  
      pFrameGrabberFormatContext->flags = AVFMT_FLAG_NOBUFFER | AVFMT_FLAG_FLUSH_PACKETS;
  
      if (avformat_open_input(&pFrameGrabberFormatContext, "video=MZ0380 PCI, Analog 01 Capture",
                              pFrameGrabberInputFormat, &pFrameGrabberOptions) != 0)
      {
          std::cerr << "Couldn't open input stream." << '\n';
          return -1;
      }
  
      if (avformat_find_stream_info(pFrameGrabberFormatContext, nullptr) < 0)
      {
          std::cerr << "Couldn't find stream information." << '\n';
          return -1;
      }
  
      bool foundVideoStream = false;
  
      for (unsigned int loop_videoIndex = 0; loop_videoIndex < pFrameGrabberFormatContext->nb_streams; loop_videoIndex++)
      {
          if (pFrameGrabberFormatContext->streams[loop_videoIndex]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
          {
              videoIndex = loop_videoIndex;
              foundVideoStream = true;
              break;
          }
      }
  
      if (!foundVideoStream)
      {
          std::cerr << "Couldn't find a video stream." << '\n';
          return -1;
      }
  
      const AVCodec* pFrameGrabberCodec = avcodec_find_decoder(
          pFrameGrabberFormatContext->streams[videoIndex]->codecpar->codec_id);
  
      AVCodecContext* pFrameGrabberCodecContext = avcodec_alloc_context3(pFrameGrabberCodec);
  
      if (pFrameGrabberCodec == nullptr)
      {
          std::cerr << "Codec not found." << '\n';
          return -1;
      }
  
      pFrameGrabberCodecContext->pix_fmt = frameGrabberPixelFormat;
      pFrameGrabberCodecContext->width = frameGrabberPixelWidth;
      pFrameGrabberCodecContext->height = frameGrabberPixelHeight;
  
      int ret = avcodec_open2(pFrameGrabberCodecContext, pFrameGrabberCodec, nullptr);
  
      if (ret < 0)
      {
          std::cerr << "Could not open pVideoCodec." << '\n';
          return -1;
      }
  
      const char* outputFilePath = "c:\\temp\\output.mp4";
      constexpr int outputWidth = frameGrabberPixelWidth;
      constexpr int outputHeight = frameGrabberPixelHeight;
      constexpr int outputFrameRate = frameGrabberFrameRate;
  
      SwsContext* img_convert_ctx = sws_getContext(frameGrabberPixelWidth, frameGrabberPixelHeight,
                                                   frameGrabberPixelFormat, outputWidth, outputHeight, AV_PIX_FMT_NV12,
                                                   SWS_BICUBIC, nullptr, nullptr, nullptr);
  
      constexpr double frameTimeinUs = 1000000.0 / frameGrabberFrameRate;
  
      LARGE_INTEGER frequency;
      LARGE_INTEGER lastTime, currentTime;
  
      QueryPerformanceFrequency(&frequency);
      QueryPerformanceCounter(&lastTime);
  
      //SDL----------------------------
  
      if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_TIMER | SDL_INIT_EVENTS))
      {
          printf("Could not initialize SDL - %s\n", SDL_GetError());
          return -1;
      }
  
      SDL_Window* screen = SDL_CreateWindow("3P FrameGrabber SuperApp", SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED,
                                            frameGrabberPixelWidth, frameGrabberPixelHeight,
                                            SDL_WINDOW_RESIZABLE | SDL_WINDOW_OPENGL);
  
      if (!screen)
      {
          printf("SDL: could not set video mode - exiting:%s\n", SDL_GetError());
          return -1;
      }
  
      SDL_Renderer* renderer = SDL_CreateRenderer(screen, -1, 0);
  
      if (!renderer)
      {
          printf("SDL: could not create renderer - exiting:%s\n", SDL_GetError());
          return -1;
      }
  
      SDL_Texture* texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_YV12, SDL_TEXTUREACCESS_STREAMING,
                                               frameGrabberPixelWidth, frameGrabberPixelHeight);
  
      if (!texture)
      {
          printf("SDL: could not create texture - exiting:%s\n", SDL_GetError());
          return -1;
      }
  
      SDL_Event event;
  
      //SDL End------------------------
  
      const AVCodec* pVideoCodec = avcodec_find_encoder_by_name("h264_qsv");
  
      if (!pVideoCodec)
      {
          std::cerr << "Codec not found" << '\n';
          return 1;
      }
  
      AVCodecContext* pVideoCodecContext = avcodec_alloc_context3(pVideoCodec);
  
      if (!pVideoCodecContext)
      {
          std::cerr << "Could not allocate video pVideoCodec context" << '\n';
          return 1;
      }
  
      AVBufferRef* pHardwareDeviceContextRef = nullptr;
  
      ret = av_hwdevice_ctx_create(&pHardwareDeviceContextRef, AV_HWDEVICE_TYPE_QSV,
                                   "PCI\\VEN_8086&DEV_5912&SUBSYS_310217AA&REV_04\\3&11583659&0&10", nullptr, 0);
      check_error(ret);
  
      pVideoCodecContext->bit_rate = static_cast(outputWidth * outputHeight) * 2;
      pVideoCodecContext->width = outputWidth;
      pVideoCodecContext->height = outputHeight;
      pVideoCodecContext->framerate = {outputFrameRate, 1};
      pVideoCodecContext->time_base = {1, outputFrameRate};
      pVideoCodecContext->pix_fmt = AV_PIX_FMT_QSV;
      pVideoCodecContext->max_b_frames = 0;
  
      AVBufferRef* pHardwareFramesContextRef = av_hwframe_ctx_alloc(pHardwareDeviceContextRef);
  
      AVHWFramesContext* pHardwareFramesContext = reinterpret_cast(pHardwareFramesContextRef->data);
  
      pHardwareFramesContext->format = AV_PIX_FMT_QSV;
      pHardwareFramesContext->sw_format = AV_PIX_FMT_NV12;
      pHardwareFramesContext->width = outputWidth;
      pHardwareFramesContext->height = outputHeight;
      pHardwareFramesContext->initial_pool_size = 20;
  
      ret = av_hwframe_ctx_init(pHardwareFramesContextRef);
      check_error(ret);
  
      pVideoCodecContext->hw_device_ctx = nullptr;
      pVideoCodecContext->hw_frames_ctx = av_buffer_ref(pHardwareFramesContextRef);
  
      ret = avcodec_open2(pVideoCodecContext, pVideoCodec, nullptr); //&pVideoOptionsDict);
      check_error(ret);
  
      AVFormatContext* pVideoFormatContext = nullptr;
  
      avformat_alloc_output_context2(&pVideoFormatContext, nullptr, nullptr, outputFilePath);
  
      if (!pVideoFormatContext)
      {
          std::cerr << "Could not create output context" << '\n';
          return 1;
      }
  
      const AVOutputFormat* pVideoOutputFormat = pVideoFormatContext->oformat;
  
      if (pVideoFormatContext->oformat->flags & AVFMT_GLOBALHEADER)
      {
          pVideoCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
      }
  
      const AVStream* pVideoStream = avformat_new_stream(pVideoFormatContext, pVideoCodec);
  
      if (!pVideoStream)
      {
          std::cerr << "Could not allocate stream" << '\n';
          return 1;
      }
  
      ret = avcodec_parameters_from_context(pVideoStream->codecpar, pVideoCodecContext);
  
      check_error(ret);
  
      if (!(pVideoOutputFormat->flags & AVFMT_NOFILE))
      {
          ret = avio_open(&pVideoFormatContext->pb, outputFilePath, AVIO_FLAG_WRITE);
          check_error(ret);
      }
  
      ret = avformat_write_header(pVideoFormatContext, nullptr);
  
      check_error(ret);
  
      AVFrame* pHardwareFrame = av_frame_alloc();
  
      if (av_hwframe_get_buffer(pVideoCodecContext->hw_frames_ctx, pHardwareFrame, 0) < 0)
      {
          std::cerr << "Error allocating a hw frame" << '\n';
          return -1;
      }
  
      AVFrame* pFrameGrabberFrame = av_frame_alloc();
      AVPacket* pFrameGrabberPacket = av_packet_alloc();
  
      AVPacket* pVideoPacket = av_packet_alloc();
      AVFrame* pVideoFrame = av_frame_alloc();
  
      AVFrame* pSwappedFrame = av_frame_alloc();
      av_frame_get_buffer(pSwappedFrame, 32);
  
      INT64 frameCount = 0;
  
      pFrameGrabberCodecContext->time_base = {1, frameGrabberFrameRate};
  
      AVFilterContext* buffersrc_ctx = nullptr;
      AVFilterContext* buffersink_ctx = nullptr;
      AVFilterContext* overlay_src_ctx = nullptr;
      AVFilterGraph* filter_graph = nullptr;
  
      if ((ret = init_overlay_filter(&filter_graph, &buffersrc_ctx, &overlay_src_ctx, &buffersink_ctx)) < 0)
      {
          return ret;
      }
  
      // Load overlay image
      AVFormatContext* overlay_fmt_ctx = nullptr;
      AVCodecContext* overlay_codec_ctx = nullptr;
      const AVCodec* overlay_codec = nullptr;
      AVFrame* overlay_frame = nullptr;
      AVDictionary* overlay_options = nullptr;
  
      const char* overlay_image_filename = "c:\\temp\\overlay.bmp";
  
      av_dict_set(&overlay_options, "video_size", "165x165", 0);
      av_dict_set(&overlay_options, "pixel_format", "bgr24", 0);
  
      if ((ret = avformat_open_input(&overlay_fmt_ctx, overlay_image_filename, nullptr, &overlay_options)) < 0)
      {
          return ret;
      }
  
      if ((ret = avformat_find_stream_info(overlay_fmt_ctx, nullptr)) < 0)
      {
          return ret;
      }
  
      int overlay_video_stream_index = -1;
  
      for (int i = 0; i < overlay_fmt_ctx->nb_streams; i++)
      {
          if (overlay_fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
          {
              overlay_video_stream_index = i;
              break;
          }
      }
  
      if (overlay_video_stream_index == -1)
      {
          return -1;
      }
  
      overlay_codec = avcodec_find_decoder(overlay_fmt_ctx->streams[overlay_video_stream_index]->codecpar->codec_id);
  
      if (!overlay_codec)
      {
          fprintf(stderr, "Overlay codec not found.\n");
          return -1;
      }
  
      overlay_codec_ctx = avcodec_alloc_context3(overlay_codec);
  
      if (!overlay_codec_ctx)
      {
          fprintf(stderr, "Could not allocate overlay codec context.\n");
          return AVERROR(ENOMEM);
      }
  
      avcodec_parameters_to_context(overlay_codec_ctx, overlay_fmt_ctx->streams[overlay_video_stream_index]->codecpar);
  
      if ((ret = avcodec_open2(overlay_codec_ctx, overlay_codec, nullptr)) < 0)
      {
          return ret;
      }
  
      overlay_frame = av_frame_alloc();
  
      if (!overlay_frame)
      {
          fprintf(stderr, "Could not allocate overlay frame.\n");
          return AVERROR(ENOMEM);
      }
  
      AVPacket* overlay_packet = av_packet_alloc();
  
      // Read frames from the file
      while (av_read_frame(overlay_fmt_ctx, overlay_packet) >= 0)
      {
          if (overlay_packet->stream_index == overlay_video_stream_index)
          {
              ret = avcodec_send_packet(overlay_codec_ctx, overlay_packet);
  
              if (ret < 0)
              {
                  break;
              }
  
              ret = avcodec_receive_frame(overlay_codec_ctx, overlay_frame);
              if (ret >= 0)
              {
                  
                  break; // We only need the first frame for the overlay
              }
  
              if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
              {
                  continue;
              }
  
              break;
          }
  
          av_packet_unref(overlay_packet);
      }
  
      av_packet_unref(overlay_packet);
  
      while (_isRunning)
      {
          while (SDL_PollEvent(&event) != 0)
          {
              switch (event.type)
              {
              case SDL_QUIT:
                  _isRunning = false;
                  break;
              case SDL_KEYDOWN:
                  if (event.key.keysym.sym == SDLK_ESCAPE)
                      _isRunning = false;
                  break;
              default: ;
              }
          }
  
          if (av_read_frame(pFrameGrabberFormatContext, pFrameGrabberPacket) == 0)
          {
              if (pFrameGrabberPacket->stream_index == videoIndex)
              {
                  ret = avcodec_send_packet(pFrameGrabberCodecContext, pFrameGrabberPacket);
  
                  if (ret < 0)
                  {
                      std::cerr << "Error sending a packet for decoding!" << '\n';
                      return -1;
                  }
  
                  ret = avcodec_receive_frame(pFrameGrabberCodecContext, pFrameGrabberFrame);
  
                  if (ret != 0)
                  {
                      std::cerr << "Receiving frame failed!" << '\n';
                      return -1;
                  }
  
                  if (ret == AVERROR(EAGAIN) || ret == AVERROR(AVERROR_EOF))
                  {
                      std::cout << "End of stream detected. Exiting now." << '\n';
                      return 0;
                  }
  
                  if (ret != 0)
                  {
                      std::cerr << "Decode Error!" << '\n';
                      return -1;
                  }
  
                  // Feed the frame into the filter graph
                  if (av_buffersrc_add_frame_flags(buffersrc_ctx, pFrameGrabberFrame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0)
                  {
                      fprintf(stderr, "Error while feeding the filtergraph\n");
                      break;
                  }
  
                  // Push the overlay frame to the overlay_src_ctx
                  ret = av_buffersrc_add_frame_flags(overlay_src_ctx, overlay_frame, AV_BUFFERSRC_FLAG_KEEP_REF);
                  if (ret < 0)
                  {
                      fprintf(stderr, "Error while feeding the filtergraph\n");
                      break;
                  }                           
  
                  // Pull filtered frame from the filter graph
                  AVFrame* filtered_frame = av_frame_alloc();
  
                  ret = av_buffersink_get_frame(buffersink_ctx, filtered_frame);
  
                  if (ret < 0)
                  {
                      check_error(ret);
                  }
  
                  QueryPerformanceCounter(&currentTime);
  
                  double elapsedTime = (currentTime.QuadPart - lastTime.QuadPart) * 1000000.0 / frequency.QuadPart;
  
                  if (elapsedTime > 0.0 && elapsedTime < frameTimeinUs)
                  {
                      uSleep(frameTimeinUs - elapsedTime, frequency);
                  }
  
                  SDL_UpdateTexture(texture, nullptr, filtered_frame->data[0], filtered_frame->linesize[0]);
                  SDL_RenderClear(renderer);
                  SDL_RenderCopy(renderer, texture, nullptr, nullptr);
                  SDL_RenderPresent(renderer);
  
                  QueryPerformanceCounter(&lastTime);
  
                  swap_uv_planes(filtered_frame);
  
                  ret = sws_scale_frame(img_convert_ctx, pVideoFrame, filtered_frame);
  
                  if (ret < 0)
                  {
                      std::cerr << "Scaling frame for Intel QS Encoder did fail!" << '\n';
                      return -1;
                  }
  
                  if (av_hwframe_transfer_data(pHardwareFrame, pVideoFrame, 0) < 0)
                  {
                      std::cerr << "Error transferring frame data to hw frame!" << '\n';
                      return -1;
                  }
  
                  pHardwareFrame->pts = frameCount++;
  
                  ret = avcodec_send_frame(pVideoCodecContext, pHardwareFrame);
  
                  if (ret < 0)
                  {
                      std::cerr << "Error sending a frame for encoding" << '\n';
                      check_error(ret);
                  }
  
                  av_packet_unref(pVideoPacket);
  
                  while (ret >= 0)
                  {
                      ret = avcodec_receive_packet(pVideoCodecContext, pVideoPacket);
  
                      if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
                      {
                          break;
                      }
  
                      if (ret < 0)
                      {
                          std::cerr << "Error during encoding" << '\n';
                          return 1;
                      }
  
                      av_packet_rescale_ts(pVideoPacket, pVideoCodecContext->time_base, pVideoStream->time_base);
  
                      pVideoPacket->stream_index = pVideoStream->index;
  
                      ret = av_interleaved_write_frame(pVideoFormatContext, pVideoPacket);
  
                      check_error(ret);
  
                      av_packet_unref(pVideoPacket);
                  }
  
                  av_packet_unref(pFrameGrabberPacket);
                  av_frame_free(&filtered_frame);
              }
          }
      }
  
      av_write_trailer(pVideoFormatContext);
      av_buffer_unref(&pHardwareDeviceContextRef);
      avcodec_free_context(&pVideoCodecContext);
      avio_closep(&pVideoFormatContext->pb);
      avformat_free_context(pVideoFormatContext);
      av_packet_free(&pVideoPacket);
  
      avcodec_free_context(&pFrameGrabberCodecContext);
      av_frame_free(&pFrameGrabberFrame);
      av_packet_free(&pFrameGrabberPacket);
      avformat_close_input(&pFrameGrabberFormatContext);
  
      return 0;
  }
  
  

  The console / log output running the code:

  [in @ 00000288ee494f40] Setting 'video_size' to value '1920x1080'
  [in @ 00000288ee494f40] Setting 'pix_fmt' to value 'yuv420p'
  [in @ 00000288ee494f40] Setting 'time_base' to value '1/25'
  [in @ 00000288ee494f40] Setting 'pixel_aspect' to value '1/1'
  [in @ 00000288ee494f40] w:1920 h:1080 pixfmt:yuv420p tb:1/25 fr:0/1 sar:1/1 csp:unknown range:unknown
  [overlay_in @ 00000288ff1013c0] Setting 'video_size' to value '165x165'
  [overlay_in @ 00000288ff1013c0] Setting 'pix_fmt' to value 'bgr24'
  [overlay_in @ 00000288ff1013c0] Setting 'time_base' to value '1/25'
  [overlay_in @ 00000288ff1013c0] Setting 'pixel_aspect' to value '1/1'
  [overlay_in @ 00000288ff1013c0] w:165 h:165 pixfmt:bgr24 tb:1/25 fr:0/1 sar:1/1 csp:unknown range:unknown
  [format @ 00000288ff1015c0] Setting 'pix_fmts' to value 'yuv420p'
  [overlay @ 00000288ff101880] Setting 'x' to value 'W-w'
  [overlay @ 00000288ff101880] Setting 'y' to value 'H-h'
  [overlay @ 00000288ff101880] Setting 'enable' to value 'between(t,0,20)'
  [overlay @ 00000288ff101880] Setting 'format' to value 'yuv420'
  [auto_scale_0 @ 00000288ff101ec0] w:iw h:ih flags:'' interl:0
  [format @ 00000288ff1015c0] auto-inserting filter 'auto_scale_0' between the filter 'overlay_in' and the filter 'format'
  [auto_scale_1 @ 00000288ee4a4cc0] w:iw h:ih flags:'' interl:0
  [overlay @ 00000288ff101880] auto-inserting filter 'auto_scale_1' between the filter 'format' and the filter 'overlay'
  [AVFilterGraph @ 00000288ee495c80] query_formats: 5 queried, 6 merged, 6 already done, 0 delayed
  [auto_scale_0 @ 00000288ff101ec0] w:165 h:165 fmt:bgr24 csp:gbr range:pc sar:1/1 -> w:165 h:165 fmt:yuv420p csp:unknown range:unknown sar:1/1 flags:0x00000004
  [auto_scale_1 @ 00000288ee4a4cc0] w:165 h:165 fmt:yuv420p csp:unknown range:unknown sar:1/1 -> w:165 h:165 fmt:yuva420p csp:unknown range:unknown sar:1/1 flags:0x00000004
  [overlay @ 00000288ff101880] main w:1920 h:1080 fmt:yuv420p overlay w:165 h:165 fmt:yuva420p
  [overlay @ 00000288ff101880] [framesync @ 00000288ff1019a8] Selected 1/25 time base
  [overlay @ 00000288ff101880] [framesync @ 00000288ff1019a8] Sync level 2
  

  I tried to change the index / order of how the two different frames become pushed into the filter graph. Once I got a frame out of the graph but with the dimensions of the overlay image, not with the dimensions of the grabbed frame from the grabber card. So I suppose I am doing something wrong building up the filter graph.

  To verify that the FFmpeg build contains all necessary modules I ran that procedure via FFmpeg executable in console and it worked and the result was as expected.

  The command-line producing the expected output is following:

  ffmpeg -f dshow -i video="MZ0380 PCI, Analog 01 Capture" -video_size 1920x1080 -framerate 25 -pixel_format yuv420p -loglevel debug -i "C:\temp\overlay.bmp" -filter_complex "[0:v][1:v] overlay=W-w:H-h:enable='between(t,0,20)'" -pix_fmt yuv420p -c:a copy output.mp4
  

• How do you convert fragmented MP4 to MP4 with ffmpeg and keep all fragments ?

  21 novembre, par Lars Rönnbäck

  I am trying to convert a fragmented MP4 (fMP4) to a single MP4 file using ffmpeg. I expected the following command to work, but it only results in the initialization segment and the last three segments being processed by ffmpeg. I am using version 6.1.1 of ffmpeg.

  ffmpeg -i playlist.m3u8 -c copy complete.mp4

  The content of the playlist is as follows.

  #EXT-X-VERSION:7
  #EXT-X-TARGETDURATION:2
  #EXT-X-MEDIA-SEQUENCE:0
  #EXT-X-PLAYLIST-TYPE:EVENT
  #EXT-X-MAP:URI="segment_00000.mp4"
  #EXTINF:0.525000,
  segment_00001.m4s
  #EXTINF:2.000000,
  segment_00002.m4s
  #EXTINF:1.933333,
  segment_00003.m4s
  #EXTINF:2.001667,
  segment_00004.m4s
  #EXTINF:2.000000,
  segment_00005.m4s
  #EXTINF:2.000000,
  segment_00006.m4s
  #EXTINF:2.000000,
  segment_00007.m4s
  #EXTINF:2.000000,
  segment_00008.m4s
  #EXTINF:2.000000,
  segment_00009.m4s
  #EXTINF:2.000000,
  segment_00010.m4s
  

  The output from the command shows which segments are processed, and it is skipping all but the three last segments. The results are the same if I try other playlists with more or less segments. It is always init + the last three.

  [hls @ 0x7f497b312600] Opening 'segment_00000.mp4' for reading
  [hls @ 0x7f497b312600] Opening 'segment_00008.m4s' for reading
  [hls @ 0x7f497b312600] Opening 'segment_00009.m4s' for reading
  [hls @ 0x7f497b312600] Opening 'segment_00010.m4s' for reading
  

  Is there anything I can do to force ffmpeg to process all the referenced files in the playlist?

• Ffmpeg for use in iOS application coded in Swift

  21 novembre, par Xavi Font

  I have been browsing the web, even this website... bu cannot find a good option to implement ffmpeg functionality in an iOS application made in swift.

  Options looked at and reasons why they are not solutions:

  SwiftFFmpeg - I am not sure it can run on iOS, plus I don't see an option to run my desired Ffmpeg command.

  MobileFFmpeg - Not maintained anymore, and a new project by the same founders created a "better altenrative" called ffmpeg-kit.

  ffmpeg-kit - looks amazing, but their API only allows for interaction in the Objective-C language.

  Any solutions anyone can give me?

• How to Record a Seekable Video File with Lossless Compression wich is seekable while recording ?

  21 novembre, par Christian Schaffner

  I am using FFmpeg to record a dshow device to a .mkv file. My goal is to use a lossless codec like H.264 or FFV1 for the best quality. Here’s the issue I’m encountering:

  In my application, the recorded file needs to be played in another application while recording is still in progress. While playback works, the file is not seekable because the index is written only after the recording ends.

  My question:
  Is there any way to record a video that remains seekable, even if the recording process is incomplete? I am open to low-filesize, uncompressed formats or other practical solutions that meet these requirements.

  Here’s my current FFmpeg command:

  ffmpeg -f dshow -i video="Your Video Device" -c:v libx264 -preset ultrafast -crf 0 output.mkv

  I've also tried FFV1 as the codec, but the seekability issue persists:

  ffmpeg -f dshow -i video="Your Video Device" -c:v ffv1 output.mkv

  Things I’ve Tried:

  1. Using the -movflags +faststart option (applicable to MP4, but not for MKV).
  2. Switching container formats to see if other containers might write the index during recording (e.g., AVI or MP4).
  3. Exploring streaming protocols (e.g., udp or rtmp) for playback but found these unsuitable for my use case.

  Requirements:

  • Lossless video quality (e.g., H.264 lossless or FFV1).
  • Seekable playback during recording.

  Examples

  1. How to record a seekable H.264 lossless video while still recording?
  2. Can FFV1 (or similar codecs) support live seekable playback during recording?

  Does FFmpeg or any container format have a feature or flag that could solve this problem? Or are there alternative workflows or tools that can help? Any help or suggestions would be appreciated!

• Is Replacing Dynamic Resolution with scale_amf in FFmpeg Command a Good Direction ?

  21 novembre, par fred

  I'm working on a Lua script for MPV that processes 360-degree videos using FFmpeg's v360 filter. The original command dynamically calculates the output resolution based on a res variable, like this:

  mp.command_native_async({
      "no-osd", "vf", "add", 
      string.format(
          "@vrrev:v360=%s:%s:reset_rot=1:in_stereo=%s:out_stereo=2d:
  id_fov=%s:d_fov=%.3f:yaw=%.3f:pitch=%s:roll=%.3f:
  w=%s*192.0:h=%.3f*108.0:h_flip=%s:interp=%s",
          in_flip, inputProjection, outputProjection, in_stereo, idfov, dfov, yaw, pitch, roll, res, res, h_flip, scaling
      )
  }, updateComplete)
  

  Change Proposal :

  I am considering replacing the dynamic width and height calculations with a scale_amf filter to handle scaling more efficiently and leverage GPU acceleration. The updated command would look like this:

  mp.command_native_async({
      "no-osd", "vf", "add", 
      string.format(
          "@vrrev:v360=%s:%s:reset_rot=1:in_stereo=%s:out_stereo=2d:
  id_fov=%s:d_fov=%.3f:yaw=%.3f:pitch=%s:roll=%.3f,
  %sscale_amf=w=%.0f:h=%.0f",
          inputProjection, outputProjection, in_stereo, idfov, dfov, yaw, pitch, roll, in_flip, res * 192.0, res * 108.0
      )
  

  Hardware Specifications :

  I am using an AMD Ryzen 5 5600G, no display card, 16GB RAM, Windows 10.

  Questions :

  Is using scale_amf for scaling a good direction in terms of performance and efficiency? Are there any potential drawbacks to this approach that I should be aware of? How does using scale_amf compare to the original dynamic resolution method in terms of output quality and processing speed? }, updateComplete)

  Any insights or experiences with this change would be greatly appreciated!