Recherche avancée

Sur d’autres sites (5647)

• increasing memory occupancy while recording screen save to disk with ffmpeg [on hold]

  19 mai 2016, par vbtang

  wonder if there are any resoures i didn’t free ?or do i need to do something special so that i can free these momery ?

  ps. run the demo step by step, and found it has the same problem, until it quit the main function, it still have 40MB memory occupancy. And i found the memory increase obviously in the screen capture thread,but when it increased to about 150MB, it won’t increase, until i quit the program, it will have 40MB memory left. i feel confused.

  pss. i download ffmpeg dev and shared version form here https://ffmpeg.zeranoe.com/builds/,it seems a dll of debug version ? do i need a release version ?
  here is my demo code.

      #include "stdafx.h"
  
  
  
  #ifdef  __cplusplus
  
  extern "C"
  
  {
  
  #endif
  
  #include "libavcodec/avcodec.h"
  
  #include "libavformat/avformat.h"
  
  #include "libswscale/swscale.h"
  
  #include "libavdevice/avdevice.h"
  
  #include "libavutil/audio_fifo.h"
  
  
  
  #pragma comment(lib, "avcodec.lib")
  
  #pragma comment(lib, "avformat.lib")
  
  #pragma comment(lib, "avutil.lib")
  
  #pragma comment(lib, "avdevice.lib")
  
  #pragma comment(lib, "avfilter.lib")
  
  
  
  //#pragma comment(lib, "avfilter.lib")
  
  //#pragma comment(lib, "postproc.lib")
  
  //#pragma comment(lib, "swresample.lib")
  
  #pragma comment(lib, "swscale.lib")
  
  #ifdef __cplusplus
  
  };
  
  #endif
  
  
  
  AVFormatContext *pFormatCtx_Video = NULL, *pFormatCtx_Audio = NULL, *pFormatCtx_Out = NULL;
  
  AVCodecContext  *pCodecCtx_Video;
  
  AVCodec         *pCodec_Video;
  
  AVFifoBuffer    *fifo_video = NULL;
  
  AVAudioFifo     *fifo_audio = NULL;
  
  int VideoIndex, AudioIndex;
  
  
  
  CRITICAL_SECTION AudioSection, VideoSection;
  
  
  
  
  
  
  
  SwsContext *img_convert_ctx;
  
  int frame_size = 0;
  
  
  
  uint8_t *picture_buf = NULL, *frame_buf = NULL;
  
  
  
  bool bCap = true;
  
  
  
  DWORD WINAPI ScreenCapThreadProc( LPVOID lpParam );
  
  DWORD WINAPI AudioCapThreadProc( LPVOID lpParam );
  
  
  
  int OpenVideoCapture()
  
  {
  
      AVInputFormat *ifmt=av_find_input_format("gdigrab");
  
      //
  
      AVDictionary *options = NULL;
  
      av_dict_set(&options, "framerate", "15", NULL);
  
      //av_dict_set(&options,"offset_x","20",0);
  
      //The distance from the top edge of the screen or desktop
  
      //av_dict_set(&options,"offset_y","40",0);
  
      //Video frame size. The default is to capture the full screen
  
      //av_dict_set(&options,"video_size","320x240",0);
  
      if(avformat_open_input(&pFormatCtx_Video, "desktop", ifmt, &options)!=0)
  
      {
  
          printf("Couldn't open input stream.（无法打开视频输入流）\n");
  
          return -1;
  
      }
  
      if(avformat_find_stream_info(pFormatCtx_Video,NULL)<0)
  
      {
  
          printf("Couldn't find stream information.（无法获取视频流信息）\n");
  
          return -1;
  
      }
  
      if (pFormatCtx_Video->streams[0]->codec->codec_type != AVMEDIA_TYPE_VIDEO)
  
      {
  
          printf("Couldn't find video stream information.（无法获取视频流信息）\n");
  
          return -1;
  
      }
  
      pCodecCtx_Video = pFormatCtx_Video->streams[0]->codec;
  
      pCodec_Video = avcodec_find_decoder(pCodecCtx_Video->codec_id);
  
      if(pCodec_Video == NULL)
  
      {
  
          printf("Codec not found.（没有找到解码器）\n");
  
          return -1;
  
      }
  
      if(avcodec_open2(pCodecCtx_Video, pCodec_Video, NULL) < 0)
  
      {
  
          printf("Could not open codec.（无法打开解码器）\n");
  
          return -1;
  
      }
  
  
  
  
  
  
  
      img_convert_ctx = sws_getContext(pCodecCtx_Video->width, pCodecCtx_Video->height, pCodecCtx_Video->pix_fmt, 
  
          pCodecCtx_Video->width, pCodecCtx_Video->height, PIX_FMT_YUV420P, SWS_BICUBIC, NULL, NULL, NULL); 
  
  
  
      frame_size = avpicture_get_size(pCodecCtx_Video->pix_fmt, pCodecCtx_Video->width, pCodecCtx_Video->height);
  
      //
  
      fifo_video = av_fifo_alloc(30 * avpicture_get_size(AV_PIX_FMT_YUV420P, pCodecCtx_Video->width, pCodecCtx_Video->height));
  
  
  
      return 0;
  
  }
  
  
  
  static char *dup_wchar_to_utf8(wchar_t *w)
  
  {
  
      char *s = NULL;
  
      int l = WideCharToMultiByte(CP_UTF8, 0, w, -1, 0, 0, 0, 0);
  
      s = (char *) av_malloc(l);
  
      if (s)
  
          WideCharToMultiByte(CP_UTF8, 0, w, -1, s, l, 0, 0);
  
      return s;
  
  }
  
  
  
  int OpenAudioCapture()
  
  {
  
      //
  
      AVInputFormat *pAudioInputFmt = av_find_input_format("dshow");
  
  
  
      //
  
      char * psDevName = dup_wchar_to_utf8(L"audio=virtual-audio-capturer");
  
  
  
      if (avformat_open_input(&pFormatCtx_Audio, psDevName, pAudioInputFmt,NULL) < 0)
  
      {
  
          printf("Couldn't open input stream.（无法打开音频输入流）\n");
  
          return -1;
  
      }
  
  
  
      if(avformat_find_stream_info(pFormatCtx_Audio,NULL)<0)  
  
          return -1; 
  
  
  
      if(pFormatCtx_Audio->streams[0]->codec->codec_type != AVMEDIA_TYPE_AUDIO)
  
      {
  
          printf("Couldn't find video stream information.（无法获取音频流信息）\n");
  
          return -1;
  
      }
  
  
  
      AVCodec *tmpCodec = avcodec_find_decoder(pFormatCtx_Audio->streams[0]->codec->codec_id);
  
      if(0 > avcodec_open2(pFormatCtx_Audio->streams[0]->codec, tmpCodec, NULL))
  
      {
  
          printf("can not find or open audio decoder!\n");
  
      }
  
  
  
  
  
  
  
      return 0;
  
  }
  
  
  
  int OpenOutPut()
  
  {
  
      AVStream *pVideoStream = NULL, *pAudioStream = NULL;
  
      const char *outFileName = "test.mp4";
  
      avformat_alloc_output_context2(&pFormatCtx_Out, NULL, NULL, outFileName);
  
  
  
      if (pFormatCtx_Video->streams[0]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
  
      {
  
          VideoIndex = 0;
  
          pVideoStream = avformat_new_stream(pFormatCtx_Out, NULL);
  
  
  
          if (!pVideoStream)
  
          {
  
              printf("can not new stream for output!\n");
  
              return -1;
  
          }
  
  
  
          //set codec context param
  
          pVideoStream->codec->codec = avcodec_find_encoder(AV_CODEC_ID_MPEG4);
  
          pVideoStream->codec->height = pFormatCtx_Video->streams[0]->codec->height;
  
          pVideoStream->codec->width = pFormatCtx_Video->streams[0]->codec->width;
  
  
  
          pVideoStream->codec->time_base = pFormatCtx_Video->streams[0]->codec->time_base;
  
          pVideoStream->codec->sample_aspect_ratio = pFormatCtx_Video->streams[0]->codec->sample_aspect_ratio;
  
          // take first format from list of supported formats
  
          pVideoStream->codec->pix_fmt = pFormatCtx_Out->streams[VideoIndex]->codec->codec->pix_fmts[0];
  
  
  
          //open encoder
  
          if (!pVideoStream->codec->codec)
  
          {
  
              printf("can not find the encoder!\n");
  
              return -1;
  
          }
  
  
  
          if (pFormatCtx_Out->oformat->flags & AVFMT_GLOBALHEADER)
  
              pVideoStream->codec->flags |= CODEC_FLAG_GLOBAL_HEADER;
  
  
  
          if ((avcodec_open2(pVideoStream->codec, pVideoStream->codec->codec, NULL)) < 0)
  
          {
  
              printf("can not open the encoder\n");
  
              return -1;
  
          }
  
      }
  
  
  
      if(pFormatCtx_Audio->streams[0]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
  
      {
  
          AVCodecContext *pOutputCodecCtx;
  
          AudioIndex = 1;
  
          pAudioStream = avformat_new_stream(pFormatCtx_Out, NULL);
  
  
  
          pAudioStream->codec->codec = avcodec_find_encoder(pFormatCtx_Out->oformat->audio_codec);
  
  
  
          pOutputCodecCtx = pAudioStream->codec;
  
  
  
          pOutputCodecCtx->sample_rate = pFormatCtx_Audio->streams[0]->codec->sample_rate;
  
          pOutputCodecCtx->channel_layout = pFormatCtx_Out->streams[0]->codec->channel_layout;
  
          pOutputCodecCtx->channels = av_get_channel_layout_nb_channels(pAudioStream->codec->channel_layout);
  
          if(pOutputCodecCtx->channel_layout == 0)
  
          {
  
              pOutputCodecCtx->channel_layout = AV_CH_LAYOUT_STEREO;
  
              pOutputCodecCtx->channels = av_get_channel_layout_nb_channels(pOutputCodecCtx->channel_layout);
  
  
  
          }
  
          pOutputCodecCtx->sample_fmt = pAudioStream->codec->codec->sample_fmts[0];
  
          AVRational time_base={1, pAudioStream->codec->sample_rate};
  
          pAudioStream->time_base = time_base;
  
          //audioCodecCtx->time_base = time_base;
  
  
  
          pOutputCodecCtx->codec_tag = 0;  
  
          if (pFormatCtx_Out->oformat->flags & AVFMT_GLOBALHEADER)  
  
              pOutputCodecCtx->flags |= CODEC_FLAG_GLOBAL_HEADER;
  
  
  
          if (avcodec_open2(pOutputCodecCtx, pOutputCodecCtx->codec, 0) < 0)
  
          {
  
              //
  
              return -1;
  
          }
  
      }
  
  
  
      if (!(pFormatCtx_Out->oformat->flags & AVFMT_NOFILE))
  
      {
  
          if(avio_open(&pFormatCtx_Out->pb, outFileName, AVIO_FLAG_WRITE) < 0)
  
          {
  
              printf("can not open output file handle!\n");
  
              return -1;
  
          }
  
      }
  
  
  
      if(avformat_write_header(pFormatCtx_Out, NULL) < 0)
  
      {
  
          printf("can not write the header of the output file!\n");
  
          return -1;
  
      }
  
  
  
      return 0;
  
  }
  
  
  
  int _tmain(int argc, _TCHAR* argv[])
  
  {
  
      av_register_all();
  
      avdevice_register_all();
  
      if (OpenVideoCapture() < 0)
  
      {
  
          return -1;
  
      }
  
      if (OpenAudioCapture() < 0)
  
      {
  
          return -1;
  
      }
  
      if (OpenOutPut() < 0)
  
      {
  
          return -1;
  
      }
  
  
  
      InitializeCriticalSection(&VideoSection);
  
      InitializeCriticalSection(&AudioSection);
  
  
  
      AVFrame *picture = av_frame_alloc();
  
      int size = avpicture_get_size(pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt, 
  
          pFormatCtx_Out->streams[VideoIndex]->codec->width, pFormatCtx_Out->streams[VideoIndex]->codec->height);
  
      picture_buf = new uint8_t[size];
  
  
  
      avpicture_fill((AVPicture *)picture, picture_buf, 
  
          pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt, 
  
          pFormatCtx_Out->streams[VideoIndex]->codec->width, 
  
          pFormatCtx_Out->streams[VideoIndex]->codec->height);
  
  
  
  
  
  
  
      //star cap screen thread
  
      CreateThread( NULL, 0, ScreenCapThreadProc, 0, 0, NULL);
  
      //star cap audio thread
  
      CreateThread( NULL, 0, AudioCapThreadProc, 0, 0, NULL);
  
      int64_t cur_pts_v=0,cur_pts_a=0;
  
      int VideoFrameIndex = 0, AudioFrameIndex = 0;
  
  
  
      while(1)
  
      {
  
          if (_kbhit() != 0 && bCap)
  
          {
  
              bCap = false;
  
              Sleep(2000);//
  
          }
  
          if (fifo_audio && fifo_video)
  
          {
  
              int sizeAudio = av_audio_fifo_size(fifo_audio);
  
              int sizeVideo = av_fifo_size(fifo_video);
  
              //
  
              if (av_audio_fifo_size(fifo_audio) <= pFormatCtx_Out->streams[AudioIndex]->codec->frame_size && 
  
                  av_fifo_size(fifo_video) <= frame_size && !bCap)
  
              {
  
                  break;
  
              }
  
          }
  
  
  
          if(av_compare_ts(cur_pts_v, pFormatCtx_Out->streams[VideoIndex]->time_base, 
  
              cur_pts_a,pFormatCtx_Out->streams[AudioIndex]->time_base) <= 0)
  
          {
  
              //read data from fifo
  
              if (av_fifo_size(fifo_video) < frame_size && !bCap)
  
              {
  
                  cur_pts_v = 0x7fffffffffffffff;
  
              }
  
              if(av_fifo_size(fifo_video) >= size)
  
              {
  
                  EnterCriticalSection(&VideoSection);
  
                  av_fifo_generic_read(fifo_video, picture_buf, size, NULL);
  
                  LeaveCriticalSection(&VideoSection);
  
  
  
                  avpicture_fill((AVPicture *)picture, picture_buf, 
  
                      pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt, 
  
                      pFormatCtx_Out->streams[VideoIndex]->codec->width, 
  
                      pFormatCtx_Out->streams[VideoIndex]->codec->height);
  
  
  
                  //pts = n * (（1 / timbase）/ fps);
  
                  picture->pts = VideoFrameIndex * ((pFormatCtx_Video->streams[0]->time_base.den / pFormatCtx_Video->streams[0]->time_base.num) / 15);
  
  
  
                  int got_picture = 0;
  
                  AVPacket pkt;
  
                  av_init_packet(&pkt);
  
  
  
                  pkt.data = NULL;
  
                  pkt.size = 0;
  
                  int ret = avcodec_encode_video2(pFormatCtx_Out->streams[VideoIndex]->codec, &pkt, picture, &got_picture);
  
                  if(ret < 0)
  
                  {
  
                      //
  
                      continue;
  
                  }
  
  
  
                  if (got_picture==1)
  
                  {
  
                      pkt.stream_index = VideoIndex;
  
                      pkt.pts = av_rescale_q_rnd(pkt.pts, pFormatCtx_Video->streams[0]->time_base, 
  
                          pFormatCtx_Out->streams[VideoIndex]->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));  
  
                      pkt.dts = av_rescale_q_rnd(pkt.dts,  pFormatCtx_Video->streams[0]->time_base, 
  
                          pFormatCtx_Out->streams[VideoIndex]->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));  
  
  
  
                      pkt.duration = ((pFormatCtx_Out->streams[0]->time_base.den / pFormatCtx_Out->streams[0]->time_base.num) / 15);
  
  
  
                      cur_pts_v = pkt.pts;
  
  
  
                      ret = av_interleaved_write_frame(pFormatCtx_Out, &pkt);
  
                      //delete[] pkt.data;
  
                      av_free_packet(&pkt);
  
                  }
  
                  VideoFrameIndex++;
  
              }
  
          }
  
          else
  
          {
  
              if (NULL == fifo_audio)
  
              {
  
                  continue;//
  
              }
  
              if (av_audio_fifo_size(fifo_audio) < pFormatCtx_Out->streams[AudioIndex]->codec->frame_size && !bCap)
  
              {
  
                  cur_pts_a = 0x7fffffffffffffff;
  
              }
  
              if(av_audio_fifo_size(fifo_audio) >= 
  
                  (pFormatCtx_Out->streams[AudioIndex]->codec->frame_size > 0 ? pFormatCtx_Out->streams[AudioIndex]->codec->frame_size : 1024))
  
              {
  
                  AVFrame *frame;
  
                  frame = av_frame_alloc();
  
                  frame->nb_samples = pFormatCtx_Out->streams[AudioIndex]->codec->frame_size>0 ? pFormatCtx_Out->streams[AudioIndex]->codec->frame_size: 1024;
  
                  frame->channel_layout = pFormatCtx_Out->streams[AudioIndex]->codec->channel_layout;
  
                  frame->format = pFormatCtx_Out->streams[AudioIndex]->codec->sample_fmt;
  
                  frame->sample_rate = pFormatCtx_Out->streams[AudioIndex]->codec->sample_rate;
  
                  av_frame_get_buffer(frame, 0);
  
  
  
                  EnterCriticalSection(&AudioSection);
  
                  av_audio_fifo_read(fifo_audio, (void **)frame->data, 
  
                      (pFormatCtx_Out->streams[AudioIndex]->codec->frame_size > 0 ? pFormatCtx_Out->streams[AudioIndex]->codec->frame_size : 1024));
  
                  LeaveCriticalSection(&AudioSection);
  
  
  
                  if (pFormatCtx_Out->streams[0]->codec->sample_fmt != pFormatCtx_Audio->streams[AudioIndex]->codec->sample_fmt 
  
                      || pFormatCtx_Out->streams[0]->codec->channels != pFormatCtx_Audio->streams[AudioIndex]->codec->channels 
  
                      || pFormatCtx_Out->streams[0]->codec->sample_rate != pFormatCtx_Audio->streams[AudioIndex]->codec->sample_rate)
  
                  {
  
                      //
  
                  }
  
  
  
                  AVPacket pkt_out;
  
                  av_init_packet(&pkt_out);
  
                  int got_picture = -1;
  
                  pkt_out.data = NULL;
  
                  pkt_out.size = 0;
  
  
  
                  frame->pts = AudioFrameIndex * pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
  
                  if (avcodec_encode_audio2(pFormatCtx_Out->streams[AudioIndex]->codec, &pkt_out, frame, &got_picture) < 0)
  
                  {
  
                      printf("can not decoder a frame");
  
                  }
  
                  av_frame_free(&frame);
  
                  if (got_picture) 
  
                  {
  
                      pkt_out.stream_index = AudioIndex;
  
                      pkt_out.pts = AudioFrameIndex * pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
  
                      pkt_out.dts = AudioFrameIndex * pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
  
                      pkt_out.duration = pFormatCtx_Out->streams[AudioIndex]->codec->frame_size;
  
  
  
                      cur_pts_a = pkt_out.pts;
  
  
  
                      int ret = av_interleaved_write_frame(pFormatCtx_Out, &pkt_out);
  
                      av_free_packet(&pkt_out);
  
                  }
  
                  AudioFrameIndex++;
  
              }
  
          }
  
      }
  
  
  
      delete[] picture_buf;
  
  
  
      delete[]frame_buf;
  
      av_fifo_free(fifo_video);
  
      av_audio_fifo_free(fifo_audio);
  
  
  
      av_write_trailer(pFormatCtx_Out);
  
  
  
      avio_close(pFormatCtx_Out->pb);
  
      avformat_free_context(pFormatCtx_Out);
  
  
  
      if (pFormatCtx_Video != NULL)
  
      {
  
          avformat_close_input(&pFormatCtx_Video);
  
          pFormatCtx_Video = NULL;
  
      }
  
      if (pFormatCtx_Audio != NULL)
  
      {
  
          avformat_close_input(&pFormatCtx_Audio);
  
          pFormatCtx_Audio = NULL;
  
      }
  
      if (NULL != img_convert_ctx)
  
      {
  
          sws_freeContext(img_convert_ctx);
  
          img_convert_ctx = NULL;
  
      }
  
  
  
      return 0;
  
  }
  
  
  
  DWORD WINAPI ScreenCapThreadProc( LPVOID lpParam )
  
  {
  
      AVPacket packet;/* = (AVPacket *)av_malloc(sizeof(AVPacket))*/;
  
      int got_picture;
  
      AVFrame *pFrame;
  
      pFrame= av_frame_alloc();
  
  
  
      AVFrame *picture = av_frame_alloc();
  
      int size = avpicture_get_size(pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt, 
  
          pFormatCtx_Out->streams[VideoIndex]->codec->width, pFormatCtx_Out->streams[VideoIndex]->codec->height);
  
      //picture_buf = new uint8_t[size];
  
  
  
      avpicture_fill((AVPicture *)picture, picture_buf, 
  
          pFormatCtx_Out->streams[VideoIndex]->codec->pix_fmt, 
  
          pFormatCtx_Out->streams[VideoIndex]->codec->width, 
  
          pFormatCtx_Out->streams[VideoIndex]->codec->height);
  
  
  
  //  FILE *p = NULL;
  
  //  p = fopen("proc_test.yuv", "wb+");
  
      av_init_packet(&packet);
  
      int height = pFormatCtx_Out->streams[VideoIndex]->codec->height;
  
      int width = pFormatCtx_Out->streams[VideoIndex]->codec->width;
  
      int y_size=height*width;
  
      while(bCap)
  
      {
  
          packet.data = NULL;
  
          packet.size = 0;
  
          if (av_read_frame(pFormatCtx_Video, &packet) < 0)
  
          {
  
              av_free_packet(&packet);
  
              continue;
  
          }
  
          if(packet.stream_index == 0)
  
          {
  
              if (avcodec_decode_video2(pCodecCtx_Video, pFrame, &got_picture, &packet) < 0)
  
              {
  
                  printf("Decode Error.（解码错误）\n");
  
                  continue;
  
              }
  
              if (got_picture)
  
              {
  
                  sws_scale(img_convert_ctx, (const uint8_t* const*)pFrame->data, pFrame->linesize, 0, 
  
                      pFormatCtx_Out->streams[VideoIndex]->codec->height, picture->data, picture->linesize);
  
  
  
                  if (av_fifo_space(fifo_video) >= size)
  
                  {
  
                      EnterCriticalSection(&VideoSection);                    
  
                      av_fifo_generic_write(fifo_video, picture->data[0], y_size, NULL);
  
                      av_fifo_generic_write(fifo_video, picture->data[1], y_size/4, NULL);
  
                      av_fifo_generic_write(fifo_video, picture->data[2], y_size/4, NULL);
  
                      LeaveCriticalSection(&VideoSection);
  
                  }
  
              }
  
          }
  
          av_free_packet(&packet);
  
          //Sleep(50);
  
      }
  
      av_frame_free(&pFrame);
  
      av_frame_free(&picture);
  
      //delete[] picture_buf;
  
      return 0;
  
  }
  
  
  
  DWORD WINAPI AudioCapThreadProc( LPVOID lpParam )
  
  {
  
      AVPacket pkt;
  
      AVFrame *frame;
  
      frame = av_frame_alloc();
  
      int gotframe;
  
      while(bCap)
  
      {
  
          pkt.data = NULL;
  
          pkt.size = 0;
  
          if(av_read_frame(pFormatCtx_Audio,&pkt) < 0)
  
          {
  
              av_free_packet(&pkt);
  
              continue;
  
          }
  
  
  
          if (avcodec_decode_audio4(pFormatCtx_Audio->streams[0]->codec, frame, &gotframe, &pkt) < 0)
  
          {
  
              av_frame_free(&frame);
  
              printf("can not decoder a frame");
  
              break;
  
          }
  
          av_free_packet(&pkt);
  
  
  
          if (!gotframe)
  
          {
  
              continue;//
  
          }
  
  
  
          if (NULL == fifo_audio)
  
          {
  
              fifo_audio = av_audio_fifo_alloc(pFormatCtx_Audio->streams[0]->codec->sample_fmt, 
  
                  pFormatCtx_Audio->streams[0]->codec->channels, 30 * frame->nb_samples);
  
          }
  
  
  
          int buf_space = av_audio_fifo_space(fifo_audio);
  
          if (av_audio_fifo_space(fifo_audio) >= frame->nb_samples)
  
          {
  
              EnterCriticalSection(&AudioSection);
  
              av_audio_fifo_write(fifo_audio, (void **)frame->data, frame->nb_samples);
  
              LeaveCriticalSection(&AudioSection);
  
          }
  
      }
  
      av_frame_free(&frame);
  
      return 0;
  
  }

• Matomo’s 2021 Year in Review

  13 décembre 2021, par erin — Community

  2021 has been an exciting year at Matomo !

  We’re grateful for all community members who reported feedback and suggestions, our awesome team of translators for their work, and our Premium features customers and Matomo Cloud hosting customers for their amazing support. 

  We wanted to share some quick highlights to remind you of the exciting things that happened in 2021.

  Matomo continues to develop

  In 2021 we released a number of new features including :

  • SEO Web Vitals

  The new SEO Web Vitals feature helps you track your critical website performance metrics, which are a core element of SEO best practice.

  

  • Advertising Conversion Export 

  Measure the performance of your ads without giving up privacy.

  This exciting new feature supports privacy and compliance requirements by eliminating the need to put third-party advertising tracking codes on your site. Now marketers can easily import conversion data from Matomo into Google Ads, Microsoft Advertising or Yandex Ads.

  • Tracking Spam Prevention 

  Say goodbye to spammers & bots making your data inaccurate and say hello to reliable data. 

  This powerful plugin provides our self-hosting users various options to prevent spammers and bots from making data inaccurate so you can rely on your data again.

  • In 2021 we moved from Matomo 4.1.0 to Matomo 4.6.0, with our new releases delivering over 600 updates to improve the stability and functionality of the product.

  Some of our team’s favourite updates in 2021 included :

  • • Graphs now show a difference for data of “unfinished” and “complete” periods, with unfinished periods now indicated by a dashed line.
    • Improvements to Matomo Tag Manager’s debugger – now you can simply enter the URL in a form and click Debug.
    • Dashboards now show proportional evolution comparison for incomplete periods (rather than absolute values).
     
  • We also rolled out general bug fixes in Matomo Mobile 2.5 for iOS and Android.
  • Continuous improvements to Matomo for WordPress. 

  In other news

  • Au revoir, tracking consent. France’s data protection authority, Commission nationale de l’informatique et des libertés (CNIL), announced that Matomo is now exempt from tracking consent in France (under conditions).
  • We lowered the pricing for Matomo Cloud, the starting price is now 19 EUR or 19 USD per month.
  • For our self-hosting users, we introduced a 30-day Free Trial for Marketplace Premium Plugins. 

  If you haven’t explored our Marketplace yet, some of our most popular plugins include :

  • • Form Analytics
    • Search Engine Keyword Performance
    • Heatmaps and Session Recordings
     
  • Out of a substantial list of 320 products, Matomo was recognised by Capterra as a leading global Web Analytics Software solution for 2021.

  Matomo Community working together

  MatomoCamp 2021 was a massive success thanks to our passionate community, sponsors and speakers. This virtual event was run by the Matomo Community, for the Matomo Community. 

  

  Some of the most popular sessions included :

  • How to stay compliant with privacy regulations with Matomo 
  • Use Matomo properly for SEO (German)
  • How does risk for a DPO differ from classical risk perceptions ?
  • Are we there yet ? The present & future of surveillance societies
  • How to manage a Matomo with millions of pageviews per month ?

  All MatomoCamp sessions are now available on-demand. Enjoy !

  More people are choosing ethical analytics 

  We surpassed the incredible milestone of 30K active Matomo for WordPress installations.

  How can you get involved in 2022 ?

  Our mission at Matomo is :

  “To create, as a community, the leading open digital analytics platform, that gives every user full control of their data”

  Join our mission by writing about Matomo on your blog, website, Twitter, talk at conferences or let your friends and colleagues know what is Matomo. 

  Use the Matomo forum if you have any questions or feedback (free support), or purchase a Support Plan to get professional support and guidance.

  To improve Matomo in your language, consider contributing to translations.

  You can also support our efforts by purchasing Premium Features for Matomo or try our Matomo Cloud solution.

  Thank you for being part of our Matomo community, we wish you all the best for 2022 !

• The problems with wavelets

  27 février 2010, par Dark Shikari — DCT, Dirac, Snow, psychovisual optimizations, wavelets

  I have periodically noted in this blog and elsewhere various problems with wavelet compression, but many readers have requested that I write a more detailed post about it, so here it is.

  Wavelets have been researched for quite some time as a replacement for the standard discrete cosine transform used in most modern video compression. Their methodology is basically opposite : each coefficient in a DCT represents a constant pattern applied to the whole block, while each coefficient in a wavelet transform represents a single, localized pattern applied to a section of the block. Accordingly, wavelet transforms are usually very large with the intention of taking advantage of large-scale redundancy in an image. DCTs are usually quite small and are intended to cover areas of roughly uniform patterns and complexity.

  Both are complete transforms, offering equally accurate frequency-domain representations of pixel data. I won’t go into the mathematical details of each here ; the real question is whether one offers better compression opportunities for real-world video.

  DCT transforms, though it isn’t mathematically required, are usually found as block transforms, handling a single sharp-edged block of data. Accordingly, they usually need a deblocking filter to smooth the edges between DCT blocks. Wavelet transforms typically overlap, avoiding such a need. But because wavelets don’t cover a sharp-edged block of data, they don’t compress well when the predicted data is in the form of blocks.

  Thus motion compensation is usually performed as overlapped-block motion compensation (OBMC), in which every pixel is calculated by performing the motion compensation of a number of blocks and averaging the result based on the distance of those blocks from the current pixel. Another option, which can be combined with OBMC, is “mesh MC“, where every pixel gets its own motion vector, which is a weighted average of the closest nearby motion vectors. The end result of either is the elimination of sharp edges between blocks and better prediction, at the cost of greatly increased CPU requirements. For an overlap factor of 2, it’s 4 times the amount of motion compensation, plus the averaging step. With mesh MC, it’s even worse, with SIMD optimizations becoming nearly impossible.

  At this point, it would seem wavelets would have pretty big advantages : when used with OBMC, they have better inter prediction, eliminate the need for deblocking, and take advantage of larger-scale correlations. Why then hasn’t everyone switched over to wavelets then ? Dirac and Snow offer modern implementations. Yet despite decades of research, wavelets have consistently disappointed for image and video compression. It turns out there are a lot of serious practical issues with wavelets, many of which are open problems.

  1. No known method exists for efficient intra coding. H.264′s spatial intra prediction is extraordinarily powerful, but relies on knowing the exact decoded pixels to the top and left of the current block. Since there is no such boundary in overlapped-wavelet coding, such prediction is impossible. Newer intra prediction methods, such as markov-chain intra prediction, also seem to require an H.264-like situation with exactly-known neighboring pixels. Intra coding in wavelets is in the same state that DCT intra coding was in 20 years ago : the best known method was to simply transform the block with no prediction at all besides DC. NB : as described by Pengvado in the comments, the switching between inter and intra coding is potentially even more costly than the inefficient intra coding.

  2. Mixing partition sizes has serious practical problems. Because the overlap between two motion partitions depends on the partitions’ size, mixing block sizes becomes quite difficult to define. While in H.264 an smaller partition always gives equal or better compression than a larger one when one ignores the extra overhead, it is actually possible for a larger partition to win when using OBMC due to the larger overlap. All of this makes both the problem of defining the result of mixed block sizes and making decisions about them very difficult.

  Both Snow and Dirac offer variable block size, but the overlap amount is constant ; larger blocks serve only to save bits on motion vectors, not offer better overlap characteristics.

  3. Lack of spatial adaptive quantization. As shown in x264 with VAQ, and correspondingly in HCEnc’s implementation and Theora’s recent implementation, spatial adaptive quantization has staggeringly impressive (before, after) effects on visual quality. Only Dirac seems to have such a feature, and the encoder doesn’t even use it. No other wavelet formats (Snow, JPEG2K, etc) seem to have such a feature. This results in serious blurring problems in areas with subtle texture (as in the comparison below).

  4. Wavelets don’t seem to code visual energy effectively. Remember that a single coefficient in a DCT represents a pattern which applies across an entire block : this makes it very easy to create apparent “detail” with a DCT. Furthermore, the sharp edges of DCT blocks, despite being an apparent weakness, often result in a “fake sharpness” that can actually improve the visual appearance of videos, as was seen with Xvid. Thus wavelet codecs have a tendency to look much blurrier than DCT-based codecs, but since PSNR likes blur, this is often seen as a benefit during video compression research. Some of the consequences of these factors can be seen in this comparison ; somewhat outdated and not general-case, but which very effectively shows the difference in how wavelets handle sharp edges and subtle textures.

  Another problem that periodically crops up is the visual aliasing that tends to be associated with wavelets at lower bitrates. Standard wavelets effectively consist of a recursive function that upscales the coefficients coded by the previous level by a factor of 2 and then adds a new set of coefficients. If the upscaling algorithm is naive — as it often is, for the sake of speed — the result can look quite ugly, as if parts of the image were coded at a lower resolution and then badly scaled up. Of course, it looks like that because they were coded at a lower resolution and then badly scaled up.

  JPEG2000 is a classic example of wavelet failure : despite having more advanced entropy coding, being designed much later than JPEG, being much more computationally intensive, and having much better PSNR, comparisons have consistently shown it to be visually worse than JPEG at sane filesizes. Here’s an example from Wikipedia. By comparison, H.264′s intra coding, when used for still image compression, can beat JPEG by a factor of 2 or more (I’ll make a post on this later). With the various advancements in DCT intra coding since H.264, I suspect that a state-of-the-art DCT compressor could win by an even larger factor.

  Despite the promised benefits of wavelets, a wavelet encoder even close to competitive with x264 has yet to be created. With some tests even showing Dirac losing to Theora in visual comparisons, it’s clear that many problems remain to be solved before wavelets can eliminate the ugliness of block-based transforms once and for all.