Recherche avancée

Sur d’autres sites (5491)

• Sending raw h264 video and aac audio frames to an RTMP server using ffmpeg

  15 août 2016, par codeimpaler

  I am receiving raw h264 and aac audio frames from an even driven source. I am trying to send these frames to an rtmp server.
  I started working from the ffmpeg example muxing.c which successfully sends a custom stream to the rtmp server. I figure I just need to replace their frame data with my own.I found this suggestion online. I have tried How to pack raw h264 stream to flv container and send over rtmp using ffmpeg (not command)
  and
  How to publish selfmade stream with ffmpeg and c++ to rtmp server ?
  and a few other suggestions but none have worked for me.
  I have tried to directly memcpy my byte buffer but my code keeps failing
  at ret = avcodec_encode_video2(c, &pkt, frame, &got_packet).
  Specifically, I get an invalid access error.
  For a little more context, anytime I receive a frame (which is event driven), void RTMPWriter::WriteVideoFrame(...) is called. Assume the constructor has already been called before the first frame is received.
  I am not that familiar with ffmpeg and there could be several things wrong with the code. Any input will be really appreciated.

      #define STREAM_FRAME_RATE 25 /* 25 images/s */
  
      #define STREAM_PIX_FMT    AV_PIX_FMT_YUV420P /* default pix_fmt */
  
      #define SCALE_FLAGS SWS_BICUBIC
  
      RTMPWriter::RTMPWriter()
  
        : seenKeyFrame(false),
  
          video_st({ 0 }), 
  
          audio_st({ 0 }),
  
          have_video(0), 
  
          have_audio(0)
  
      {
  
  
  
          const char *filename;
  
          AVCodec *audio_codec = NULL, *video_codec = NULL;
  
          int ret;
  
  
  
          int encode_video = 0, encode_audio = 0;
  
          AVDictionary *opt = NULL;
  
          int i;
  
  
  
          /* Initialize libavcodec, and register all codecs and formats. */
  
          av_register_all();
  
  
  
          avformat_network_init();
  
  
  
         String^ StreamURL = "StreamURL";
  
         String^ out_uri = safe_cast(ApplicationData::Current->LocalSettings->Values->Lookup(StreamURL));
  
         std::wstring out_uriW(out_uri->Begin());
  
         std::string out_uriA(out_uriW.begin(), out_uriW.end());
  
         filename = out_uriA.c_str();  
  
  
  
         /* allocate the output media context */
  
         avformat_alloc_output_context2(&oc, NULL, "flv", filename);
  
         if (!oc)
  
         {
  
             OutputDebugString(L"Could not deduce output format from file extension: using MPEG.\n");
  
             avformat_alloc_output_context2(&oc, NULL, "mpeg", filename);
  
         }
  
         if (!oc)
  
         {
  
             OutputDebugString(L"Could not allocate  using MPEG.\n");
  
         }
  
  
  
  
  
         fmt = oc->oformat;
  
  
  
         /* Add the audio and video streams using the default format codecs
  
         * and initialize the codecs. */
  
         if (fmt->video_codec != AV_CODEC_ID_NONE) {
  
             add_stream(&video_st, oc, &video_codec, fmt->video_codec);
  
             have_video = 1;
  
             encode_video = 1;
  
         }
  
         if (fmt->audio_codec != AV_CODEC_ID_NONE) {
  
             add_stream(&audio_st, oc, &audio_codec, fmt->audio_codec);
  
             have_audio = 1;
  
             encode_audio = 1;
  
         }
  
  
  
         /* Now that all the parameters are set, we can open the audio and
  
          * video codecs and allocate the necessary encode buffers. */
  
         if (have_video)
  
         {
  
             open_video(oc, video_codec, &video_st, opt);
  
         }
  
  
  
         if (have_audio)
  
         {
  
             open_audio(oc, audio_codec, &audio_st, opt);
  
         }
  
  
  
         av_dump_format(oc, 0, filename, 1);
  
  
  
         /* open the output file, if needed */
  
         if (!(fmt->flags & AVFMT_NOFILE))
  
         {
  
             ret = avio_open(&oc->pb, filename, AVIO_FLAG_WRITE);
  
             if (ret < 0)
  
             {
  
                 OutputDebugString(L"Could not open ");
  
                 OutputDebugString(out_uri->Data());
  
             }
  
         }
  
  
  
         /* Write the stream header, if any. */
  
         ret = avformat_write_header(oc, &opt);
  
         if (ret < 0)
  
         {
  
             OutputDebugString(L"Error occurred when writing stream header \n");
  
         }
  
  
  
      }
  
  
  
      void RTMPWriter::WriteVideoFrame(
  
          boolean isKeyFrame,
  
          boolean hasDiscontinuity,
  
          UINT64 frameId,
  
          UINT32 videoBufferLength,
  
          BYTE *videoBytes)
  
      {
  
  
  
          int ret;
  
          AVCodecContext *c;
  
          AVFrame* frame;
  
          int got_packet = 0;
  
          AVPacket pkt = { 0 };
  
  
  
          c = video_st.enc;
  
  
  
          frame = get_video_frame(videoBufferLength, videoBytes);
  
  
  
          /* encode the image */
  
          ret = avcodec_encode_video2(c, &pkt, frame, &got_packet);
  
          if (ret < 0) {
  
               OutputDebugString(L"Error encoding video frame: \n")
  
          }
  
  
  
          if (got_packet) 
  
          {
  
              ret = write_frame(oc, &c->time_base, video_st.st, &pkt);
  
          }
  
          else {
  
              ret = 0;
  
          }
  
  
  
          if (ret < 0) {
  
               OutputDebugString(L"Error while writing video frame: %s\n");
  
          }
  
      }
  
  
  
      AVFrame * RTMPWriter::get_video_frame(
  
         UINT32 videoBufferLength,
  
         BYTE *videoBytes)
  
      {
  
          AVCodecContext *c = video_st.enc;
  
  
  
          if (c->pix_fmt != AV_PIX_FMT_YUV420P) {
  
              /* as we only generate a YUV420P picture, we must convert it
  
              * to the codec pixel format if needed */
  
              if (!video_st.sws_ctx) {
  
                  video_st.sws_ctx = sws_getContext(c->width, c->height,
  
                      AV_PIX_FMT_YUV420P,
  
                      c->width, c->height,
  
                      c->pix_fmt,
  
                      SCALE_FLAGS, NULL, NULL, NULL);
  
                  if (!video_st.sws_ctx) {
  
                      fprintf(stderr,
  
                          "Could not initialize the conversion context\n");
  
                          exit(1);
  
                  }
  
              }
  
              fill_yuv_image(video_st.tmp_frame, video_st.next_pts, c->width, c->height, videoBufferLength, videoBytes);
  
              sws_scale(video_st.sws_ctx,
  
              (const uint8_t * const *)video_st.tmp_frame->data, video_st.tmp_frame->linesize,
  
              0, c->height, video_st.frame->data, video_st.frame->linesize);
  
          }
  
          else {
  
              fill_yuv_image(video_st.frame, video_st.next_pts, c->width, c->height, videoBufferLength, videoBytes);
  
          }
  
  
  
          video_st.frame->pts = video_st.next_pts++;
  
  
  
          return video_st.frame;
  
      }
  
  
  
      /* Prepare a dummy image. */
  
      void  RTMPWriter::fill_yuv_image(
  
           AVFrame *pict, 
  
           int frame_index,
  
           int width, 
  
           int height, 
  
           UINT32 videoBufferLength,
  
           BYTE *videoBytes)
  
      {
  
          //int x, y, i, ret;
  
  
  
          /* when we pass a frame to the encoder, it may keep a reference to it
  
          * internally;
  
          * make sure we do not overwrite it here
  
          */
  
          ret = av_frame_make_writable(pict);
  
          if (ret < 0) 
  
          {
  
               OutputDebugString(L"Unable to make piture writable");
  
          }
  
  
  
          memcpy(pict->data, videoBytes, videoBufferLength);
  
  
  
          //i = frame_index;
  
  
  
          ///* Y */
  
          //for (y = 0; y < height; y++)
  
          //  for (x = 0; x < width; x++)
  
          //      pict->data[0][y * pict->linesize[0] + x] = x + y + i * 3;
  
  
  
          ///* Cb and Cr */
  
          //for (y = 0; y < height / 2; y++) {
  
          //  for (x = 0; x < width / 2; x++) {
  
          //      pict->data[1][y * pict->linesize[1] + x] = 128 + y + i * 2;
  
          //      pict->data[2][y * pict->linesize[2] + x] = 64 + x + i * 5;
  
          //  }
  
          //}
  
      }
  
  
  
      void RTMPWriter::WriteAudioFrame()
  
      {
  
  
  
      }
  
  
  
      /* Add an output stream. */
  
      void  RTMPWriter::add_stream(
  
          OutputStream *ost, 
  
          AVFormatContext *oc,
  
          AVCodec **codec,
  
          enum AVCodecID codec_id)
  
     {
  
      AVCodecContext *c;
  
      int i;
  
  
  
      /* find the encoder */
  
      *codec = avcodec_find_encoder(codec_id);
  
      if (!(*codec)) {
  
          OutputDebugString(L"Could not find encoder for '%s'\n");
  
          //avcodec_get_name(codec_id));
  
          exit(1);
  
      }
  
  
  
      ost->st = avformat_new_stream(oc, NULL);
  
      if (!ost->st) {
  
          OutputDebugString(L"Could not allocate stream\n");
  
          exit(1);
  
      }
  
      ost->st->id = oc->nb_streams - 1;
  
      c = avcodec_alloc_context3(*codec);
  
      if (!c) {
  
          OutputDebugString(L"Could not alloc an encoding context\n");
  
          exit(1);
  
      }
  
      ost->enc = c;
  
  
  
      switch ((*codec)->type) {
  
      case AVMEDIA_TYPE_AUDIO:
  
          c->sample_fmt = (*codec)->sample_fmts ?
  
              (*codec)->sample_fmts[0] : AV_SAMPLE_FMT_FLTP;
  
          c->bit_rate = 64000;
  
          c->sample_rate = 44100;
  
          if ((*codec)->supported_samplerates) {
  
              c->sample_rate = (*codec)->supported_samplerates[0];
  
              for (i = 0; (*codec)->supported_samplerates[i]; i++) {
  
                  if ((*codec)->supported_samplerates[i] == 44100)
  
                      c->sample_rate = 44100;
  
              }
  
          }
  
          c->channels = av_get_channel_layout_nb_channels(c->channel_layout);
  
          c->channel_layout = AV_CH_LAYOUT_STEREO;
  
          if ((*codec)->channel_layouts) {
  
              c->channel_layout = (*codec)->channel_layouts[0];
  
              for (i = 0; (*codec)->channel_layouts[i]; i++) {
  
                  if ((*codec)->channel_layouts[i] == AV_CH_LAYOUT_STEREO)
  
                      c->channel_layout = AV_CH_LAYOUT_STEREO;
  
              }
  
          }
  
          c->channels = av_get_channel_layout_nb_channels(c->channel_layout);
  
          ost->st->time_base = /*(AVRational)*/{ 1, c->sample_rate };
  
          break;
  
  
  
      case AVMEDIA_TYPE_VIDEO:
  
          c->codec_id = codec_id;
  
  
  
          c->bit_rate = 400000;
  
          /* Resolution must be a multiple of two. */
  
          c->width = 352;
  
          c->height = 288;
  
          /* timebase: This is the fundamental unit of time (in seconds) in terms
  
          * of which frame timestamps are represented. For fixed-fps content,
  
          * timebase should be 1/framerate and timestamp increments should be
  
          * identical to 1. */
  
          ost->st->time_base = /*(AVRational)*/{ 1, STREAM_FRAME_RATE };
  
          c->time_base = ost->st->time_base;
  
  
  
          c->gop_size = 12; /* emit one intra frame every twelve frames at most */
  
          c->pix_fmt = STREAM_PIX_FMT;
  
              if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
  
                  /* just for testing, we also add B-frames */
  
                  c->max_b_frames = 2;
  
              }
  
              if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO) {
  
                  /* Needed to avoid using macroblocks in which some coeffs overflow.
  
                  * This does not happen with normal video, it just happens here as
  
                  * the motion of the chroma plane does not match the luma plane. */
  
                  c->mb_decision = 2;
  
              }
  
              break;
  
  
  
          default:
  
              break;
  
          }
  
  
  
           /* Some formats want stream headers to be separate. */
  
          if (oc->oformat->flags & AVFMT_GLOBALHEADER)
  
              c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
  
      }
  
  
  
  AVFrame * RTMPWriter::alloc_audio_frame(
  
      enum AVSampleFormat sample_fmt,
  
      uint64_t channel_layout,
  
      int sample_rate, int nb_samples)
  
  {
  
      AVFrame *frame = av_frame_alloc();
  
      int ret;
  
  
  
      if (!frame) {
  
          OutputDebugString(L"Error allocating an audio frame\n");
  
          exit(1);
  
      }
  
  
  
      frame->format = sample_fmt;
  
      frame->channel_layout = channel_layout;
  
      frame->sample_rate = sample_rate;
  
      frame->nb_samples = nb_samples;
  
  
  
      if (nb_samples) {
  
          ret = av_frame_get_buffer(frame, 0);
  
          if (ret < 0) {
  
              OutputDebugString(L"Error allocating an audio buffer\n");
  
              exit(1);
  
          }
  
      }
  
  
  
          return frame;
  
      }
  
  
  
  
  
  
  
  
  
  void  RTMPWriter::open_audio(
  
      AVFormatContext *oc, 
  
      AVCodec *codec, 
  
      OutputStream *ost, 
  
      AVDictionary *opt_arg)
  
  {
  
      AVCodecContext *c;
  
      int nb_samples;
  
      int ret;
  
      AVDictionary *opt = NULL;
  
  
  
      c = ost->enc;
  
  
  
      /* open it */
  
      av_dict_copy(&opt, opt_arg, 0);
  
      ret = avcodec_open2(c, codec, &opt);
  
      av_dict_free(&opt);
  
      if (ret < 0) {
  
          OutputDebugString(L"Could not open audio codec: %s\n");// , av_err2str(ret));
  
          exit(1);
  
      }
  
  
  
      /* init signal generator */
  
      ost->t = 0;
  
      ost->tincr = 2 * M_PI * 110.0 / c->sample_rate;
  
      /* increment frequency by 110 Hz per second */
  
      ost->tincr2 = 2 * M_PI * 110.0 / c->sample_rate / c->sample_rate;
  
  
  
      if (c->codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE)
  
          nb_samples = 10000;
  
      else
  
          nb_samples = c->frame_size;
  
  
  
      ost->frame = alloc_audio_frame(c->sample_fmt, c->channel_layout,
  
          c->sample_rate, nb_samples);
  
      ost->tmp_frame = alloc_audio_frame(AV_SAMPLE_FMT_S16, c->channel_layout,
  
          c->sample_rate, nb_samples);
  
  
  
      /* copy the stream parameters to the muxer */
  
      ret = avcodec_parameters_from_context(ost->st->codecpar, c);
  
      if (ret < 0) {
  
          OutputDebugString(L"Could not copy the stream parameters\n");
  
          exit(1);
  
      }
  
  
  
      /* create resampler context */
  
      ost->swr_ctx = swr_alloc();
  
      if (!ost->swr_ctx) {
  
          OutputDebugString(L"Could not allocate resampler context\n");
  
          exit(1);
  
      }
  
  
  
      /* set options */
  
      av_opt_set_int(ost->swr_ctx, "in_channel_count", c->channels, 0);
  
      av_opt_set_int(ost->swr_ctx, "in_sample_rate", c->sample_rate, 0);
  
      av_opt_set_sample_fmt(ost->swr_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
  
      av_opt_set_int(ost->swr_ctx, "out_channel_count", c->channels, 0);
  
      av_opt_set_int(ost->swr_ctx, "out_sample_rate", c->sample_rate, 0);
  
      av_opt_set_sample_fmt(ost->swr_ctx, "out_sample_fmt", c->sample_fmt, 0);
  
  
  
      /* initialize the resampling context */
  
      if ((ret = swr_init(ost->swr_ctx)) < 0) {
  
          OutputDebugString(L"Failed to initialize the resampling context\n");
  
          exit(1);
  
      }
  
  }
  
  
  
  int RTMPWriter::write_frame(
  
      AVFormatContext *fmt_ctx, 
  
      const AVRational *time_base, 
  
      AVStream *st, 
  
      AVPacket *pkt)
  
  {
  
      /* rescale output packet timestamp values from codec to stream timebase */
  
      av_packet_rescale_ts(pkt, *time_base, st->time_base);
  
      pkt->stream_index = st->index;
  
  
  
      /* Write the compressed frame to the media file. */
  
      //log_packet(fmt_ctx, pkt);
  
      OutputDebugString(L"Actually sending video frame: %s\n");
  
      return av_interleaved_write_frame(fmt_ctx, pkt);
  
  }
  
  
  
  
  
  AVFrame  *RTMPWriter::alloc_picture(
  
      enum AVPixelFormat pix_fmt, 
  
      int width, 
  
      int height)
  
  {
  
      AVFrame *picture;
  
      int ret;
  
  
  
      picture = av_frame_alloc();
  
      if (!picture)
  
          return NULL;
  
  
  
      picture->format = pix_fmt;
  
      picture->width = width;
  
      picture->height = height;
  
  
  
      /* allocate the buffers for the frame data */
  
      ret = av_frame_get_buffer(picture, 32);
  
      if (ret < 0) {
  
          fprintf(stderr, "Could not allocate frame data.\n");
  
          exit(1);
  
      }
  
  
  
      return picture;
  
  }
  
  
  
  void RTMPWriter::open_video(
  
      AVFormatContext *oc, 
  
      AVCodec *codec, 
  
      OutputStream *ost, 
  
      AVDictionary *opt_arg)
  
  {
  
      int ret;
  
      AVCodecContext *c = ost->enc;
  
      AVDictionary *opt = NULL;
  
  
  
      av_dict_copy(&opt, opt_arg, 0);
  
  
  
      /* open the codec */
  
      ret = avcodec_open2(c, codec, &opt);
  
      av_dict_free(&opt);
  
      if (ret < 0) {
  
          OutputDebugString(L"Could not open video codec: %s\n");// , av_err2str(ret));
  
          exit(1);
  
      }
  
  
  
      /* allocate and init a re-usable frame */
  
      ost->frame = alloc_picture(c->pix_fmt, c->width, c->height);
  
      if (!ost->frame) {
  
          OutputDebugString(L"Could not allocate video frame\n");
  
          exit(1);
  
      }
  
  
  
      /* If the output format is not YUV420P, then a temporary YUV420P
  
      * picture is needed too. It is then converted to the required
  
      * output format. */
  
      ost->tmp_frame = NULL;
  
      if (c->pix_fmt != AV_PIX_FMT_YUV420P) {
  
          ost->tmp_frame = alloc_picture(AV_PIX_FMT_YUV420P, c->width, c->height);
  
          if (!ost->tmp_frame) {
  
              OutputDebugString(L"Could not allocate temporary picture\n");
  
              exit(1);
  
          }
  
      }
  
  
  
      /* copy the stream parameters to the muxer */
  
      ret = avcodec_parameters_from_context(ost->st->codecpar, c);
  
      if (ret < 0) {
  
          OutputDebugString(L"Could not copy the stream parameters\n");
  
          exit(1);
  
      }
  
  }
  
  
  
  void RTMPWriter::close_stream(AVFormatContext *oc, OutputStream *ost)
  
  {
  
      avcodec_free_context(&ost->enc);
  
      av_frame_free(&ost->frame);
  
      av_frame_free(&ost->tmp_frame);
  
      sws_freeContext(ost->sws_ctx);
  
      swr_free(&ost->swr_ctx);
  
  }
  
  
  
  RTMPWriter::~RTMPWriter()
  
  {
  
      av_write_trailer(oc);
  
      /* Close each codec. */
  
      if (have_video)
  
          close_stream(oc, &video_st);
  
      if (have_audio)
  
          close_stream(oc, &audio_st);
  
  
  
      if (!(fmt->flags & AVFMT_NOFILE))
  
          /* Close the output file. */
  
          avio_closep(&oc->pb);
  
  
  
      /* free the stream */
  
      avformat_free_context(oc);
  
  }

• live video streaming not playing on dash.js player using ffmpeg, nginx-rtmp in raspberry pi

  15 août 2016, par sparks

  I am trying to stream a live video from raspberry pi & pi camera module to web browser.But the video doesn’t play.I am using Nginx-rtmp and ffmpeg to process the video. On the client side i use dash js player trying to play the video but no luck.I see the warning on the terminal

  [flv @ 0x2c3b950] Failed to update header with correct duration.
[flv @ 0x2c3b950] Failed to update header with correct filesize.

  But i am not sure if this could be the issue. Anybody knows what might be wrong ?I don’t expect a straight answer but i am sure somebody can guide me in the right direction. Thank you in advance. Here is my set up

  Nginx.conf

  events {
  
    worker_connections  1024;
  
   }
  
  
  
  
  
   http     {
  
          include       mime.types;
  
          default_type  application/octet-stream;
  
  
  
  
  
  sendfile        on;
  
  
  
  keepalive_timeout  65;
  
  
  
  server {
  
      listen       80;
  
      server_name 192.168.1.114 localhost;
  
      location / {
  
          root   /var/www;
  
          index  index.html index.htm;
  
      }
  
  
  
  
  
      location /dash {
  
          root /var/www;
  
          add_header Cache-Control no-cache;
  
      }
  
  
  
       location /dash.js{
  
          root /var/www;
  
        }
  
  
  
       location /hls {
  
         types {
  
          application/vnd.apple.mpegurl m3u8;
  
          video/mp2t ts;
  
        }
  
  
  
        root /var/www;
  
        index index.html;
  
        add_header Cache-Control no-cache;
  
      }
  
  
  
      location /rtmpcontrol{
  
          rtmp_control all;
  
       }
  
  
  
      location /rtmpstat{
  
          rtmp_stat all;
  
       }
  
  
  
      #error_page  404              /404.html;
  
  
  
  
  
             error_page   500 502 503 504  /50x.html;
  
      location = /50x.html {
  
          root   html;
  
      }
  
  
  
         }
  
   rtmp {
  
     server {
  
      listen 1935;
  
  
  
      chunk_size 4000;
  
  
  
      application rtmp {
  
          live on;
  
          hls on;
  
          dash on;
  
          dash_path /var/www/dash;
  
          hls_path /var/www/hls;
  
      }
  
    }
  
  }

  baseline.html

      
  
      
  
      
  
      
  
  
  
      
  
      
  
  
  
      
  
      
  
  
  
      
  
      <code class="echappe-js">&lt;script src='http://stackoverflow.com/feeds/tag/dash.all.js'&gt;&lt;/script&gt;
  &lt;script&gt;<br />
  
          function getUrlVars() {<br />
  
              var vars = {};<br />
  
              var parts = window.location.href.replace(/[?&amp;amp;]+([^=&amp;amp;]+)=([^&amp;amp;]*)/gi, function(m,key,value) {<br />
  
                  vars[key] = value;<br />
  
              });<br />
  
              return vars;<br />
  
          }<br />
  
  <br />
  
          function startVideo() {<br />
  
              var vars = getUrlVars(),<br />
  
                  url = &quot;http://192.168.1.114:80/dash/stream.mpd&quot;,<br />
  
                  video,<br />
  
                  context,<br />
  
                  player;<br />
  
  <br />
  
              if (vars &amp;amp;&amp;amp; vars.hasOwnProperty(&quot;url&quot;)) {<br />
  
                  url = vars.url;<br />
  
              }<br />
  
  <br />
  
              video = document.querySelector(&quot;.dash-video-player video&quot;);<br />
  
              context = new Dash.di.DashContext();<br />
  
              player = new MediaPlayer(context);<br />
  
  <br />
  
              player.startup();<br />
  
  <br />
  
              player.attachView(video);<br />
  
              player.setAutoPlay(false);<br />
  
  <br />
  
              player.attachSource(url);<br />
  
          }<br />
  
      &lt;/script&gt;
      &lt;body onload=&quot;startVideo()&quot;&gt;
  
              
          
  

  How i grab and push video stream

   raspivid -w 640 -h 480 -fps 25 -t 0 -b 1800000 -o - | ffmpeg -y -f h264 -i - -vcodec libx264  -f flv -rtmp_buffer 100 -rtmp_live live rtmp://localhost:1935/rtmp/stream

  When i run the above command, i see the following on the terminal

      ffmpeg version N-81256-gd3426fb Copyright (c) 2000-2016 the FFmpeg developers
  
    built with gcc 4.9.2 (Raspbian 4.9.2-10)
  
    configuration: --enable-gpl --enable-libx264 --enable-nonfree
  
    libavutil      55. 28.100 / 55. 28.100
  
    libavcodec     57. 51.100 / 57. 51.100
  
    libavformat    57. 44.100 / 57. 44.100
  
    libavdevice    57.  0.102 / 57.  0.102
  
    libavfilter     6. 49.100 /  6. 49.100
  
    libswscale      4.  1.100 /  4.  1.100
  
    libswresample   2.  1.100 /  2.  1.100
  
    libpostproc    54.  0.100 / 54.  0.100
  
  Input #0, h264, from 'pipe:':
  
    Duration: N/A, bitrate: N/A
  
      Stream #0:0: Video: h264 (High), yuv420p, 640x480, 25 fps, 25 tbr, 1200k tbn, 50 tbc
  
  [tcp @ 0x2c3c850] Connection to tcp://localhost:1935 failed (Connection refused), trying next address
  
  [libx264 @ 0x2c50d80] using cpu capabilities: ARMv6 NEON
  
  [libx264 @ 0x2c50d80] profile High, level 3.0
  
  [libx264 @ 0x2c50d80] 264 - core 148 r2705 3f5ed56 - H.264/MPEG-4 AVC codec - Copyleft 2003-2016 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=6 lookahead_threads=1 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=25 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00
  
  [flv @ 0x2c3b950] Using AVStream.codec to pass codec parameters to muxers is deprecated, use AVStream.codecpar instead.
  
  Output #0, flv, to 'rtmp://localhost:1935/rtmp/stream':
  
    Metadata:
  
      encoder         : Lavf57.44.100
  
      Stream #0:0: Video: h264 (libx264) ([7][0][0][0] / 0x0007), yuv420p, 640x480, q=-1--1, 25 fps, 1k tbn, 25 tbc
  
      Metadata:
  
        encoder         : Lavc57.51.100 libx264
  
      Side data:
  
        cpb: bitrate max/min/avg: 0/0/0 buffer size: 0 vbv_delay: -1
  
  Stream mapping:
  
    Stream #0:0 -> #0:0 (h264 (native) -> h264 (libx264))
  
  ^Cmmal: Aborting program.0 size=     930kB time=00:00:45.44 bitrate= 167.7kbits/s speed=1.03x     
  
  
  
  [flv @ 0x2c3b950] Failed to update header with correct duration.
  
  [flv @ 0x2c3b950] Failed to update header with correct filesize.
  
  frame= 1197 fps= 26 q=-1.0 Lsize=     976kB time=00:00:47.76 bitrate= 167.5kbits/s speed=1.04x    
  
  video:953kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 2.484349%
  
  [libx264 @ 0x2c50d80] frame I:5     Avg QP:18.91  size:  3600
  
  [libx264 @ 0x2c50d80] frame P:299   Avg QP:21.38  size:  1486
  
  [libx264 @ 0x2c50d80] frame B:893   Avg QP:20.62  size:   574
  
  [libx264 @ 0x2c50d80] consecutive B-frames:  0.4%  0.0%  1.0% 98.6%
  
  [libx264 @ 0x2c50d80] mb I  I16..4: 10.1% 87.1%  2.8%
  
  [libx264 @ 0x2c50d80] mb P  I16..4:  3.8%  7.4%  0.0%  P16..4: 34.2%  2.1%  1.5%  0.0%  0.0%    skip:51.0%
  
  [libx264 @ 0x2c50d80] mb B  I16..4:  0.2%  0.3%  0.0%  B16..8: 20.1%  0.4%  0.0%  direct: 3.8%  skip:75.1%  L0:48.6% L1:50.9% BI: 0.6%
  
  [libx264 @ 0x2c50d80] 8x8 transform intra:68.1% inter:97.0%
  
  [libx264 @ 0x2c50d80] coded y,uvDC,uvAC intra: 10.1% 27.5% 1.8% inter: 2.0% 12.9% 0.1%
  
  [libx264 @ 0x2c50d80] i16 v,h,dc,p: 18% 20%  9% 53%
  
  [libx264 @ 0x2c50d80] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 21% 11% 58%  2%  2%  1%  2%  1%  1%
  
  [libx264 @ 0x2c50d80] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 22% 26% 30%  3%  4%  4%  6%  2%  2%
  
  [libx264 @ 0x2c50d80] i8c dc,h,v,p: 66% 18% 15%  1%
  
  [libx264 @ 0x2c50d80] Weighted P-Frames: Y:1.7% UV:1.3%
  
  [libx264 @ 0x2c50d80] ref P L0: 58.2%  2.2% 25.5% 14.1%  0.1%
  
  [libx264 @ 0x2c50d80] ref B L0: 81.7% 13.3%  5.0%
  
  [libx264 @ 0x2c50d80] ref B L1: 93.6%  6.4%
  
  [libx264 @ 0x2c50d80] kb/s:162.87
  
  Exiting normally, received signal 2.

  On my var/www/dash directory

   stream-0.m4a  stream-19200.m4a  stream-29200.m4a  
  
   stream-9600.m4a    stream-init.m4a  stream.mpd      stream-raw.m4v
  
   stream-0.m4v  stream-19200.m4v  stream-29200.m4v  stream-9600.m4v    stream-init.m4v  stream-raw.m4a

• Recording a video using MediaRecorder

  21 juillet 2016, par Cédric Portmann

  I am currently using the TextureFromCameraActivity from Grafika to record a video in square ( 1:1 ) resolution. Therefor I the GLES20.glViewport so that the video gets moved to the top and it appears to be squared. Now I would like to record this square view using the MediaRecorder or at least record the camera with normal resolutiona and then crop it using FFmpeg. However I get the same error over and over again and I cant figure out why.

  The error I get :

  start called in an invalid state : 4

  And yes I added all the necessary permissions.

  android.permission.WRITE_EXTERNAL_STORAGE android.permission.CAMERA
  android.permission.RECORD_VIDEO android.permission.RECORD_AUDIO
  android.permission.STORAGE android.permission.READ_EXTERNAL_STORAGE

  Here the modified code :

  https://github.com/google/grafika

  Thanks for your help :D

  package com.android.grafika;
  
  
  
  import android.graphics.SurfaceTexture;
  
  import android.hardware.Camera;
  
  import android.media.CamcorderProfile;
  
  import android.media.MediaRecorder;
  
  import android.opengl.GLES20;
  
  import android.opengl.Matrix;
  
  import android.os.Bundle;
  
  import android.os.Environment;
  
  import android.os.Handler;
  
  import android.os.Looper;
  
  import android.os.Message;
  
  import android.util.Log;
  
  import android.view.MotionEvent;
  
  import android.view.Surface;
  
  import android.view.SurfaceHolder;
  
  import android.view.SurfaceView;
  
  import android.view.View;
  
  import android.widget.Button;
  
  import android.widget.SeekBar;
  
  import android.widget.TextView;
  
  import android.app.Activity;
  
  import android.widget.Toast;
  
  
  
  import com.android.grafika.gles.Drawable2d;
  
  import com.android.grafika.gles.EglCore;
  
  import com.android.grafika.gles.GlUtil;
  
  import com.android.grafika.gles.Sprite2d;
  
  import com.android.grafika.gles.Texture2dProgram;
  
  import com.android.grafika.gles.WindowSurface;
  
  
  
  import java.io.File;
  
  import java.io.IOException;
  
  import java.lang.ref.WeakReference;
  
  
  
  
  
  public class TextureFromCameraActivity extends Activity implements View.OnClickListener, SurfaceHolder.Callback,
  
          SeekBar.OnSeekBarChangeListener {
  
  
  
  
  
      private static final int DEFAULT_ZOOM_PERCENT = 0;      // 0-100
  
      private static final int DEFAULT_SIZE_PERCENT = 80;     // 0-100
  
      private static final int DEFAULT_ROTATE_PERCENT = 75;    // 0-100
  
  
  
      // Requested values; actual may differ.
  
      private static final int REQ_CAMERA_WIDTH = 720;
  
      private static final int REQ_CAMERA_HEIGHT = 720;
  
      private static final int REQ_CAMERA_FPS = 30;
  
  
  
      // The holder for our SurfaceView.  The Surface can outlive the Activity (e.g. when
  
      // the screen is turned off and back on with the power button).
  
      //
  
      // This becomes non-null after the surfaceCreated() callback is called, and gets set
  
      // to null when surfaceDestroyed() is called.
  
      private static SurfaceHolder sSurfaceHolder;
  
  
  
      // Thread that handles rendering and controls the camera.  Started in onResume(),
  
      // stopped in onPause().
  
      private RenderThread mRenderThread;
  
  
  
      // Receives messages from renderer thread.
  
      private MainHandler mHandler;
  
  
  
      // User controls.
  
      private SeekBar mZoomBar;
  
      private SeekBar mSizeBar;
  
      private SeekBar mRotateBar;
  
  
  
      // These values are passed to us by the camera/render thread, and displayed in the UI.
  
      // We could also just peek at the values in the RenderThread object, but we'd need to
  
      // synchronize access carefully.
  
      private int mCameraPreviewWidth, mCameraPreviewHeight;
  
      private float mCameraPreviewFps;
  
      private int mRectWidth, mRectHeight;
  
      private int mZoomWidth, mZoomHeight;
  
      private int mRotateDeg;
  
      SurfaceHolder sh;
  
      MediaRecorder recorder;
  
      SurfaceHolder holder;
  
      boolean recording = false;
  
  
  
      public static final String TAG = "VIDEOCAPTURE";
  
  
  
      private static final File OUTPUT_DIR = Environment.getExternalStorageDirectory();
  
  
  
  
  
      @Override
  
      protected void onCreate(Bundle savedInstanceState) {
  
          super.onCreate(savedInstanceState);
  
  
  
          recorder = new MediaRecorder();
  
  
  
  
  
  
  
          setContentView(R.layout.activity_texture_from_camera);
  
  
  
          mHandler = new MainHandler(this);
  
  
  
          SurfaceView cameraView = (SurfaceView) findViewById(R.id.cameraOnTexture_surfaceView);
  
          sh = cameraView.getHolder();
  
          cameraView.setClickable(true);// make the surface view clickable
  
          sh.addCallback(this);
  
  
  
  
  
          //prepareRecorder();
  
  
  
  
  
          mZoomBar = (SeekBar) findViewById(R.id.tfcZoom_seekbar);
  
          mSizeBar = (SeekBar) findViewById(R.id.tfcSize_seekbar);
  
          mRotateBar = (SeekBar) findViewById(R.id.tfcRotate_seekbar);
  
          mZoomBar.setProgress(DEFAULT_ZOOM_PERCENT);
  
          mSizeBar.setProgress(DEFAULT_SIZE_PERCENT);
  
          mRotateBar.setProgress(DEFAULT_ROTATE_PERCENT);
  
          mZoomBar.setOnSeekBarChangeListener(this);
  
          mSizeBar.setOnSeekBarChangeListener(this);
  
          mRotateBar.setOnSeekBarChangeListener(this);
  
  
  
          Button record_btn = (Button)findViewById(R.id.button);
  
          record_btn.setOnClickListener(this);
  
          initRecorder();
  
  
  
  
  
          updateControls();
  
  
  
  
  
  
  
  
  
      }
  
  
  
  
  
  
  
  
  
  
  
      @Override
  
      protected void onResume() {
  
          Log.d(TAG, "onResume BEGIN");
  
          super.onResume();
  
  
  
          mRenderThread = new RenderThread(mHandler);
  
          mRenderThread.setName("TexFromCam Render");
  
          mRenderThread.start();
  
          mRenderThread.waitUntilReady();
  
  
  
          RenderHandler rh = mRenderThread.getHandler();
  
          rh.sendZoomValue(mZoomBar.getProgress());
  
          rh.sendSizeValue(mSizeBar.getProgress());
  
          rh.sendRotateValue(mRotateBar.getProgress());
  
  
  
          if (sSurfaceHolder != null) {
  
              Log.d(TAG, "Sending previous surface");
  
              rh.sendSurfaceAvailable(sSurfaceHolder, false);
  
          } else {
  
              Log.d(TAG, "No previous surface");
  
          }
  
          Log.d(TAG, "onResume END");
  
      }
  
  
  
      @Override
  
      protected void onPause() {
  
          Log.d(TAG, "onPause BEGIN");
  
          super.onPause();
  
  
  
          RenderHandler rh = mRenderThread.getHandler();
  
          rh.sendShutdown();
  
          try {
  
              mRenderThread.join();
  
          } catch (InterruptedException ie) {
  
              // not expected
  
              throw new RuntimeException("join was interrupted", ie);
  
          }
  
          mRenderThread = null;
  
          Log.d(TAG, "onPause END");
  
      }
  
  
  
      @Override   // SurfaceHolder.Callback
  
      public void surfaceCreated(SurfaceHolder holder) {
  
          Log.d(TAG, "surfaceCreated holder=" + holder + " (static=" + sSurfaceHolder + ")");
  
          if (sSurfaceHolder != null) {
  
              throw new RuntimeException("sSurfaceHolder is already set");
  
          }
  
  
  
          sSurfaceHolder = holder;
  
  
  
          if (mRenderThread != null) {
  
              // Normal case -- render thread is running, tell it about the new surface.
  
              RenderHandler rh = mRenderThread.getHandler();
  
              rh.sendSurfaceAvailable(holder, true);
  
          } else {
  
              // Sometimes see this on 4.4.x N5: power off, power on, unlock, with device in
  
              // landscape and a lock screen that requires portrait.  The surface-created
  
              // message is showing up after onPause().
  
              //
  
              // Chances are good that the surface will be destroyed before the activity is
  
              // unpaused, but we track it anyway.  If the activity is un-paused and we start
  
              // the RenderThread, the SurfaceHolder will be passed in right after the thread
  
              // is created.
  
              Log.d(TAG, "render thread not running");
  
          }
  
  
  
          recorder.setPreviewDisplay(holder.getSurface());
  
  
  
      }
  
  
  
      @Override   // SurfaceHolder.Callback
  
      public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
  
          Log.d(TAG, "surfaceChanged fmt=" + format + " size=" + width + "x" + height +
  
                  " holder=" + holder);
  
  
  
          if (mRenderThread != null) {
  
              RenderHandler rh = mRenderThread.getHandler();
  
              rh.sendSurfaceChanged(format, width, height);
  
          } else {
  
              Log.d(TAG, "Ignoring surfaceChanged");
  
              return;
  
          }
  
      }
  
  
  
      @Override   // SurfaceHolder.Callback
  
      public void surfaceDestroyed(SurfaceHolder holder) {
  
          // In theory we should tell the RenderThread that the surface has been destroyed.
  
          if (mRenderThread != null) {
  
              RenderHandler rh = mRenderThread.getHandler();
  
              rh.sendSurfaceDestroyed();
  
          }
  
          Log.d(TAG, "surfaceDestroyed holder=" + holder);
  
          sSurfaceHolder = null;
  
      }
  
  
  
      @Override   // SeekBar.OnSeekBarChangeListener
  
      public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) {
  
          if (mRenderThread == null) {
  
              // Could happen if we programmatically update the values after setting a listener
  
              // but before starting the thread.  Also, easy to cause this by scrubbing the seek
  
              // bar with one finger then tapping "recents" with another.
  
              Log.w(TAG, "Ignoring onProgressChanged received w/o RT running");
  
              return;
  
          }
  
          RenderHandler rh = mRenderThread.getHandler();
  
  
  
          // "progress" ranges from 0 to 100
  
          if (seekBar == mZoomBar) {
  
              //Log.v(TAG, "zoom: " + progress);
  
              rh.sendZoomValue(progress);
  
          } else if (seekBar == mSizeBar) {
  
              //Log.v(TAG, "size: " + progress);
  
              rh.sendSizeValue(progress);
  
          } else if (seekBar == mRotateBar) {
  
              //Log.v(TAG, "rotate: " + progress);
  
              rh.sendRotateValue(progress);
  
          } else {
  
              throw new RuntimeException("unknown seek bar");
  
          }
  
  
  
          // If we're getting preview frames quickly enough we don't really need this, but
  
          // we don't want to have chunky-looking resize movement if the camera is slow.
  
          // OTOH, if we get the updates too quickly (60fps camera?), this could jam us
  
          // up and cause us to run behind.  So use with caution.
  
          rh.sendRedraw();
  
      }
  
  
  
      @Override   // SeekBar.OnSeekBarChangeListener
  
      public void onStartTrackingTouch(SeekBar seekBar) {}
  
      @Override   // SeekBar.OnSeekBarChangeListener
  
      public void onStopTrackingTouch(SeekBar seekBar) {}
  
      @Override
  
  
  
      /**
  
       * Handles any touch events that aren't grabbed by one of the controls.
  
       */
  
      public boolean onTouchEvent(MotionEvent e) {
  
          float x = e.getX();
  
          float y = e.getY();
  
  
  
          switch (e.getAction()) {
  
              case MotionEvent.ACTION_MOVE:
  
              case MotionEvent.ACTION_DOWN:
  
                  //Log.v(TAG, "onTouchEvent act=" + e.getAction() + " x=" + x + " y=" + y);
  
                  if (mRenderThread != null) {
  
                      RenderHandler rh = mRenderThread.getHandler();
  
                      rh.sendPosition((int) x, (int) y);
  
  
  
                      // Forcing a redraw can cause sluggish-looking behavior if the touch
  
                      // events arrive quickly.
  
                      //rh.sendRedraw();
  
                  }
  
                  break;
  
              default:
  
                  break;
  
          }
  
  
  
          return true;
  
      }
  
  
  
      /**
  
       * Updates the current state of the controls.
  
       */
  
      private void updateControls() {
  
          String str = getString(R.string.tfcCameraParams, mCameraPreviewWidth,
  
                  mCameraPreviewHeight, mCameraPreviewFps);
  
          TextView tv = (TextView) findViewById(R.id.tfcCameraParams_text);
  
          tv.setText(str);
  
  
  
          str = getString(R.string.tfcRectSize, mRectWidth, mRectHeight);
  
          tv = (TextView) findViewById(R.id.tfcRectSize_text);
  
          tv.setText(str);
  
  
  
          str = getString(R.string.tfcZoomArea, mZoomWidth, mZoomHeight);
  
          tv = (TextView) findViewById(R.id.tfcZoomArea_text);
  
          tv.setText(str);
  
      }
  
  
  
      @Override
  
      public void onClick(View view) {
  
  
  
          if (recording) {
  
              recorder.stop();
  
              recording = false;
  
  
  
              // Let's initRecorder so we can record again
  
              initRecorder();
  
              prepareRecorder();
  
          } else {
  
              recording = true;
  
              recorder.start();
  
          }
  
      }
  
  
  
  
  
      private void initRecorder() {
  
          recorder.setAudioSource(MediaRecorder.AudioSource.DEFAULT);
  
          recorder.setVideoSource(MediaRecorder.VideoSource.DEFAULT);
  
  
  
          CamcorderProfile cpHigh = CamcorderProfile
  
                  .get(CamcorderProfile.QUALITY_HIGH);
  
          recorder.setProfile(cpHigh);
  
          String path = Environment.getExternalStorageDirectory() + File.separator
  
                  + Environment.DIRECTORY_DCIM + File.separator + "AlphaRun";
  
  
  
          recorder.setOutputFile(path);
  
          recorder.setMaxDuration(50000); // 50 seconds
  
          recorder.setMaxFileSize(5000000); // Approximately 5 megabytes
  
  
  
      }
  
  
  
      private void prepareRecorder() {
  
  
  
  
  
          try {
  
              recorder.prepare();
  
          } catch (IllegalStateException e) {
  
              e.printStackTrace();
  
              finish();
  
          } catch (IOException e) {
  
              e.printStackTrace();
  
              finish();
  
          }
  
      }
  
  
  
  
  
  
  
  
  
      /**
  
       * Thread that handles all rendering and camera operations.
  
       */
  
      private static class RenderThread extends Thread implements
  
              SurfaceTexture.OnFrameAvailableListener {
  
          // Object must be created on render thread to get correct Looper, but is used from
  
          // UI thread, so we need to declare it volatile to ensure the UI thread sees a fully
  
          // constructed object.
  
          private volatile RenderHandler mHandler;
  
  
  
          // Used to wait for the thread to start.
  
          private Object mStartLock = new Object();
  
          private boolean mReady = false;
  
  
  
          private MainHandler mMainHandler;
  
  
  
          private Camera mCamera;
  
          private int mCameraPreviewWidth, mCameraPreviewHeight;
  
  
  
          private EglCore mEglCore;
  
          private WindowSurface mWindowSurface;
  
          private int mWindowSurfaceWidth;
  
          private int mWindowSurfaceHeight;
  
  
  
          // Receives the output from the camera preview.
  
          private SurfaceTexture mCameraTexture;
  
  
  
          // Orthographic projection matrix.
  
          private float[] mDisplayProjectionMatrix = new float[16];
  
  
  
          private Texture2dProgram mTexProgram;
  
          private final ScaledDrawable2d mRectDrawable =
  
                  new ScaledDrawable2d(Drawable2d.Prefab.RECTANGLE);
  
          private final Sprite2d mRect = new Sprite2d(mRectDrawable);
  
  
  
          private int mZoomPercent = DEFAULT_ZOOM_PERCENT;
  
          private int mSizePercent = DEFAULT_SIZE_PERCENT;
  
          private int mRotatePercent = DEFAULT_ROTATE_PERCENT;
  
          private float mPosX, mPosY;
  
  
  
  
  
          /**
  
           * Constructor.  Pass in the MainHandler, which allows us to send stuff back to the
  
           * Activity.
  
           */
  
          public RenderThread(MainHandler handler) {
  
              mMainHandler = handler;
  
  
  
          }
  
  
  
          /**
  
           * Thread entry point.
  
           */
  
          @Override
  
          public void run() {
  
              Looper.prepare();
  
  
  
              // We need to create the Handler before reporting ready.
  
              mHandler = new RenderHandler(this);
  
              synchronized (mStartLock) {
  
                  mReady = true;
  
                  mStartLock.notify();    // signal waitUntilReady()
  
              }
  
  
  
              // Prepare EGL and open the camera before we start handling messages.
  
              mEglCore = new EglCore(null, 0);
  
              openCamera(REQ_CAMERA_WIDTH, REQ_CAMERA_HEIGHT, REQ_CAMERA_FPS);
  
  
  
              Looper.loop();
  
  
  
              Log.d(TAG, "looper quit");
  
              releaseCamera();
  
              releaseGl();
  
              mEglCore.release();
  
  
  
              synchronized (mStartLock) {
  
                  mReady = false;
  
              }
  
          }
  
  
  
          /**
  
           * Waits until the render thread is ready to receive messages.
  
           * <p>
  
           * Call from the UI thread.
  
           */
  
          public void waitUntilReady() {
  
              synchronized (mStartLock) {
  
                  while (!mReady) {
  
                      try {
  
                          mStartLock.wait();
  
                      } catch (InterruptedException ie) { /* not expected */ }
  
                  }
  
              }
  
          }
  
  
  
          /**
  
           * Shuts everything down.
  
           */
  
          private void shutdown() {
  
              Log.d(TAG, "shutdown");
  
              Looper.myLooper().quit();
  
          }
  
  
  
          /**
  
           * Returns the render thread's Handler.  This may be called from any thread.
  
           */
  
          public RenderHandler getHandler() {
  
              return mHandler;
  
          }
  
  
  
          /**
  
           * Handles the surface-created callback from SurfaceView.  Prepares GLES and the Surface.
  
           */
  
          private void surfaceAvailable(SurfaceHolder holder, boolean newSurface) {
  
  
  
              Surface surface = holder.getSurface();
  
              mWindowSurface = new WindowSurface(mEglCore, surface, false);
  
              mWindowSurface.makeCurrent();
  
  
  
              // Create and configure the SurfaceTexture, which will receive frames from the
  
              // camera.  We set the textured rect's program to render from it.
  
              mTexProgram = new Texture2dProgram(Texture2dProgram.ProgramType.TEXTURE_EXT);
  
              int textureId = mTexProgram.createTextureObject();
  
              mCameraTexture = new SurfaceTexture(textureId);
  
              mRect.setTexture(textureId);
  
  
  
              if (!newSurface) {
  
                  // This Surface was established on a previous run, so no surfaceChanged()
  
                  // message is forthcoming.  Finish the surface setup now.
  
                  //
  
                  // We could also just call this unconditionally, and perhaps do an unnecessary
  
                  // bit of reallocating if a surface-changed message arrives.
  
                  mWindowSurfaceWidth = mWindowSurface.getWidth();
  
                  mWindowSurfaceHeight = mWindowSurface.getWidth();
  
                  finishSurfaceSetup();
  
              }
  
  
  
              mCameraTexture.setOnFrameAvailableListener(this);
  
  
  
  
  
  
  
          }
  
  
  
          /**
  
           * Releases most of the GL resources we currently hold (anything allocated by
  
           * surfaceAvailable()).
  
           * </p><p>
  
           * Does not release EglCore.
  
           */
  
          private void releaseGl() {
  
              GlUtil.checkGlError("releaseGl start");
  
  
  
              if (mWindowSurface != null) {
  
                  mWindowSurface.release();
  
                  mWindowSurface = null;
  
              }
  
              if (mTexProgram != null) {
  
                  mTexProgram.release();
  
                  mTexProgram = null;
  
              }
  
              GlUtil.checkGlError("releaseGl done");
  
  
  
              mEglCore.makeNothingCurrent();
  
          }
  
  
  
          /**
  
           * Handles the surfaceChanged message.
  
           * </p><p>
  
           * We always receive surfaceChanged() after surfaceCreated(), but surfaceAvailable()
  
           * could also be called with a Surface created on a previous run.  So this may not
  
           * be called.
  
           */
  
          private void surfaceChanged(int width, int height) {
  
              Log.d(TAG, "RenderThread surfaceChanged " + width + "x" + height);
  
  
  
              mWindowSurfaceWidth = width;
  
              mWindowSurfaceHeight = width;
  
              finishSurfaceSetup();
  
          }
  
  
  
          /**
  
           * Handles the surfaceDestroyed message.
  
           */
  
          private void surfaceDestroyed() {
  
              // In practice this never appears to be called -- the activity is always paused
  
              // before the surface is destroyed.  In theory it could be called though.
  
              Log.d(TAG, "RenderThread surfaceDestroyed");
  
              releaseGl();
  
          }
  
  
  
          /**
  
           * Sets up anything that depends on the window size.
  
           * </p><p>
  
           * Open the camera (to set mCameraAspectRatio) before calling here.
  
           */
  
          private void finishSurfaceSetup() {
  
              int width = mWindowSurfaceWidth;
  
              int height = mWindowSurfaceHeight;
  
              Log.d(TAG, "finishSurfaceSetup size=" + width + "x" + height +
  
                      " camera=" + mCameraPreviewWidth + "x" + mCameraPreviewHeight);
  
  
  
              // Use full window.
  
              GLES20.glViewport(0, 700, width, height);
  
  
  
              // Simple orthographic projection, with (0,0) in lower-left corner.
  
              Matrix.orthoM(mDisplayProjectionMatrix, 0, 0, width, 0, height, -1, 1);
  
  
  
              // Default position is center of screen.
  
              mPosX = width / 2.0f;
  
              mPosY = height / 2.0f;
  
  
  
              updateGeometry();
  
  
  
              // Ready to go, start the camera.
  
              Log.d(TAG, "starting camera preview");
  
              try {
  
                  mCamera.setPreviewTexture(mCameraTexture);
  
  
  
              } catch (IOException ioe) {
  
                  throw new RuntimeException(ioe);
  
              }
  
              mCamera.startPreview();
  
          }
  
  
  
          /**
  
           * Updates the geometry of mRect, based on the size of the window and the current
  
           * values set by the UI.
  
           */
  
          private void updateGeometry() {
  
              int width = mWindowSurfaceWidth;
  
              int height = mWindowSurfaceHeight;
  
  
  
  
  
              int smallDim = Math.min(width, height);
  
              // Max scale is a bit larger than the screen, so we can show over-size.
  
              float scaled = smallDim * (mSizePercent / 100.0f) * 1.25f;
  
              float cameraAspect = (float) mCameraPreviewWidth / mCameraPreviewHeight;
  
              int newWidth = Math.round(scaled * cameraAspect);
  
              int newHeight = Math.round(scaled);
  
  
  
              float zoomFactor = 1.0f - (mZoomPercent / 100.0f);
  
              int rotAngle = Math.round(360 * (mRotatePercent / 100.0f));
  
  
  
              mRect.setScale(newWidth, newHeight);
  
              mRect.setPosition(mPosX, mPosY);
  
              mRect.setRotation(rotAngle);
  
              mRectDrawable.setScale(zoomFactor);
  
  
  
              mMainHandler.sendRectSize(newWidth, newHeight);
  
              mMainHandler.sendZoomArea(Math.round(mCameraPreviewWidth * zoomFactor),
  
                      Math.round(mCameraPreviewHeight * zoomFactor));
  
              mMainHandler.sendRotateDeg(rotAngle);
  
          }
  
  
  
          @Override   // SurfaceTexture.OnFrameAvailableListener; runs on arbitrary thread
  
          public void onFrameAvailable(SurfaceTexture surfaceTexture) {
  
              mHandler.sendFrameAvailable();
  
          }
  
  
  
          /**
  
           * Handles incoming frame of data from the camera.
  
           */
  
          private void frameAvailable() {
  
              mCameraTexture.updateTexImage();
  
  
  
              draw();
  
          }
  
  
  
          /**
  
           * Draws the scene and submits the buffer.
  
           */
  
          private void draw() {
  
              GlUtil.checkGlError("draw start");
  
  
  
              GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
  
              GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
  
              mRect.draw(mTexProgram, mDisplayProjectionMatrix);
  
              mWindowSurface.swapBuffers();
  
  
  
              GlUtil.checkGlError("draw done");
  
          }
  
  
  
          /**
  
           * Opens a camera, and attempts to establish preview mode at the specified width
  
           * and height with a fixed frame rate.
  
           * </p><p>
  
           * Sets mCameraPreviewWidth / mCameraPreviewHeight.
  
           */
  
          private void openCamera(int desiredWidth, int desiredHeight, int desiredFps) {
  
              if (mCamera != null) {
  
                  throw new RuntimeException("camera already initialized");
  
              }
  
  
  
              Camera.CameraInfo info = new Camera.CameraInfo();
  
  
  
              // Try to find a front-facing camera (e.g. for videoconferencing).
  
              int numCameras = Camera.getNumberOfCameras();
  
              for (int i = 0; i &lt; numCameras; i++) {
  
                  Camera.getCameraInfo(i, info);
  
                  if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) {
  
                      mCamera = Camera.open(i);
  
                      break;
  
                  }
  
              }
  
              if (mCamera == null) {
  
                  Log.d(TAG, "No front-facing camera found; opening default");
  
                  mCamera = Camera.open();    // opens first back-facing camera
  
              }
  
              if (mCamera == null) {
  
                  throw new RuntimeException("Unable to open camera");
  
              }
  
  
  
              Camera.Parameters parms = mCamera.getParameters();
  
  
  
              CameraUtils.choosePreviewSize(parms, desiredWidth, desiredHeight);
  
              parms.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_PICTURE);
  
              // Try to set the frame rate to a constant value.
  
              int thousandFps = CameraUtils.chooseFixedPreviewFps(parms, desiredFps * 1000);
  
  
  
              // Give the camera a hint that we're recording video.  This can have a big
  
              // impact on frame rate.
  
              parms.setRecordingHint(true);
  
  
  
              mCamera.setParameters(parms);
  
  
  
              int[] fpsRange = new int[2];
  
              Camera.Size mCameraPreviewSize = parms.getPreviewSize();
  
              parms.getPreviewFpsRange(fpsRange);
  
              String previewFacts = mCameraPreviewSize.width + "x" + mCameraPreviewSize.height;
  
              if (fpsRange[0] == fpsRange[1]) {
  
                  previewFacts += " @" + (fpsRange[0] / 1000.0) + "fps";
  
              } else {
  
                  previewFacts += " @[" + (fpsRange[0] / 1000.0) +
  
                          " - " + (fpsRange[1] / 1000.0) + "] fps";
  
              }
  
              Log.i(TAG, "Camera config: " + previewFacts);
  
  
  
              mCameraPreviewWidth = mCameraPreviewSize.width;
  
              mCameraPreviewHeight = mCameraPreviewSize.height;
  
              mMainHandler.sendCameraParams(mCameraPreviewWidth, mCameraPreviewHeight,
  
                      thousandFps / 1000.0f);
  
          }
  
  
  
          /**
  
           * Stops camera preview, and releases the camera to the system.
  
           */
  
          private void releaseCamera() {
  
              if (mCamera != null) {
  
                  mCamera.stopPreview();
  
                  mCamera.release();
  
                  mCamera = null;
  
                  Log.d(TAG, "releaseCamera -- done");
  
              }
  
          }
  
      }
  
  
  
  }
  
  </p>