Recherche avancée

Sur d’autres sites (7206)

• Revision 0e23048303 : BITSTREAM - "update_map" SEMANTICS BROKEN IN 398ddafb629b7f49cf255bf09d3e38b4abd

  3 octobre 2013, par Dmitry Kovalev

  Changed Paths :
   Modify /vp9/decoder/vp9_decodemv.c

  
   Modify /vp9/decoder/vp9_decodframe.c

  
  
  BITSTREAM - "update_map" SEMANTICS BROKEN IN
  398ddafb629b7f49cf255bf09d3e38b4abd0bb95

  This patch reverts old commit 398ddafb629b7f49cf255bf09d3e38b4abd0bb95
  "New way of updating last frame segmentation map.".

  Change-Id : Iba730f433c30ed7f5e5449d6768049cbf9a2b2c5

• H264 codec encode, decode and write to file

  30 novembre 2020, par Алекс Аникей

  I try to use ffmpeg and h264 codec to translate the video in realtime. But at the state of decoding encoded frame, I get some "bad" image.
Init encoder and decoder :

  


      VCSession *vc_new_x264(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_receive_frame_cb *cb, void *cb_data,
                       VCSession *vc)
{

// ------ ffmpeg encoder ------
    AVCodec *codec2 = NULL;
    vc->h264_encoder_ctx = NULL;//AVCodecContext type

    codec2 = NULL;
    avcodec_register_all();
    codec2 = avcodec_find_encoder(AV_CODEC_ID_H264);
    if (codec2 == NULL)
    {
        LOGGER_WARNING(log, "h264: not find encoder");
    }

    vc->h264_encoder_ctx = avcodec_alloc_context3(codec2);

    vc->h264_out_pic2 = av_packet_alloc();

    vc->h264_encoder_ctx->bit_rate = 10 *1000 * 1000;
    vc->h264_encoder_ctx->width = 800;
    vc->h264_encoder_ctx->height = 600;

    vc->h264_enc_width = vc->h264_encoder_ctx->width;
    vc->h264_enc_height = vc->h264_encoder_ctx->height;
    vc->h264_encoder_ctx->time_base = (AVRational) {
            1, 30
    };
    vc->h264_encoder_ctx->gop_size = 30;
    vc->h264_encoder_ctx->max_b_frames = 1;
    vc->h264_encoder_ctx->pix_fmt = AV_PIX_FMT_YUV420P;


    av_opt_set(vc->h264_encoder_ctx->priv_data, "preset", "veryfast", 0);


    av_opt_set(vc->h264_encoder_ctx->priv_data, "annex_b", "1", 0);
    av_opt_set(vc->h264_encoder_ctx->priv_data, "repeat_headers", "1", 0);
    av_opt_set(vc->h264_encoder_ctx->priv_data, "tune", "zerolatency", 0);
    av_opt_set_int(vc->h264_encoder_ctx->priv_data, "zerolatency", 1, 0);

    vc->h264_encoder_ctx->time_base.num = 1;
    vc->h264_encoder_ctx->time_base.den = 1000;

    vc->h264_encoder_ctx->framerate = (AVRational) {
        1000, 40
    };

    AVDictionary *opts = NULL;

    if (avcodec_open2(vc->h264_encoder_ctx, codec2, &opts) < 0) {
        LOGGER_ERROR(log, "could not open codec H264 on encoder");
    }

    av_dict_free(&opts);



    AVCodec *codec = NULL;
    vc->h264_decoder_ctx = NULL;// AVCodecContext - type
    codec = NULL;

    codec = avcodec_find_decoder(AV_CODEC_ID_H264);

    if (!codec) {
        LOGGER_WARNING(log, "codec not found H264 on decoder");
    }

    vc->h264_decoder_ctx = avcodec_alloc_context3(codec);

    if (codec->capabilities & AV_CODEC_CAP_TRUNCATED) {
        vc->h264_decoder_ctx->flags |= AV_CODEC_FLAG_TRUNCATED; /* we do not send complete frames */
    }

    if (codec->capabilities & AV_CODEC_FLAG_LOW_DELAY) {
        vc->h264_decoder_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
    }

    vc->h264_decoder_ctx->flags |= AV_CODEC_FLAG2_SHOW_ALL;

     vc->h264_decoder_ctx->refcounted_frames = 0;

    vc->h264_decoder_ctx->delay = 0;
    vc->h264_decoder_ctx->sw_pix_fmt = AV_PIX_FMT_YUV420P;
    av_opt_set_int(vc->h264_decoder_ctx->priv_data, "delay", 0, AV_OPT_SEARCH_CHILDREN);
    vc->h264_decoder_ctx->time_base = (AVRational) {
            40, 1000
};
    vc->h264_decoder_ctx->framerate = (AVRational) {
        1000, 40
    };

    if (avcodec_open2(vc->h264_decoder_ctx, codec, NULL) < 0) {
        LOGGER_WARNING(log, "could not open codec H264 on decoder");
    }
    vc->h264_decoder_ctx->refcounted_frames = 0;

    return vc;
}


  


  Encoding (in this function i encode frame and for debugging decode and save him in file) :

  


  uint32_t encode_frame_h264_p(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height,
                           const uint8_t *y,
                           const uint8_t *u, const uint8_t *v, ToxAVCall *call,
                           uint64_t *video_frame_record_timestamp,
                           int vpx_encode_flags,
                           x264_nal_t **nal,
                           int *i_frame_size)
{
    AVFrame *frame;
    int ret;
    uint32_t result = 1;

    frame = av_frame_alloc();

    frame->format = call->video->h264_encoder_ctx->pix_fmt;
    frame->width  = width;
    frame->height = height;

    ret = av_frame_get_buffer(frame, 32);

    if (ret < 0) {
        LOGGER_ERROR(av->m->log, "av_frame_get_buffer:Could not allocate the video frame data");
    }

    /* make sure the frame data is writable */
    ret = av_frame_make_writable(frame);

    if (ret < 0) {
        LOGGER_ERROR(av->m->log, "av_frame_make_writable:ERROR");
    }

    frame->pts = (int64_t)(*video_frame_record_timestamp);


    // copy YUV frame data into buffers
    memcpy(frame->data[0], y, width * height);
    memcpy(frame->data[1], u, (width / 2) * (height / 2));
    memcpy(frame->data[2], v, (width / 2) * (height / 2));

    // encode the frame
    ret = avcodec_send_frame(call->video->h264_encoder_ctx, frame);

    if (ret < 0) {
        LOGGER_ERROR(av->m->log, "Error sending a frame for encoding:ERROR");
    }


    ret = avcodec_receive_packet(call->video->h264_encoder_ctx, call->video->h264_out_pic2);



    if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
        *i_frame_size = 0;
    } else if (ret < 0) {
        *i_frame_size = 0;
        // fprintf(stderr, "Error during encoding\n");
    } else {

        // Decoded encoded frame and save him to file

        saveInFile(call->video->h264_decoder_ctx, frame, call->video->h264_out_pic2, "/home/user/testSave");

        // printf("Write packet %3"PRId64" (size=%5d)\n", call->video->h264_out_pic2->pts, call->video->h264_out_pic2->size);
        // fwrite(call->video->h264_out_pic2->data, 1, call->video->h264_out_pic2->size, outfile);

        global_encoder_delay_counter++;

        if (global_encoder_delay_counter > 60) {
            global_encoder_delay_counter = 0;
            LOGGER_DEBUG(av->m->log, "enc:delay=%ld",
                         (long int)(frame->pts - (int64_t)call->video->h264_out_pic2->pts)
                        );
        }


        *i_frame_size = call->video->h264_out_pic2->size;
        *video_frame_record_timestamp = (uint64_t)call->video->h264_out_pic2->pts;

        result = 0;
    }

    av_frame_free(&frame);

    return result;

}


  


  Decode and save frame code :

  


  void saveInFile(AVCodecContext *dec_ctx, AVFrame *frame, AVPacket *pkt, const char *filename)
{
    if (!pkt)
        return;
    char buf[1024];
    int ret;
    static int curNumber = 0;
    ret = avcodec_send_packet(dec_ctx, pkt);
    if (ret < 0 && ret != AVERROR_EOF)
    {
        fprintf(stderr, "Error sending a packet for decoding'\n");
        if ( ret == AVERROR(EAGAIN))
            return;
        if (ret == AVERROR(EINVAL))
            return;
        if (ret == AVERROR(ENOMEM))
            return;

    }

        ret = avcodec_receive_frame(dec_ctx, frame);
        if (ret == AVERROR(EAGAIN) )
            return;
        if (ret == AVERROR_EOF)
        {
            return;
        }
        else if (ret < 0)
        {
            fprintf(stderr, "Error during decoding\n");
        }
        printf("saving frame %3d\n", dec_ctx->frame_number);
        sprintf(buf, "%s%d", filename, curNumber);
        curNumber++;
        pgm_save(frame->data[0], frame->linesize[0], frame->width, frame->height, buf);

}

void pgm_save(unsigned char* buf, int wrap, int xsize, int ysize, char *filename)
{
    FILE *f;
    int i;
    f = fopen(filename, "w");
    fprintf(f, "P5\n%d %d\n%d\n", xsize, ysize, 255);
    for (i =0; i < ysize; i++)
        fwrite(buf + i* wrap, 1, xsize, f);
    fclose(f);
}


  


  After this manipulation I have smth like that :


  


• Frame sliced into 2 under some degree. How fix it ?

  26 novembre 2020, par Алекс Аникей

  I trying use h264 codec in videochat application. And in some reason frame sliced into 2 triangle (picture below). I try send my desktop image to another person and get this image on another client.

  


  What settings i set wrong ?
My code :

  


  Init :

  


  VCSession *vc_new_x264(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_receive_frame_cb *cb, void *cb_data,
                       VCSession *vc)
{

    if (x264_param_default_preset(&param, "slow", "zerolatency") < 0) {
        // goto fail;
    }

    param.i_csp = X264_CSP_I420;
    param.i_width  = 1920;
    param.i_height = 1080;
    vc->h264_enc_width = param.i_width;
    vc->h264_enc_height = param.i_height;

    param.i_keyint_max = 30;

    param.b_vfr_input = 1; /* VFR input.  If 1, use timebase and timestamps for ratecontrol purposes.
                            * If 0, use fps only. */
    param.i_timebase_num = 1;       // 1 ms = timebase units = (1/1000)s
    param.i_timebase_den = 1000;   // 1 ms = timebase units = (1/1000)s
    param.b_repeat_headers = 1;
    param.b_annexb = 1;

    param.rc.f_rate_tolerance = VIDEO_F_RATE_TOLERANCE_H264;
    param.rc.i_vbv_buffer_size = 1500;
    param.rc.i_vbv_max_bitrate = VIDEO_BITRATE_INITIAL_VALUE_H264 * 1;

    vc->h264_enc_bitrate = VIDEO_BITRATE_INITIAL_VALUE_H264;

    param.rc.i_qp_min = 13;
    param.rc.i_qp_max = 35; // max quantizer for x264

    vc->h264_enc_bitrate = VIDEO_BITRATE_INITIAL_VALUE_H264;

    param.rc.b_stat_read = 0;
    param.rc.b_stat_write = 0;


    if (x264_param_apply_profile(&param,
                                 "high") < 0) { // "baseline", "main", "high", "high10", "high422", "high444"
        // goto fail;
    }


    if (x264_picture_alloc(&(vc->h264_in_pic), param.i_csp, param.i_width, param.i_height) < 0) {
        // goto fail;
    }

    vc->h264_encoder = x264_encoder_open(&param);

    AVCodec *codec = NULL;
    vc->h264_decoder = NULL;
    avcodec_register_all();
    codec = NULL;

    codec = avcodec_find_decoder(AV_CODEC_ID_H264);

    if (!codec) {
        LOGGER_WARNING(log, "codec not found H264 on decoder");
    }

    vc->h264_decoder = avcodec_alloc_context3(codec);

    if (codec->capabilities & AV_CODEC_CAP_TRUNCATED) {
        vc->h264_decoder->flags |= AV_CODEC_FLAG_TRUNCATED; /* we do not send complete frames */
    }


    vc->h264_decoder->delay = 5;

    if (avcodec_open2(vc->h264_decoder, codec, NULL) < 0) {
        LOGGER_WARNING(log, "could not open codec H264 on decoder");
    }


    return vc;
}


  


  Get frame and decoding him :

  


  void vc_iterate_x264(VCSession *vc)
{

    if (!vc) {
        return;
    }

    pthread_mutex_lock(vc->queue_mutex);

    struct RTPMessage *p;

    if (!rb_read(vc->vbuf_raw, (void **)&p)) {
        LOGGER_TRACE(vc->log, "no Video frame data available");
        pthread_mutex_unlock(vc->queue_mutex);
        return;
    }

    pthread_mutex_unlock(vc->queue_mutex);
    const struct RTPHeader *const header = &p->header;

    uint32_t full_data_len;

    if (header->flags & RTP_LARGE_FRAME) {
        full_data_len = header->data_length_full;
        LOGGER_WARNING(vc->log, "vc_iterate:001:full_data_len=%d", (int)full_data_len);
    } else {
        full_data_len = p->len;
        if (header->data_length_lower != full_data_len)
        {
            LOGGER_ERROR("Data header and packet don't equal: %d - header %d - packet", header->data_length_lower, full_data_len);
        }
        LOGGER_DEBUG(vc->log, "vc_iterate:002");
    }

    decode_frame_h264(vc, p, full_data_len);
}

void decode_frame_h264(VCSession *vc,
                       struct RTPMessage *p,
                       uint32_t full_data_len)
{

    AVPacket *compr_data;
    compr_data = av_packet_alloc();


    uint8_t *tmp_buf = calloc(1, full_data_len + FF_INPUT_BUFFER_PADDING_SIZE);
    memcpy(tmp_buf, p->data, full_data_len);

    compr_data->data = tmp_buf; // p->data;
    compr_data->size = (int)full_data_len; // hmm, "int" again

    avcodec_send_packet(vc->h264_decoder, compr_data);

    int ret_ = 0;
    while (ret_ >= 0) {
        AVFrame *frame = av_frame_alloc();
        ret_ = avcodec_receive_frame(vc->h264_decoder, frame);
        if (ret_ == AVERROR(EAGAIN) || ret_ == AVERROR_EOF) {
            // error
            break;
        } else if (ret_ < 0) {
            // Error during decoding
            break;
        } else if (ret_ == 0) {
            vc->vcb(vc->av, vc->friend_number, frame->width, frame->height,
                          (const uint8_t *)frame->data[0],
                          (const uint8_t *)frame->data[1],
                          (const uint8_t *)frame->data[2],
                          frame->linesize[0], frame->linesize[1],
                          frame->linesize[2], vc->vcb_user_data);
        } else {
            // some other error
        }
        av_frame_free(&frame);
    }
    av_packet_free(&compr_data);
    free(tmp_buf);
    free(p);
}


  


  Send frame and encoding :

  


  bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t *y,
                            const uint8_t *u, const uint8_t *v, Toxav_Err_Send_Frame *error, int16_t kf_max_dist, vpx_codec_er_flags_t error_resilient,
                            unsigned int my_lag_in_frames, uint16_t kf_mode, uint16_t quality_mode)
{
    Toxav_Err_Send_Frame rc = TOXAV_ERR_SEND_FRAME_OK;
    ToxAVCall *call;
    uint64_t video_frame_record_timestamp = current_time_monotonic(av->m->mono_time);

    int vpx_encode_flags = 0;

    pthread_mutex_lock(call->mutex_video);
    pthread_mutex_unlock(av->mutex);

    if (y == nullptr || u == nullptr || v == nullptr) {
        pthread_mutex_unlock(call->mutex_video);
        rc = TOXAV_ERR_SEND_FRAME_NULL;
        goto RETURN;
    }


    if (call->video_rtp->ssrc < VIDEO_SEND_X_KEYFRAMES_FIRST) {
        // Key frame flag for first frames
        vpx_encode_flags = VPX_EFLAG_FORCE_KF;
        LOGGER_INFO(av->m->log, "I_FRAME_FLAG:%d only-i-frame mode", call->video_rtp->ssrc);

        ++call->video_rtp->ssrc;
    } else if (call->video_rtp->ssrc == VIDEO_SEND_X_KEYFRAMES_FIRST) {
        // normal keyframe placement
        vpx_encode_flags = 0;
        LOGGER_INFO(av->m->log, "I_FRAME_FLAG:%d normal mode", call->video_rtp->ssrc);

        ++call->video_rtp->ssrc;
    }


    x264_nal_t *nal = NULL;
    int i_frame_size = 0;

    uint32_t result = encode_frame_h264(av, friend_number, width, height,
                                        y, u, v,
                                        &video_frame_record_timestamp,
                                        vpx_encode_flags,
                                        &nal,
                                        &i_frame_size);
    if (result != 0) {
        pthread_mutex_unlock(call->mutex_video);
        rc = TOXAV_ERR_SEND_FRAME_INVALID;
        goto RETURN;
    }

    ++call->video->frame_counter;

    rc = send_frames_h264(av, friend_number, width, height,
                          y, u, v, call,
                          &video_frame_record_timestamp,
                          vpx_encode_flags,
                          &nal,
                          &i_frame_size,
                          &rc);

    pthread_mutex_unlock(call->mutex_video);

RETURN:

    if (error) {
        *error = rc;
    }

    return rc == TOXAV_ERR_SEND_FRAME_OK;
}

uint32_t send_frames_h264(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height,
                          const uint8_t *y,
                          const uint8_t *u, const uint8_t *v, ToxAVCall *call,
                          uint64_t *video_frame_record_timestamp,
                          int vpx_encode_flags,
                          x264_nal_t **nal,
                          int *i_frame_size,
                          TOXAV_ERR_SEND_FRAME *rc)
{

    if (*i_frame_size > 0) {

        // use the record timestamp that was actually used for this frame
        *video_frame_record_timestamp = (uint64_t)call->video->h264_in_pic.i_pts;
        const uint32_t frame_length_in_bytes = *i_frame_size;
        const int keyframe = (int)call->video->h264_out_pic.b_keyframe;

        LOGGER_DEBUG(av->m->log, "video packet record time: %lu", (*video_frame_record_timestamp));

        int res = rtp_send_data
                  (
                      call->video_rtp,
                      (const uint8_t *)((*nal)->p_payload),
                      frame_length_in_bytes,
                      keyframe,
                      *video_frame_record_timestamp,
                      av->m->log
                  );

        (*video_frame_record_timestamp)++;

        if (res < 0) {
            LOGGER_WARNING(av->m->log, "Could not send video frame: %s", strerror(errno));
            *rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED;
            return 1;
        }

        return 0;
    } else {
        *rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED;
        return 1;
    }

}


  


  I get image like this :