From 2f239cf8988ceb37ab7d2d83b53636f9d13db8e1 Mon Sep 17 00:00:00 2001 From: Hertz Wang Date: Fri, 2 Nov 2018 19:15:55 +0800 Subject: [PATCH 06/11] v4l2: support queue drm buffers Test Passed: ffmpeg -loglevel debug -video_size 1280*720 -pixel_format drm_prime:nv12 -i /dev/video0 -vcodec h264_rkmpp -b 6000000 /userdata/out.mp4 TODO: drm_prime:mjpeg/h264 Change-Id: I92e1db3576162b81a041f3a61e54c213a35ea1f9 Signed-off-by: Hertz Wang --- libavcodec/avcodec.h | 15 ++++ libavcodec/avpacket.c | 16 ++++ libavcodec/rawdec.c | 137 ++++++++++++++++------------ libavcodec/utils.c | 30 +++++-- libavdevice/v4l2.c | 244 ++++++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 362 insertions(+), 80 deletions(-) diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h index bee2234..ffafe79 100644 --- a/libavcodec/avcodec.h +++ b/libavcodec/avcodec.h @@ -1473,6 +1473,15 @@ typedef struct AVPacket { attribute_deprecated int64_t convergence_duration; #endif + /* + * 1. zero copying path for input device, such as camera grab. + * AVPacket may be actually an AVFrame. + * 2. the API of hwaccel always handle AVFrame while contains the hw buffer + * descriptor + * + * This is a reference to a hwaccel avframe. + */ + AVFrame *hw_frame; } AVPacket; #define AV_PKT_FLAG_KEY 0x0001 ///< The packet contains a keyframe #define AV_PKT_FLAG_CORRUPT 0x0002 ///< The packet content is corrupted @@ -3924,6 +3933,12 @@ typedef struct AVCodecParameters { int format; /** + * video only: the pixel format, the value corresponds to enum AVPixelFormat, + * except the hw_accel pixel format. + */ + int sw_format; + + /** * The average bitrate of the encoded data (in bits per second). */ int64_t bit_rate; diff --git a/libavcodec/avpacket.c b/libavcodec/avpacket.c index e160ad3..20fa203 100644 --- a/libavcodec/avpacket.c +++ b/libavcodec/avpacket.c @@ -46,6 +46,7 @@ FF_ENABLE_DEPRECATION_WARNINGS pkt->buf = NULL; pkt->side_data = NULL; pkt->side_data_elems = 0; + pkt->hw_frame = NULL; } AVPacket *av_packet_alloc(void) @@ -599,6 +600,7 @@ void av_packet_unref(AVPacket *pkt) { av_packet_free_side_data(pkt); av_buffer_unref(&pkt->buf); + av_frame_free(&pkt->hw_frame); av_init_packet(pkt); pkt->data = NULL; pkt->size = 0; @@ -626,6 +628,20 @@ int av_packet_ref(AVPacket *dst, const AVPacket *src) ret = AVERROR(ENOMEM); goto fail; } + if (src->hw_frame) { + dst->hw_frame = av_frame_alloc(); + if (!dst->hw_frame) { + av_buffer_unref(&dst->buf); + ret = AVERROR(ENOMEM); + goto fail; + } + ret = av_frame_ref(dst->hw_frame, src->hw_frame); + if (ret < 0) { + av_buffer_unref(&dst->buf); + av_frame_free(&dst->hw_frame); + goto fail; + } + } dst->data = src->data; } diff --git a/libavcodec/rawdec.c b/libavcodec/rawdec.c index 53f5b76..d6f1f81 100644 --- a/libavcodec/rawdec.c +++ b/libavcodec/rawdec.c @@ -73,37 +73,39 @@ static av_cold int raw_init_decoder(AVCodecContext *avctx) ff_bswapdsp_init(&context->bbdsp); - if ( avctx->codec_tag == MKTAG('r','a','w',' ') - || avctx->codec_tag == MKTAG('N','O','1','6')) - avctx->pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_mov, - avctx->bits_per_coded_sample); - else if (avctx->codec_tag == MKTAG('W', 'R', 'A', 'W')) - avctx->pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_avi, - avctx->bits_per_coded_sample); - else if (avctx->codec_tag && (avctx->codec_tag & 0xFFFFFF) != MKTAG('B','I','T', 0)) - avctx->pix_fmt = avpriv_find_pix_fmt(ff_raw_pix_fmt_tags, avctx->codec_tag); - else if (avctx->pix_fmt == AV_PIX_FMT_NONE && avctx->bits_per_coded_sample) - avctx->pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_avi, - avctx->bits_per_coded_sample); - - desc = av_pix_fmt_desc_get(avctx->pix_fmt); - if (!desc) { - av_log(avctx, AV_LOG_ERROR, "Invalid pixel format.\n"); - return AVERROR(EINVAL); - } + if (!(avctx->codec_tag == MKTAG('I', 'G', 'N', 'R'))) { + if ( avctx->codec_tag == MKTAG('r','a','w',' ') + || avctx->codec_tag == MKTAG('N','O','1','6')) + avctx->pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_mov, + avctx->bits_per_coded_sample); + else if (avctx->codec_tag == MKTAG('W', 'R', 'A', 'W')) + avctx->pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_avi, + avctx->bits_per_coded_sample); + else if (avctx->codec_tag && (avctx->codec_tag & 0xFFFFFF) != MKTAG('B','I','T', 0)) + avctx->pix_fmt = avpriv_find_pix_fmt(ff_raw_pix_fmt_tags, avctx->codec_tag); + else if (avctx->pix_fmt == AV_PIX_FMT_NONE && avctx->bits_per_coded_sample) + avctx->pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_avi, + avctx->bits_per_coded_sample); + + desc = av_pix_fmt_desc_get(avctx->pix_fmt); + if (!desc) { + av_log(avctx, AV_LOG_ERROR, "Invalid pixel format.\n"); + return AVERROR(EINVAL); + } - if (desc->flags & (AV_PIX_FMT_FLAG_PAL | FF_PSEUDOPAL)) { - context->palette = av_buffer_alloc(AVPALETTE_SIZE); - if (!context->palette) - return AVERROR(ENOMEM); -#if FF_API_PSEUDOPAL - if (desc->flags & AV_PIX_FMT_FLAG_PSEUDOPAL) - avpriv_set_systematic_pal2((uint32_t*)context->palette->data, avctx->pix_fmt); -#endif - else { - memset(context->palette->data, 0, AVPALETTE_SIZE); - if (avctx->bits_per_coded_sample == 1) - memset(context->palette->data, 0xff, 4); + if (desc->flags & (AV_PIX_FMT_FLAG_PAL | FF_PSEUDOPAL)) { + context->palette = av_buffer_alloc(AVPALETTE_SIZE); + if (!context->palette) + return AVERROR(ENOMEM); + #if FF_API_PSEUDOPAL + if (desc->flags & AV_PIX_FMT_FLAG_PSEUDOPAL) + avpriv_set_systematic_pal2((uint32_t*)context->palette->data, avctx->pix_fmt); + #endif + else { + memset(context->palette->data, 0, AVPALETTE_SIZE); + if (avctx->bits_per_coded_sample == 1) + memset(context->palette->data, 0xff, 4); + } } } @@ -179,7 +181,7 @@ static int raw_decode(AVCodecContext *avctx, void *data, int *got_frame, int res, len; int need_copy; - AVFrame *frame = data; + AVFrame *frame = avpkt->hw_frame? avpkt->hw_frame : data; if (avctx->width <= 0) { av_log(avctx, AV_LOG_ERROR, "width is not set\n"); @@ -204,33 +206,35 @@ static int raw_decode(AVCodecContext *avctx, void *data, int *got_frame, return AVERROR_INVALIDDATA; } - desc = av_pix_fmt_desc_get(avctx->pix_fmt); - - if ((avctx->bits_per_coded_sample == 8 || avctx->bits_per_coded_sample == 4 || - avctx->bits_per_coded_sample == 2 || avctx->bits_per_coded_sample == 1 || - (avctx->bits_per_coded_sample == 0 && (context->is_nut_pal8 || context->is_mono)) ) && - (context->is_mono || context->is_pal8) && - (!avctx->codec_tag || avctx->codec_tag == MKTAG('r','a','w',' ') || - context->is_nut_mono || context->is_nut_pal8)) { - context->is_1_2_4_8_bpp = 1; - if (context->is_mono) { - int row_bytes = avctx->width / 8 + (avctx->width & 7 ? 1 : 0); - context->frame_size = av_image_get_buffer_size(avctx->pix_fmt, - FFALIGN(row_bytes, 16) * 8, - avctx->height, 1); - } else - context->frame_size = av_image_get_buffer_size(avctx->pix_fmt, - FFALIGN(avctx->width, 16), - avctx->height, 1); - } else { - context->is_lt_16bpp = av_get_bits_per_pixel(desc) == 16 && avctx->bits_per_coded_sample && avctx->bits_per_coded_sample < 16; - context->frame_size = av_image_get_buffer_size(avctx->pix_fmt, avctx->width, - avctx->height, 1); - } - if (context->frame_size < 0) - return context->frame_size; + if (!avpkt->hw_frame) { + desc = av_pix_fmt_desc_get(avctx->pix_fmt); + + if ((avctx->bits_per_coded_sample == 8 || avctx->bits_per_coded_sample == 4 || + avctx->bits_per_coded_sample == 2 || avctx->bits_per_coded_sample == 1 || + (avctx->bits_per_coded_sample == 0 && (context->is_nut_pal8 || context->is_mono)) ) && + (context->is_mono || context->is_pal8) && + (!avctx->codec_tag || avctx->codec_tag == MKTAG('r','a','w',' ') || + context->is_nut_mono || context->is_nut_pal8)) { + context->is_1_2_4_8_bpp = 1; + if (context->is_mono) { + int row_bytes = avctx->width / 8 + (avctx->width & 7 ? 1 : 0); + context->frame_size = av_image_get_buffer_size(avctx->pix_fmt, + FFALIGN(row_bytes, 16) * 8, + avctx->height, 1); + } else + context->frame_size = av_image_get_buffer_size(avctx->pix_fmt, + FFALIGN(avctx->width, 16), + avctx->height, 1); + } else { + context->is_lt_16bpp = av_get_bits_per_pixel(desc) == 16 && avctx->bits_per_coded_sample && avctx->bits_per_coded_sample < 16; + context->frame_size = av_image_get_buffer_size(avctx->pix_fmt, avctx->width, + avctx->height, 1); + } + if (context->frame_size < 0) + return context->frame_size; - need_copy = !avpkt->buf || context->is_1_2_4_8_bpp || context->is_yuv2 || context->is_lt_16bpp; + need_copy = !avpkt->buf || context->is_1_2_4_8_bpp || context->is_yuv2 || context->is_lt_16bpp; + } frame->pict_type = AV_PICTURE_TYPE_I; frame->key_frame = 1; @@ -250,6 +254,24 @@ static int raw_decode(AVCodecContext *avctx, void *data, int *got_frame, if ((res = av_image_check_size(avctx->width, avctx->height, 0, avctx)) < 0) return res; + if (avpkt->hw_frame) { + AVFrame *out_frame = data; + + res = av_frame_ref(out_frame, avpkt->hw_frame); + if (res) + return res; + + out_frame->buf[1] = av_buffer_ref(avpkt->buf); + if (!out_frame->buf[1]) { + av_frame_unref(out_frame); + return AVERROR(ENOMEM); + } + if (context->flip) + flip(avctx, out_frame); + frame = out_frame; + goto out; + } + if (need_copy) frame->buf[0] = av_buffer_alloc(FFMAX(context->frame_size, buf_size)); else @@ -474,6 +496,7 @@ static int raw_decode(AVCodecContext *avctx, void *data, int *got_frame, } } +out: if (avctx->field_order > AV_FIELD_PROGRESSIVE) { /* we have interlaced material flagged in container */ frame->interlaced_frame = 1; if (avctx->field_order == AV_FIELD_TT || avctx->field_order == AV_FIELD_TB) diff --git a/libavcodec/utils.c b/libavcodec/utils.c index 1661d48..9ead54b 100644 --- a/libavcodec/utils.c +++ b/libavcodec/utils.c @@ -1204,22 +1204,33 @@ void avcodec_string(char *buf, int buf_size, AVCodecContext *enc, int encode) ", %d reference frame%s", enc->refs, enc->refs > 1 ? "s" : ""); - if (enc->codec_tag) + if (enc->codec_tag) { + unsigned int tag = enc->codec_tag; + if (enc->codec_tag == MKTAG('I', 'G', 'N', 'R') && + enc->sw_pix_fmt != AV_PIX_FMT_NONE) + tag = avcodec_pix_fmt_to_codec_tag(enc->sw_pix_fmt); snprintf(buf + strlen(buf), buf_size - strlen(buf), " (%s / 0x%04X)", - av_fourcc2str(enc->codec_tag), enc->codec_tag); + av_fourcc2str(tag), tag); + } switch (enc->codec_type) { case AVMEDIA_TYPE_VIDEO: { char detail[256] = "("; + enum AVPixelFormat pix_fmt = enc->pix_fmt; av_strlcat(buf, separator, buf_size); snprintf(buf + strlen(buf), buf_size - strlen(buf), - "%s", enc->pix_fmt == AV_PIX_FMT_NONE ? "none" : - av_get_pix_fmt_name(enc->pix_fmt)); - if (enc->bits_per_raw_sample && enc->pix_fmt != AV_PIX_FMT_NONE && - enc->bits_per_raw_sample < av_pix_fmt_desc_get(enc->pix_fmt)->comp[0].depth) + "%s", pix_fmt == AV_PIX_FMT_NONE ? "none" : + av_get_pix_fmt_name(pix_fmt)); + if (enc->sw_pix_fmt != AV_PIX_FMT_NONE) { + pix_fmt = enc->sw_pix_fmt; + snprintf(buf + strlen(buf), buf_size - strlen(buf), + "(%s) ", av_get_pix_fmt_name(pix_fmt)); + } + if (enc->bits_per_raw_sample && pix_fmt != AV_PIX_FMT_NONE && + enc->bits_per_raw_sample < av_pix_fmt_desc_get(pix_fmt)->comp[0].depth) av_strlcatf(detail, sizeof(detail), "%d bpc, ", enc->bits_per_raw_sample); if (enc->color_range != AVCOL_RANGE_UNSPECIFIED) av_strlcatf(detail, sizeof(detail), "%s, ", @@ -1978,6 +1989,7 @@ static void codec_parameters_reset(AVCodecParameters *par) par->codec_type = AVMEDIA_TYPE_UNKNOWN; par->codec_id = AV_CODEC_ID_NONE; par->format = -1; + par->sw_format = AV_PIX_FMT_NONE; par->field_order = AV_FIELD_UNKNOWN; par->color_range = AVCOL_RANGE_UNSPECIFIED; par->color_primaries = AVCOL_PRI_UNSPECIFIED; @@ -2046,6 +2058,7 @@ int avcodec_parameters_from_context(AVCodecParameters *par, switch (par->codec_type) { case AVMEDIA_TYPE_VIDEO: par->format = codec->pix_fmt; + par->sw_format = codec->sw_pix_fmt; par->width = codec->width; par->height = codec->height; par->field_order = codec->field_order; @@ -2101,6 +2114,7 @@ int avcodec_parameters_to_context(AVCodecContext *codec, switch (par->codec_type) { case AVMEDIA_TYPE_VIDEO: codec->pix_fmt = par->format; + codec->sw_pix_fmt = par->sw_format; codec->width = par->width; codec->height = par->height; codec->field_order = par->field_order; @@ -2197,7 +2211,9 @@ int64_t ff_guess_coded_bitrate(AVCodecContext *avctx) return 0; if (!bits_per_coded_sample) { - const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->pix_fmt); + enum AVPixelFormat pix_fmt = avctx->sw_pix_fmt == AV_PIX_FMT_NONE ? + avctx->pix_fmt : avctx->sw_pix_fmt; + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt); bits_per_coded_sample = av_get_bits_per_pixel(desc); } bitrate = (int64_t)bits_per_coded_sample * avctx->width * avctx->height * diff --git a/libavdevice/v4l2.c b/libavdevice/v4l2.c index 10a0ff0..d42abf5 100644 --- a/libavdevice/v4l2.c +++ b/libavdevice/v4l2.c @@ -1,6 +1,7 @@ /* * Copyright (c) 2000,2001 Fabrice Bellard * Copyright (c) 2006 Luca Abeni + * Copyright (c) 2018 Hertz Wang * * This file is part of FFmpeg. * @@ -39,7 +40,11 @@ #include #endif +#include "libavutil/hwcontext.h" +#include "libavutil/hwcontext_drm.h" + static const int desired_video_buffers = 256; +static const int desired_drm_buffers = 4; #define V4L_ALLFORMATS 3 #define V4L_RAWFORMATS 1 @@ -79,6 +84,8 @@ struct video_data { TimeFilter *timefilter; int64_t last_time_m; + enum v4l2_memory memory_type; + int buffers; atomic_int buffers_queued; void **buf_start; @@ -91,6 +98,12 @@ struct video_data { int list_standard; /**< Set by a private option. */ char *framerate; /**< Set by a private option. */ + enum AVPixelFormat av_format; + AVBufferRef *hw_frames_ref; + AVFrame **pixel_frames; + int (*stream_init)(AVFormatContext *ctx); + void (*stream_close)(struct video_data *s); + int use_libv4l2; int (*open_f)(const char *file, int oflag, ...); int (*close_f)(int fd); @@ -106,6 +119,9 @@ struct buff_data { int index; }; +static int mmap_init(AVFormatContext *ctx); +static void mmap_close(struct video_data *s); + static int device_open(AVFormatContext *ctx, const char* device_path) { struct video_data *s = ctx->priv_data; @@ -135,6 +151,11 @@ static int device_open(AVFormatContext *ctx, const char* device_path) SET_WRAPPERS(); } + // default MMAP + s->memory_type = V4L2_MEMORY_MMAP; + s->stream_init = mmap_init; + s->stream_close = mmap_close; + #define v4l2_open s->open_f #define v4l2_close s->close_f #define v4l2_dup s->dup_f @@ -396,6 +417,111 @@ static int mmap_init(AVFormatContext *ctx) return 0; } +static void free_pixel_frames(struct video_data *s) { + int i; + + for (i = 0; i < s->buffers; i++) { + if (s->pixel_frames[i]) + av_frame_free(&s->pixel_frames[i]); + } + av_freep(&s->pixel_frames); +} + +static int drm_init(AVFormatContext *ctx) +{ + int i, res; + struct video_data *s = ctx->priv_data; + AVHWFramesContext *hwframes = (AVHWFramesContext*)s->hw_frames_ref->data; + + struct v4l2_requestbuffers req = { + .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, + .count = desired_drm_buffers, + .memory = V4L2_MEMORY_DMABUF + }; + + if (v4l2_ioctl(s->fd, VIDIOC_REQBUFS, &req) < 0) { + res = AVERROR(errno); + av_log(ctx, AV_LOG_ERROR, "ioctl(VIDIOC_REQBUFS): %s\n", av_err2str(res)); + return res; + } + + s->memory_type = V4L2_MEMORY_DMABUF; + + hwframes->format = AV_PIX_FMT_DRM_PRIME; + if (s->av_format != AV_PIX_FMT_NONE) + hwframes->sw_format = s->av_format; + else + hwframes->sw_format = AV_PIX_FMT_BGR0; // jpeg or h264 + hwframes->width = s->width; + hwframes->height = s->height; + hwframes->initial_pool_size = req.count; + res = av_hwframe_ctx_init(s->hw_frames_ref); + if (res < 0) + return res; + + if (hwframes->initial_pool_size < 2) { + av_log(ctx, AV_LOG_ERROR, "Insufficient buffer memory\n"); + return AVERROR(ENOMEM); + } + + s->buffers = hwframes->initial_pool_size; + s->pixel_frames = av_mallocz_array(s->buffers, sizeof(AVFrame *)); + if (!s->pixel_frames) { + av_log(ctx, AV_LOG_ERROR, "Cannot allocate pixel frame pointers\n"); + return AVERROR(ENOMEM); + } + s->buf_start = av_malloc_array(s->buffers, sizeof(void *)); + if (!s->buf_start) { + av_log(ctx, AV_LOG_ERROR, "Cannot allocate buffer pointers\n"); + res = AVERROR(ENOMEM); + goto fail; + } + s->buf_len = av_malloc_array(s->buffers, sizeof(unsigned int)); + if (!s->buf_len) { + av_log(ctx, AV_LOG_ERROR, "Cannot allocate buffer sizes\n"); + res = AVERROR(ENOMEM); + goto fail; + } + + for (i = 0; i < s->buffers; i++) { + AVDRMFrameDescriptor *desc; + AVDRMObjectDescriptor *object; + + s->pixel_frames[i] = av_frame_alloc(); + if (!s->pixel_frames[i]) { + res = AVERROR(ENOMEM); + goto fail; + } + res = av_hwframe_get_buffer(s->hw_frames_ref, s->pixel_frames[i], 0); + if (res < 0) + goto fail; + + desc = (AVDRMFrameDescriptor*)s->pixel_frames[i]->data[0]; + object = &desc->objects[0]; + // pool always provide only one object, and mmapped userspace viraddr + av_assert0(desc->nb_objects == 1 && desc->objects[0].ptr); + + s->buf_start[i] = object->ptr; + s->buf_len[i] = object->size; + if (s->frame_size > 0 && s->buf_len[i] < s->frame_size) { + av_log(ctx, AV_LOG_ERROR, + "buf_len[%d] = %d < expected frame size %d\n", + i, s->buf_len[i], s->frame_size); + return AVERROR(ENOMEM); + } + } + + return 0; + +fail: + free_pixel_frames(s); + if (s->buf_start) + av_freep(&s->buf_start); + if (s->buf_len) + av_freep(&s->buf_len); + return res; +} + static int enqueue_buffer(struct video_data *s, struct v4l2_buffer *buf) { int res = 0; @@ -410,16 +536,22 @@ static int enqueue_buffer(struct video_data *s, struct v4l2_buffer *buf) return res; } -static void mmap_release_buffer(void *opaque, uint8_t *data) +static void stream_release_buffer(void *opaque, uint8_t *data) { struct v4l2_buffer buf = { 0 }; struct buff_data *buf_descriptor = opaque; struct video_data *s = buf_descriptor->s; buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - buf.memory = V4L2_MEMORY_MMAP; + buf.memory = s->memory_type; buf.index = buf_descriptor->index; av_free(buf_descriptor); + if (s->hw_frames_ref) { + // V4L2_MEMORY_DMABUF + AVFrame *frame = s->pixel_frames[buf.index]; + buf.m.fd = ((AVDRMFrameDescriptor*)frame->data[0])->objects[0].fd; + buf.length = s->buf_len[buf.index]; + } enqueue_buffer(s, &buf); } @@ -485,12 +617,12 @@ static int convert_timestamp(AVFormatContext *ctx, int64_t *ts) return 0; } -static int mmap_read_frame(AVFormatContext *ctx, AVPacket *pkt) +static int stream_read_frame(AVFormatContext *ctx, AVPacket *pkt) { struct video_data *s = ctx->priv_data; struct v4l2_buffer buf = { .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, - .memory = V4L2_MEMORY_MMAP + .memory = s->memory_type }; struct timeval buf_ts; int res; @@ -543,7 +675,8 @@ static int mmap_read_frame(AVFormatContext *ctx, AVPacket *pkt) } /* Image is at s->buff_start[buf.index] */ - if (atomic_load(&s->buffers_queued) == FFMAX(s->buffers / 8, 1)) { + if (s->memory_type == V4L2_MEMORY_MMAP && + atomic_load(&s->buffers_queued) == FFMAX(s->buffers / 8, 1)) { /* when we start getting low on queued buffers, fall back on copying data */ res = av_new_packet(pkt, buf.bytesused); if (res < 0) { @@ -577,11 +710,27 @@ static int mmap_read_frame(AVFormatContext *ctx, AVPacket *pkt) buf_descriptor->index = buf.index; buf_descriptor->s = s; - pkt->buf = av_buffer_create(pkt->data, pkt->size, mmap_release_buffer, + if (s->hw_frames_ref) { + if (pkt->hw_frame) { + av_log(ctx, AV_LOG_WARNING, "Input packet is not a empty packet!\n"); + av_frame_free(&pkt->hw_frame); + } + pkt->hw_frame = av_frame_alloc(); + if (!pkt->hw_frame || + av_frame_ref(pkt->hw_frame, s->pixel_frames[buf.index])) { + enqueue_buffer(s, &buf); + av_frame_free(&pkt->hw_frame); + av_freep(&buf_descriptor); + return AVERROR(ENOMEM); + } + } + + pkt->buf = av_buffer_create(pkt->data, pkt->size, stream_release_buffer, buf_descriptor, 0); if (!pkt->buf) { av_log(ctx, AV_LOG_ERROR, "Failed to create a buffer\n"); enqueue_buffer(s, &buf); + av_frame_free(&pkt->hw_frame); av_freep(&buf_descriptor); return AVERROR(ENOMEM); } @@ -592,7 +741,7 @@ static int mmap_read_frame(AVFormatContext *ctx, AVPacket *pkt) return pkt->size; } -static int mmap_start(AVFormatContext *ctx) +static int stream_start(AVFormatContext *ctx) { struct video_data *s = ctx->priv_data; enum v4l2_buf_type type; @@ -602,9 +751,16 @@ static int mmap_start(AVFormatContext *ctx) struct v4l2_buffer buf = { .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, .index = i, - .memory = V4L2_MEMORY_MMAP + .memory = s->memory_type }; + if (s->hw_frames_ref) { + // V4L2_MEMORY_DMABUF + AVFrame *frame = s->pixel_frames[i]; + buf.m.fd = ((AVDRMFrameDescriptor*)frame->data[0])->objects[0].fd; + buf.length = s->buf_len[i]; + } + if (v4l2_ioctl(s->fd, VIDIOC_QBUF, &buf) < 0) { res = AVERROR(errno); av_log(ctx, AV_LOG_ERROR, "ioctl(VIDIOC_QBUF): %s\n", @@ -642,6 +798,19 @@ static void mmap_close(struct video_data *s) av_freep(&s->buf_len); } +static void drm_close(struct video_data *s) +{ + enum v4l2_buf_type type; + + type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + if (-1 == v4l2_ioctl(s->fd, VIDIOC_STREAMOFF, &type)) + av_log(NULL, AV_LOG_FATAL, "V4L2 Could not steam off (errno = %d).\n", + errno); + free_pixel_frames(s); + av_freep(&s->buf_start); + av_freep(&s->buf_len); +} + static int v4l2_set_parameters(AVFormatContext *ctx) { struct video_data *s = ctx->priv_data; @@ -887,8 +1056,37 @@ static int v4l2_read_header(AVFormatContext *ctx) avpriv_set_pts_info(st, 64, 1, 1000000); /* 64 bits pts in us */ if (s->pixel_format) { - const AVCodecDescriptor *desc = avcodec_descriptor_get_by_name(s->pixel_format); + const AVCodecDescriptor *desc; + const char *ff_pix_fmt = NULL; + + // drm_prime:ff_pix_fmt + if (av_strstart(s->pixel_format, "drm_prime:", &ff_pix_fmt)) { + // if set drm_prime, alloc drm buffer externally and queue to v4l2 + AVBufferRef *hw_device_ref; + // libv4l2 has implicit format conversion, + // conflict to our intention of using drm external buffer + if (s->use_libv4l2) { + av_log(ctx, AV_LOG_WARNING, + "use_libv4l2 conflict to pixel format drm_prime\n"); + res = AVERROR_EXIT; + goto fail; + } + res = av_hwdevice_ctx_create(&hw_device_ref, AV_HWDEVICE_TYPE_DRM, + NULL, NULL, 0); + if (res < 0) + goto fail; + s->hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ref); + av_buffer_unref(&hw_device_ref); + if (!s->hw_frames_ref) { + res = AVERROR(ENOMEM); + goto fail; + } + memmove(s->pixel_format, ff_pix_fmt, strlen(ff_pix_fmt) + 1); + s->stream_init = drm_init; + s->stream_close = drm_close; + } + desc = avcodec_descriptor_get_by_name(s->pixel_format); if (desc) ctx->video_codec_id = desc->id; @@ -940,29 +1138,39 @@ static int v4l2_read_header(AVFormatContext *ctx) if ((res = v4l2_set_parameters(ctx)) < 0) goto fail; + s->av_format = st->codecpar->format = ff_fmt_v4l2ff(desired_format, codec_id); if (st->codecpar->format != AV_PIX_FMT_NONE) s->frame_size = av_image_get_buffer_size(st->codecpar->format, s->width, s->height, 1); - if ((res = mmap_init(ctx)) || - (res = mmap_start(ctx)) < 0) + if ((res = s->stream_init(ctx)) || + (res = stream_start(ctx)) < 0) goto fail; s->top_field_first = first_field(s); st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; st->codecpar->codec_id = codec_id; - if (codec_id == AV_CODEC_ID_RAWVIDEO) + if (codec_id == AV_CODEC_ID_RAWVIDEO) { st->codecpar->codec_tag = avcodec_pix_fmt_to_codec_tag(st->codecpar->format); - else if (codec_id == AV_CODEC_ID_H264) { + } else if (codec_id == AV_CODEC_ID_H264) { st->need_parsing = AVSTREAM_PARSE_FULL_ONCE; } if (desired_format == V4L2_PIX_FMT_YVU420) st->codecpar->codec_tag = MKTAG('Y', 'V', '1', '2'); else if (desired_format == V4L2_PIX_FMT_YVU410) st->codecpar->codec_tag = MKTAG('Y', 'V', 'U', '9'); + + if (s->hw_frames_ref) { + // tag: ignore + st->codecpar->codec_tag = MKTAG('I', 'G', 'N', 'R'); + st->codecpar->sw_format = + st->internal->avctx->sw_pix_fmt = st->codecpar->format; + st->codecpar->format = AV_PIX_FMT_DRM_PRIME; + } + st->codecpar->width = s->width; st->codecpar->height = s->height; if (st->avg_frame_rate.den) @@ -971,6 +1179,8 @@ static int v4l2_read_header(AVFormatContext *ctx) return 0; fail: + if (s->hw_frames_ref) + av_buffer_unref(&s->hw_frames_ref); v4l2_close(s->fd); return res; } @@ -985,9 +1195,8 @@ FF_ENABLE_DEPRECATION_WARNINGS #endif int res; - if ((res = mmap_read_frame(ctx, pkt)) < 0) { + if ((res = stream_read_frame(ctx, pkt)) < 0) return res; - } #if FF_API_CODED_FRAME && FF_API_LAVF_AVCTX FF_DISABLE_DEPRECATION_WARNINGS @@ -1009,7 +1218,10 @@ static int v4l2_read_close(AVFormatContext *ctx) av_log(ctx, AV_LOG_WARNING, "Some buffers are still owned by the caller on " "close.\n"); - mmap_close(s); + s->stream_close(s); + + if (s->hw_frames_ref) + av_buffer_unref(&s->hw_frames_ref); v4l2_close(s->fd); return 0; -- 2.7.4