From ca86cad12b37bffb01ed425fc1796f6e4838ce38 Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Fri, 12 Nov 2021 11:14:37 +0800 Subject: [PATCH 25/33] waylandsink: Support NV12_10LE40 and NV12|NV12_10LE40|NV16 (AFBC) Tested on RK356x with: export GST_MPP_VIDEODEC_DEFAULT_ARM_AFBC=1 gst-play-1.0 video.mp4 --videosink=waylandsink Signed-off-by: Jeffy Chen --- ext/wayland/gstwaylandsink.c | 98 +++++++++++++++++++++++++++++++++++- ext/wayland/wldisplay.c | 19 ++++++- ext/wayland/wldisplay.h | 2 + ext/wayland/wllinuxdmabuf.c | 26 +++++++++- ext/wayland/wlvideoformat.c | 1 + ext/wayland/wlvideoformat.h | 42 ++++++++++++++++ ext/wayland/wlwindow.c | 25 +++++---- ext/wayland/wlwindow.h | 2 + 8 files changed, 203 insertions(+), 12 deletions(-) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index 07bfec0..fe7e0e3 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -78,7 +78,7 @@ GST_DEBUG_CATEGORY (gstwayland_debug); #define WL_VIDEO_FORMATS \ "{ BGRx, BGRA, RGBx, xBGR, xRGB, RGBA, ABGR, ARGB, RGB, BGR, " \ "RGB16, BGR16, YUY2, YVYU, UYVY, AYUV, NV12, NV21, NV16, NV61, " \ - "YUV9, YVU9, Y41B, I420, YV12, Y42B, v308 }" + "YUV9, YVU9, Y41B, I420, YV12, Y42B, v308, NV12_10LE40 }" static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, @@ -603,6 +603,53 @@ gst_wayland_sink_set_context (GstElement * element, GstContext * context) GST_ELEMENT_CLASS (parent_class)->set_context (element, context); } +static GstCaps * +gst_wayland_sink_fixup_caps (GstWaylandSink * sink, GstCaps * caps) +{ + GstCaps *tmp_caps = NULL; + + /* HACK: Allow nv12-10le40 and arm-afbc in main caps */ + + if (sink->display->support_nv12_10le40) { + tmp_caps = gst_caps_from_string ( + GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_DMABUF, + "NV12_10LE40")); + + /* NV15(AFBC) */ + if (sink->display->support_afbc) { + gst_caps_ref (tmp_caps); + gst_caps_append (caps, tmp_caps); + + gst_caps_set_simple (tmp_caps, "arm-afbc", G_TYPE_INT, 1, NULL); + } + + gst_caps_append (caps, tmp_caps); + } + + /* NV12|NV16 (AFBC) */ + if (sink->display->support_afbc) { + if (gst_wl_display_check_format_for_dmabuf (sink->display, + GST_VIDEO_FORMAT_NV12)) { + tmp_caps = gst_caps_from_string ( + GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_DMABUF, + "NV12")); + gst_caps_set_simple (tmp_caps, "arm-afbc", G_TYPE_INT, 1, NULL); + gst_caps_append (caps, tmp_caps); + } + + if (gst_wl_display_check_format_for_dmabuf (sink->display, + GST_VIDEO_FORMAT_NV16)) { + tmp_caps = gst_caps_from_string ( + GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_DMABUF, + "NV16")); + gst_caps_set_simple (tmp_caps, "arm-afbc", G_TYPE_INT, 1, NULL); + gst_caps_append (caps, tmp_caps); + } + } + + return caps; +} + static GstCaps * gst_wayland_sink_get_caps (GstBaseSink * bsink, GstCaps * filter) { @@ -657,6 +704,8 @@ gst_wayland_sink_get_caps (GstBaseSink * bsink, GstCaps * filter) gst_structure_take_value (gst_caps_get_structure (caps, 1), "format", &dmabuf_list); + caps = gst_wayland_sink_fixup_caps (sink, caps); + GST_DEBUG_OBJECT (sink, "display caps: %" GST_PTR_FORMAT, caps); } @@ -704,6 +753,8 @@ gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) GstWaylandSink *sink; gboolean use_dmabuf; GstVideoFormat format; + GstStructure *s; + gint value; sink = GST_WAYLAND_SINK (bsink); @@ -713,6 +764,15 @@ gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) if (!gst_video_info_from_caps (&sink->video_info, caps)) goto invalid_format; + /* parse AFBC from caps */ + s = gst_caps_get_structure (caps, 0); + if (gst_structure_get_int (s, "arm-afbc", &value)) { + if (value) + GST_VIDEO_INFO_SET_AFBC (&sink->video_info); + else + GST_VIDEO_INFO_UNSET_AFBC (&sink->video_info); + } + format = GST_VIDEO_INFO_FORMAT (&sink->video_info); sink->video_info_changed = TRUE; @@ -758,9 +818,17 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) GstBufferPool *pool = NULL; gboolean need_pool; GstAllocator *alloc; + GstStructure *s; + gint value; gst_query_parse_allocation (query, &caps, &need_pool); + s = gst_caps_get_structure (caps, 0); + if (gst_structure_get_int (s, "arm-afbc", &value) && value) { + GST_DEBUG_OBJECT (sink, "no allocation for AFBC"); + return FALSE; + } + if (need_pool) pool = gst_wayland_create_pool (sink, caps); @@ -771,6 +839,7 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) alloc = gst_wl_shm_allocator_get (); gst_query_add_allocation_param (query, alloc, NULL); gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + gst_query_add_allocation_meta (query, GST_VIDEO_CROP_META_API_TYPE, NULL); g_object_unref (alloc); return TRUE; @@ -845,6 +914,7 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) GstWaylandSink *sink = GST_WAYLAND_SINK (vsink); GstBuffer *to_render; GstWlBuffer *wlbuffer; + GstVideoCropMeta *crop; GstVideoMeta *vmeta; GstVideoFormat format; GstVideoInfo old_vinfo; @@ -882,6 +952,23 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) } } + crop = gst_buffer_get_video_crop_meta (buffer); + if (crop) { + GstWlWindow *window = sink->window; + + if (window->crop_x != crop->x || window->crop_y != crop->y || + window->crop_w != crop->width || window->crop_h != crop->height) { + window->crop_x = crop->x; + window->crop_y = crop->y; + window->crop_w = crop->width; + window->crop_h = crop->height; + window->crop_dirty = TRUE; + + GST_LOG_OBJECT (sink, + "crop %dx%d-%dx%d", crop->x, crop->y, crop->width, crop->height); + } + } + /* drop buffers until we get a frame callback */ if (sink->redraw_pending) { GST_LOG_OBJECT (sink, "buffer %p dropped (redraw pending)", buffer); @@ -932,6 +1019,9 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) &sink->video_info); } + if (!wbuf && GST_VIDEO_INFO_IS_AFBC (&sink->video_info)) + goto no_afbc; + if (!wbuf && gst_wl_display_check_format_for_shm (sink->display, format)) { if (gst_buffer_n_memory (buffer) == 1 && gst_is_fd_memory (mem)) wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display, @@ -1060,6 +1150,12 @@ no_wl_buffer: ret = GST_FLOW_ERROR; goto done; } +no_afbc: + { + GST_ERROR_OBJECT (sink, "could not import AFBC"); + ret = GST_FLOW_ERROR; + goto done; + } activate_failed: { GST_ERROR_OBJECT (sink, "failed to activate bufferpool."); diff --git a/ext/wayland/wldisplay.c b/ext/wayland/wldisplay.c index e582506..fcf2853 100644 --- a/ext/wayland/wldisplay.c +++ b/ext/wayland/wldisplay.c @@ -145,10 +145,27 @@ dmabuf_format (void *data, struct zwp_linux_dmabuf_v1 *zwp_linux_dmabuf, if (gst_wl_dmabuf_format_to_video_format (format) != GST_VIDEO_FORMAT_UNKNOWN) g_array_append_val (self->dmabuf_formats, format); + + if (format == DRM_FORMAT_NV15) + self->support_nv12_10le40 = TRUE; +} + +static void +dmabuf_modifier (void *data, struct zwp_linux_dmabuf_v1 *zwp_linux_dmabuf, + uint32_t format, uint32_t modifier_hi, uint32_t modifier_lo) +{ + GstWlDisplay *self = data; + uint64_t modifier = ((uint64_t) modifier_hi << 32) | modifier_lo; + + if (modifier == DRM_AFBC_MODIFIER) + self->support_afbc = TRUE; + + dmabuf_format (data, zwp_linux_dmabuf, format); } static const struct zwp_linux_dmabuf_v1_listener dmabuf_listener = { dmabuf_format, + dmabuf_modifier, }; gboolean @@ -235,7 +252,7 @@ registry_handle_global (void *data, struct wl_registry *registry, wl_registry_bind (registry, id, &wp_viewporter_interface, 1); } else if (g_strcmp0 (interface, "zwp_linux_dmabuf_v1") == 0) { self->dmabuf = - wl_registry_bind (registry, id, &zwp_linux_dmabuf_v1_interface, 2); + wl_registry_bind (registry, id, &zwp_linux_dmabuf_v1_interface, 3); zwp_linux_dmabuf_v1_add_listener (self->dmabuf, &dmabuf_listener, self); } } diff --git a/ext/wayland/wldisplay.h b/ext/wayland/wldisplay.h index f2025a6..29c15f6 100644 --- a/ext/wayland/wldisplay.h +++ b/ext/wayland/wldisplay.h @@ -62,6 +62,8 @@ struct _GstWlDisplay struct zwp_linux_dmabuf_v1 *dmabuf; GArray *shm_formats; GArray *dmabuf_formats; + gboolean support_afbc; + gboolean support_nv12_10le40; /* private */ gboolean own_display; diff --git a/ext/wayland/wllinuxdmabuf.c b/ext/wayland/wllinuxdmabuf.c index bc1742c..de3660a 100644 --- a/ext/wayland/wllinuxdmabuf.c +++ b/ext/wayland/wllinuxdmabuf.c @@ -44,8 +44,10 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, int format; guint i, width, height; guint nplanes, flags = 0; + gfloat stride_scale = 1.0f; struct zwp_linux_buffer_params_v1 *params; ConstructBufferData data; + guint64 modifier = GST_VIDEO_INFO_IS_AFBC (info) ? DRM_AFBC_MODIFIER : 0; g_return_val_if_fail (gst_wl_display_check_format_for_dmabuf (display, GST_VIDEO_INFO_FORMAT (info)), NULL); @@ -61,6 +63,27 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, G_GSSIZE_FORMAT " (%d x %d), format %s", info->size, width, height, gst_wl_dmabuf_format_to_string (format)); + if (GST_VIDEO_INFO_IS_AFBC (info)) { + /* Mali uses these formats instead */ + if (format == DRM_FORMAT_NV12) { + format = DRM_FORMAT_YUV420_8BIT; + nplanes = 1; + stride_scale = 1.5; + } else if (format == DRM_FORMAT_NV15) { + format = DRM_FORMAT_YUV420_10BIT; + nplanes = 1; + stride_scale = 1.5; + } else if (format == DRM_FORMAT_NV16) { + format = DRM_FORMAT_YUYV; + nplanes = 1; + stride_scale = 2; + } else { + GST_ERROR_OBJECT (mem->allocator, "unsupported format for AFBC"); + data.wbuf = NULL; + goto out; + } + } + /* Creation and configuration of planes */ params = zwp_linux_dmabuf_v1_create_params (display->dmabuf); @@ -70,11 +93,12 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, offset = GST_VIDEO_INFO_PLANE_OFFSET (info, i); stride = GST_VIDEO_INFO_PLANE_STRIDE (info, i); + stride *= stride_scale; if (gst_buffer_find_memory (buf, offset, 1, &mem_idx, &length, &skip)) { GstMemory *m = gst_buffer_peek_memory (buf, mem_idx); gint fd = gst_dmabuf_memory_get_fd (m); zwp_linux_buffer_params_v1_add (params, fd, i, m->offset + skip, - stride, 0, 0); + stride, modifier >> 32, modifier & 0xFFFFFFFF); } else { GST_ERROR_OBJECT (mem->allocator, "memory does not seem to contain " "enough data for the specified format"); diff --git a/ext/wayland/wlvideoformat.c b/ext/wayland/wlvideoformat.c index 68cec50..11ca051 100644 --- a/ext/wayland/wlvideoformat.c +++ b/ext/wayland/wlvideoformat.c @@ -56,6 +56,7 @@ static const wl_VideoFormat wl_formats[] = { {WL_SHM_FORMAT_UYVY, DRM_FORMAT_UYVY, GST_VIDEO_FORMAT_UYVY}, {WL_SHM_FORMAT_AYUV, DRM_FORMAT_AYUV, GST_VIDEO_FORMAT_AYUV}, {WL_SHM_FORMAT_NV12, DRM_FORMAT_NV12, GST_VIDEO_FORMAT_NV12}, + {-1, DRM_FORMAT_NV15, GST_VIDEO_FORMAT_NV12_10LE40}, {WL_SHM_FORMAT_NV21, DRM_FORMAT_NV21, GST_VIDEO_FORMAT_NV21}, {WL_SHM_FORMAT_NV16, DRM_FORMAT_NV16, GST_VIDEO_FORMAT_NV16}, {WL_SHM_FORMAT_NV61, DRM_FORMAT_NV61, GST_VIDEO_FORMAT_NV61}, diff --git a/ext/wayland/wlvideoformat.h b/ext/wayland/wlvideoformat.h index 331f582..ddfb1e0 100644 --- a/ext/wayland/wlvideoformat.h +++ b/ext/wayland/wlvideoformat.h @@ -30,6 +30,48 @@ G_BEGIN_DECLS +#ifndef DRM_FORMAT_NV15 +#define DRM_FORMAT_NV15 fourcc_code('N', 'V', '1', '5') +#endif + +#ifndef DRM_FORMAT_YUV420_8BIT +#define DRM_FORMAT_YUV420_8BIT fourcc_code('Y', 'U', '0', '8') +#endif + +#ifndef DRM_FORMAT_YUV420_10BIT +#define DRM_FORMAT_YUV420_10BIT fourcc_code('Y', 'U', '1', '0') +#endif + +#ifndef DRM_FORMAT_MOD_VENDOR_ARM +#define DRM_FORMAT_MOD_VENDOR_ARM 0x08 +#endif + +#ifndef DRM_FORMAT_MOD_ARM_AFBC +#define DRM_FORMAT_MOD_ARM_AFBC(__afbc_mode) fourcc_mod_code(ARM, __afbc_mode) +#endif + +#ifndef AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 +#define AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 (1ULL) +#endif + +#ifndef AFBC_FORMAT_MOD_SPARSE +#define AFBC_FORMAT_MOD_SPARSE (((__u64)1) << 6) +#endif + +#define DRM_AFBC_MODIFIER \ + (DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_SPARSE) | \ + DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16)) + +#ifndef GST_VIDEO_FLAG_ARM_AFBC +#define GST_VIDEO_FLAG_ARM_AFBC (1UL << 31) +#define GST_VIDEO_INFO_SET_AFBC(i) \ + GST_VIDEO_INFO_FLAG_SET (i, GST_VIDEO_FLAG_ARM_AFBC) +#define GST_VIDEO_INFO_UNSET_AFBC(i) \ + GST_VIDEO_INFO_FLAG_UNSET (i, GST_VIDEO_FLAG_ARM_AFBC) +#define GST_VIDEO_INFO_IS_AFBC(i) \ + GST_VIDEO_INFO_FLAG_IS_SET (i, GST_VIDEO_FLAG_ARM_AFBC) +#endif + enum wl_shm_format gst_video_format_to_wl_shm_format (GstVideoFormat format); gint gst_video_format_to_wl_dmabuf_format (GstVideoFormat format); GstVideoFormat gst_wl_shm_format_to_video_format (enum wl_shm_format wl_format); diff --git a/ext/wayland/wlwindow.c b/ext/wayland/wlwindow.c index f4e7ca9..2ba0eee 100644 --- a/ext/wayland/wlwindow.c +++ b/ext/wayland/wlwindow.c @@ -466,6 +466,14 @@ gst_wl_window_resize_video_surface (GstWlWindow * window, gboolean commit) dst.w = window->render_rectangle.w; dst.h = window->render_rectangle.h; + if (window->crop_w && window->crop_h) { + src.x = window->crop_x; + src.y = window->crop_y; + src.w = window->crop_w; + src.h = window->crop_h; + } + window->crop_dirty = FALSE; + if (window->video_viewport) { if (window->fill_mode == GST_WL_WINDOW_STRETCH) { res = dst; @@ -479,22 +487,20 @@ gst_wl_window_resize_video_surface (GstWlWindow * window, gboolean commit) if (src_ratio < dst_ratio) { int h = src.w / dst_ratio; - src.y = (src.h - h) / 2; + src.y += (src.h - h) / 2; src.h = h; } else if (src_ratio > dst_ratio) { int w = src.h * dst_ratio; - src.x = (src.w - w) / 2; + src.x += (src.w - w) / 2; src.w = w; } - - wp_viewport_set_source (window->video_viewport, - wl_fixed_from_int (src.x), wl_fixed_from_int (src.y), - wl_fixed_from_int (src.w), wl_fixed_from_int (src.h)); - res = dst; } wp_viewport_set_destination (window->video_viewport, res.w, res.h); + wp_viewport_set_source (window->video_viewport, + wl_fixed_from_int (src.x), wl_fixed_from_int (src.y), + wl_fixed_from_int (src.w), wl_fixed_from_int (src.h)); } else { gst_video_sink_center_rect (src, dst, &res, FALSE); } @@ -532,13 +538,14 @@ gst_wl_window_render (GstWlWindow * window, GstWlBuffer * buffer, const GstVideoInfo * info) { if (G_UNLIKELY (info)) { - window->video_width = - gst_util_uint64_scale_int_round (info->width, info->par_n, info->par_d); + window->video_width = info->width; window->video_height = info->height; wl_subsurface_set_sync (window->video_subsurface); gst_wl_window_resize_video_surface (window, FALSE); gst_wl_window_set_opaque (window, info); + } else if (window->crop_dirty) { + gst_wl_window_resize_video_surface (window, FALSE); } if (G_LIKELY (buffer)) { diff --git a/ext/wayland/wlwindow.h b/ext/wayland/wlwindow.h index 35d9d3c..672e15a 100644 --- a/ext/wayland/wlwindow.h +++ b/ext/wayland/wlwindow.h @@ -74,6 +74,8 @@ struct _GstWlWindow /* the size of the video in the buffers */ gint video_width, video_height; + gint crop_x, crop_y, crop_w, crop_h; + gboolean crop_dirty; /* when this is not set both the area_surface and the video_surface are not * visible and certain steps should be skipped */ -- 2.20.1