From 06e14dbe11da2a4007f0feacac1a758a94b09267 Mon Sep 17 00:00:00 2001 From: Jonas Karlman Date: Tue, 6 Aug 2024 09:06:00 +0000 Subject: [PATCH 01/13] avutil/hwcontext: Add hwdevice type for V4L2 Request API Add a hwdevice type for use with V4L2 Request API stateless decoding. AVV4L2RequestFramesContext is expected to be filled with a V4L2 coded pixel format and optional codec-specific extended controls before an AVHWFramesContext is initialized. Once initialized AVV4L2RequestFramesContextInternal describe the opened V4L2 stateless decoder session, with details about media/video file descriptors, formats of CAPTURE/OUTPUT queues and buffer pools for creating CAPTURE/OUTPUT buffers. AVHWFramesContext.pool defaults to create frame descriptors around a CAPTURE buffer, and frames are returned with AVFrame.data[0] pointing to a AVDRMFrameDescriptor and AVFrame.data[1] the CAPTURE buffer index. Signed-off-by: Jonas Karlman --- configure | 8 + libavutil/Makefile | 4 + libavutil/hwcontext.c | 4 + libavutil/hwcontext.h | 1 + libavutil/hwcontext_internal.h | 1 + libavutil/hwcontext_v4l2request.c | 596 +++++++++++++++++++++ libavutil/hwcontext_v4l2request.h | 70 +++ libavutil/hwcontext_v4l2request_internal.h | 67 +++ 8 files changed, 751 insertions(+) create mode 100644 libavutil/hwcontext_v4l2request.c create mode 100644 libavutil/hwcontext_v4l2request.h create mode 100644 libavutil/hwcontext_v4l2request_internal.h diff --git a/configure b/configure index 98b582a5d5..85abceeea7 100755 --- a/configure +++ b/configure @@ -359,6 +359,7 @@ External library support: --enable-omx-rpi enable OpenMAX IL code for Raspberry Pi [no] --enable-rkmpp enable Rockchip Media Process Platform code [no] --disable-v4l2-m2m disable V4L2 mem2mem code [autodetect] + --enable-v4l2-request enable V4L2 Request API code [no] --disable-vaapi disable Video Acceleration API (mainly Unix/Intel) code [autodetect] --disable-vdpau disable Nvidia Video Decode and Presentation API for Unix code [autodetect] --disable-videotoolbox disable VideoToolbox code [autodetect] @@ -2027,6 +2028,7 @@ HWACCEL_LIBRARY_LIST=" mmal omx opencl + v4l2_request " DOCUMENT_LIST=" @@ -3150,6 +3152,8 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32" ffnvcodec_deps_any="libdl LoadLibrary" mediacodec_deps="android mediandk" nvdec_deps="ffnvcodec" +v4l2_request_deps="linux_media_h v4l2_timeval_to_ns libdrm" +v4l2_request_suggest="libdrm" vaapi_x11_deps="xlib_x11" videotoolbox_hwaccel_deps="videotoolbox pthreads" videotoolbox_hwaccel_extralibs="-framework QuartzCore" @@ -7218,6 +7222,10 @@ if enabled v4l2_m2m; then check_cc vp9_v4l2_m2m linux/videodev2.h "int i = V4L2_PIX_FMT_VP9;" fi +if enabled v4l2_request; then + check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns +fi + check_headers sys/videoio.h test_code cc sys/videoio.h "struct v4l2_frmsizeenum vfse; vfse.discrete.width = 0;" && enable_sanitized struct_v4l2_frmivalenum_discrete diff --git a/libavutil/Makefile b/libavutil/Makefile index 6e6fa8d800..34c4838d63 100644 --- a/libavutil/Makefile +++ b/libavutil/Makefile @@ -48,6 +48,7 @@ HEADERS = adler32.h \ hwcontext_qsv.h \ hwcontext_mediacodec.h \ hwcontext_opencl.h \ + hwcontext_v4l2request.h \ hwcontext_vaapi.h \ hwcontext_videotoolbox.h \ hwcontext_vdpau.h \ @@ -201,6 +202,7 @@ OBJS-$(CONFIG_MACOS_KPERF) += macos_kperf.o OBJS-$(CONFIG_MEDIACODEC) += hwcontext_mediacodec.o OBJS-$(CONFIG_OPENCL) += hwcontext_opencl.o OBJS-$(CONFIG_QSV) += hwcontext_qsv.o +OBJS-$(CONFIG_V4L2_REQUEST) += hwcontext_v4l2request.o OBJS-$(CONFIG_VAAPI) += hwcontext_vaapi.o OBJS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.o OBJS-$(CONFIG_VDPAU) += hwcontext_vdpau.o @@ -222,6 +224,8 @@ SKIPHEADERS-$(CONFIG_D3D12VA) += hwcontext_d3d12va.h SKIPHEADERS-$(CONFIG_DXVA2) += hwcontext_dxva2.h SKIPHEADERS-$(CONFIG_QSV) += hwcontext_qsv.h SKIPHEADERS-$(CONFIG_OPENCL) += hwcontext_opencl.h +SKIPHEADERS-$(CONFIG_V4L2_REQUEST) += hwcontext_v4l2request.h \ + hwcontext_v4l2request_internal.h SKIPHEADERS-$(CONFIG_VAAPI) += hwcontext_vaapi.h SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.h SKIPHEADERS-$(CONFIG_VDPAU) += hwcontext_vdpau.h diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c index fa99a0d8a4..7fae9381da 100644 --- a/libavutil/hwcontext.c +++ b/libavutil/hwcontext.c @@ -65,6 +65,9 @@ static const HWContextType * const hw_table[] = { #endif #if CONFIG_VULKAN &ff_hwcontext_type_vulkan, +#endif +#if CONFIG_V4L2_REQUEST + &ff_hwcontext_type_v4l2request, #endif NULL, }; @@ -77,6 +80,7 @@ static const char *const hw_type_names[] = { [AV_HWDEVICE_TYPE_D3D12VA] = "d3d12va", [AV_HWDEVICE_TYPE_OPENCL] = "opencl", [AV_HWDEVICE_TYPE_QSV] = "qsv", + [AV_HWDEVICE_TYPE_V4L2REQUEST] = "v4l2request", [AV_HWDEVICE_TYPE_VAAPI] = "vaapi", [AV_HWDEVICE_TYPE_VDPAU] = "vdpau", [AV_HWDEVICE_TYPE_VIDEOTOOLBOX] = "videotoolbox", diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h index bac30debae..8cf50ddbd0 100644 --- a/libavutil/hwcontext.h +++ b/libavutil/hwcontext.h @@ -38,6 +38,7 @@ enum AVHWDeviceType { AV_HWDEVICE_TYPE_MEDIACODEC, AV_HWDEVICE_TYPE_VULKAN, AV_HWDEVICE_TYPE_D3D12VA, + AV_HWDEVICE_TYPE_V4L2REQUEST, }; /** diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h index e32b786238..fd0cf29c6e 100644 --- a/libavutil/hwcontext_internal.h +++ b/libavutil/hwcontext_internal.h @@ -158,6 +158,7 @@ extern const HWContextType ff_hwcontext_type_drm; extern const HWContextType ff_hwcontext_type_dxva2; extern const HWContextType ff_hwcontext_type_opencl; extern const HWContextType ff_hwcontext_type_qsv; +extern const HWContextType ff_hwcontext_type_v4l2request; extern const HWContextType ff_hwcontext_type_vaapi; extern const HWContextType ff_hwcontext_type_vdpau; extern const HWContextType ff_hwcontext_type_videotoolbox; diff --git a/libavutil/hwcontext_v4l2request.c b/libavutil/hwcontext_v4l2request.c new file mode 100644 index 0000000000..56d3fef8e2 --- /dev/null +++ b/libavutil/hwcontext_v4l2request.c @@ -0,0 +1,596 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include "avassert.h" +#include "hwcontext_drm.h" +#include "hwcontext_internal.h" +#include "hwcontext_v4l2request_internal.h" +#include "mem.h" + +typedef struct V4L2RequestDeviceContext { +} V4L2RequestDeviceContext; + +typedef struct V4L2RequestFramesContext { + AVV4L2RequestFramesContext p; + AVV4L2RequestFramesContextInternal internal; +} V4L2RequestFramesContext; + +typedef struct V4L2RequestFrameDescriptor { + AVDRMFrameDescriptor base; + AVBufferRef *ref; + uint32_t index; + int fd[AV_DRM_MAX_PLANES]; +} V4L2RequestFrameDescriptor; + +static const struct { + uint32_t pixelformat; + enum AVPixelFormat sw_format; + uint32_t drm_format; + uint64_t format_modifier; + uint32_t bit_depth; +} v4l2request_capture_pixelformats[] = { + { V4L2_PIX_FMT_NV12, AV_PIX_FMT_NV12, DRM_FORMAT_NV12, DRM_FORMAT_MOD_LINEAR, 8 }, +#if defined(V4L2_PIX_FMT_NV12_32L32) + { V4L2_PIX_FMT_NV12_32L32, AV_PIX_FMT_YUV420P, DRM_FORMAT_NV12, DRM_FORMAT_MOD_ALLWINNER_TILED, 8 }, +#endif +#if defined(V4L2_PIX_FMT_NV15) && defined(DRM_FORMAT_NV15) + { V4L2_PIX_FMT_NV15, AV_PIX_FMT_YUV420P10, DRM_FORMAT_NV15, DRM_FORMAT_MOD_LINEAR, 10 }, +#endif + { V4L2_PIX_FMT_NV16, AV_PIX_FMT_NV16, DRM_FORMAT_NV16, DRM_FORMAT_MOD_LINEAR, 8 }, +#if defined(V4L2_PIX_FMT_NV20) && defined(DRM_FORMAT_NV20) + { V4L2_PIX_FMT_NV20, AV_PIX_FMT_YUV422P10, DRM_FORMAT_NV20, DRM_FORMAT_MOD_LINEAR, 10 }, +#endif +#if defined(V4L2_PIX_FMT_P010) && defined(DRM_FORMAT_P010) + { V4L2_PIX_FMT_P010, AV_PIX_FMT_P010, DRM_FORMAT_P010, DRM_FORMAT_MOD_LINEAR, 10 }, +#endif +}; + +static int v4l2request_set_drm_descriptor(AVDRMFrameDescriptor *desc, + struct v4l2_format *format) +{ + AVDRMLayerDescriptor *layer = &desc->layers[0]; + uint32_t pixelformat = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? + format->fmt.pix_mp.pixelformat : + format->fmt.pix.pixelformat; + uint64_t format_modifier; + + layer->format = 0; + for (int i = 0; i < FF_ARRAY_ELEMS(v4l2request_capture_pixelformats); i++) { + if (pixelformat == v4l2request_capture_pixelformats[i].pixelformat) { + layer->format = v4l2request_capture_pixelformats[i].drm_format; + format_modifier = v4l2request_capture_pixelformats[i].format_modifier; + break; + } + } + if (!layer->format) + return AVERROR(ENOENT); + + for (int i = 0; i < desc->nb_objects; i++) { + desc->objects[i].format_modifier = format_modifier; + desc->objects[i].size = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? + format->fmt.pix_mp.plane_fmt[i].sizeimage : + format->fmt.pix.sizeimage; + } + + desc->nb_layers = 1; + layer->nb_planes = 2; + + layer->planes[0].object_index = 0; + layer->planes[0].offset = 0; + layer->planes[0].pitch = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? + format->fmt.pix_mp.plane_fmt[0].bytesperline : + format->fmt.pix.bytesperline; + + layer->planes[1].object_index = 0; + layer->planes[1].offset = layer->planes[0].pitch * + (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? + format->fmt.pix_mp.height : + format->fmt.pix.height); + layer->planes[1].pitch = layer->planes[0].pitch; + + return 0; +} + +static void v4l2request_device_uninit(AVHWDeviceContext *hwdev) +{ +} + +static int v4l2request_device_create(AVHWDeviceContext *hwdev, const char *device, + AVDictionary *opts, int flags) +{ + return 0; +} + +static int v4l2request_open_decoder(AVHWFramesContext *hwfc) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + + // Ensure codec pixelformat is set + if (!fctx->pixelformat) + return AVERROR(EINVAL); + + // TODO: locate a decoder supporting the requested pixelformat + + return AVERROR(ENOSYS); +} + +static AVBufferRef *v4l2request_v4l2_buffer_alloc(AVHWFramesContext *hwfc, + struct v4l2_format *format) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + struct v4l2_create_buffers buffers = { + .count = 1, + .memory = V4L2_MEMORY_MMAP, + .format = *format, + }; + struct v4l2_buffer *buffer; + uint8_t num_planes; + AVBufferRef *ref; + + num_planes = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? + format->fmt.pix_mp.num_planes : 0; + + ref = av_buffer_allocz(sizeof(struct v4l2_buffer) + + (sizeof(struct v4l2_plane) * num_planes)); + if (!ref) + return NULL; + + buffer = (struct v4l2_buffer *)ref->data; + buffer->type = format->type; + + if (num_planes) { + buffer->length = num_planes; + buffer->m.planes = (struct v4l2_plane *)(buffer + 1); + } + + // Create the buffer + if (ioctl(fctxi->video_fd, VIDIOC_CREATE_BUFS, &buffers) < 0) { + av_log(hwfc, AV_LOG_ERROR, "Failed to create buffer of type %d: %s (%d)\n", + buffer->type, strerror(errno), errno); + goto fail; + } + + buffer->memory = buffers.memory; + buffer->index = buffers.index; + + // Query more details of the created buffer + if (ioctl(fctxi->video_fd, VIDIOC_QUERYBUF, buffer) < 0) { + av_log(hwfc, AV_LOG_ERROR, "Failed to query buffer %d of type %d: %s (%d)\n", + buffer->index, buffer->type, strerror(errno), errno); + goto fail; + } + + return ref; + +fail: + av_buffer_unref(&ref); + return NULL; +} + +static AVBufferRef *v4l2request_capture_buffer_alloc(void *opaque, size_t size) +{ + AVHWFramesContext *hwfc = opaque; + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + + return v4l2request_v4l2_buffer_alloc(hwfc, &fctxi->capture.format); +} + +static AVBufferRef *v4l2request_output_buffer_alloc(void *opaque, size_t size) +{ + AVHWFramesContext *hwfc = opaque; + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + + return v4l2request_v4l2_buffer_alloc(hwfc, &fctxi->output.format); +} + +static void v4l2request_frame_free(void *opaque, uint8_t *data) +{ + V4L2RequestFrameDescriptor *desc = (V4L2RequestFrameDescriptor *)data; + + // Close the exported CAPTURE buffer memory planes + for (int i = 0; i < FF_ARRAY_ELEMS(desc->fd); i++) { + if (desc->fd[i] >= 0) { + close(desc->fd[i]); + desc->fd[i] = -1; + } + } + + // Return the CAPTURE buffer to the frames context CAPTURE pool + av_buffer_unref(&desc->ref); +} + +static AVBufferRef *v4l2request_frame_alloc(void *opaque, size_t size) +{ + AVHWFramesContext *hwfc = opaque; + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + struct v4l2_format *format = &fctxi->capture.format; + V4L2RequestFrameDescriptor *desc; + struct v4l2_buffer *buffer; + AVBufferRef *ref; + uint8_t *data; + + data = av_mallocz(size); + if (!data) + return NULL; + + ref = av_buffer_create(data, size, v4l2request_frame_free, + hwfc, AV_BUFFER_FLAG_READONLY); + if (!ref) { + av_free(data); + return NULL; + } + + // Set initial default values + desc = (V4L2RequestFrameDescriptor *)data; + for (int i = 0; i < FF_ARRAY_ELEMS(desc->fd); i++) + desc->fd[i] = -1; + + // Get a CAPTURE buffer from frames context CAPTURE pool + desc->ref = av_buffer_pool_get(fctxi->capture.pool); + if (!desc->ref) + goto fail; + + buffer = (struct v4l2_buffer *)desc->ref->data; + desc->index = buffer->index; + + // Export CAPTURE buffer memory planes + desc->base.nb_objects = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? + format->fmt.pix_mp.num_planes : 1; + av_assert0(desc->base.nb_objects <= AV_DRM_MAX_PLANES); + for (int i = 0; i < desc->base.nb_objects; i++) { + struct v4l2_exportbuffer exportbuffer = { + .type = buffer->type, + .index = buffer->index, + .plane = i, + .flags = O_RDONLY, + }; + if (ioctl(fctxi->video_fd, VIDIOC_EXPBUF, &exportbuffer) < 0) { + av_log(hwfc, AV_LOG_ERROR, "Failed to export memory plane %d (%d): %s (%d)\n", + i, buffer->index, strerror(errno), errno); + goto fail; + } + desc->base.objects[i].fd = desc->fd[i] = exportbuffer.fd; + } + + // Set AVDRMFrameDescriptor based on CAPTURE buffer format + if (v4l2request_set_drm_descriptor(&desc->base, format) < 0) + goto fail; + + return ref; + +fail: + av_buffer_unref(&ref); + return NULL; +} + +static int v4l2request_frames_init(AVHWFramesContext *hwfc) +{ + V4L2RequestFramesContext *hwctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi; + uint32_t pixelformat; + int ret; + + // Set initial default values + fctxi = &hwctx->internal; + hwctx->p.internal = fctxi; + fctxi->media_fd = -1; + fctxi->video_fd = -1; + + // Locate and open a capable video decoder device + ret = v4l2request_open_decoder(hwfc); + if (ret < 0) + return ret; + + // Reset init controls after video device is opened + hwctx->p.init_controls = NULL; + hwctx->p.nb_init_controls = 0; + + // Update frames context with CAPTURE format details + if (V4L2_TYPE_IS_MULTIPLANAR(fctxi->capture.format.type)) { + hwfc->width = fctxi->capture.format.fmt.pix_mp.width; + hwfc->height = fctxi->capture.format.fmt.pix_mp.height; + pixelformat = fctxi->capture.format.fmt.pix_mp.pixelformat; + } else { + hwfc->width = fctxi->capture.format.fmt.pix.width; + hwfc->height = fctxi->capture.format.fmt.pix.height; + pixelformat = fctxi->capture.format.fmt.pix.pixelformat; + } + + hwfc->sw_format = AV_PIX_FMT_NONE; + for (int i = 0; i < FF_ARRAY_ELEMS(v4l2request_capture_pixelformats); i++) { + if (pixelformat == v4l2request_capture_pixelformats[i].pixelformat) { + hwctx->p.bit_depth = v4l2request_capture_pixelformats[i].bit_depth; + hwfc->sw_format = v4l2request_capture_pixelformats[i].sw_format; + break; + } + } + + // Initialize buffer pool for CAPTURE buffers + fctxi->capture.pool = av_buffer_pool_init2(sizeof(struct v4l2_buffer), hwfc, + v4l2request_capture_buffer_alloc, NULL); + if (!fctxi->capture.pool) + return AVERROR(ENOMEM); + + // Initialize buffer pool for OUTPUT buffers + fctxi->output.pool = av_buffer_pool_init2(sizeof(struct v4l2_buffer), hwfc, + v4l2request_output_buffer_alloc, NULL); + if (!fctxi->output.pool) + return AVERROR(ENOMEM); + + // Initialize buffer pool for frame descriptors + ffhwframesctx(hwfc)->pool_internal = + av_buffer_pool_init2(sizeof(V4L2RequestFrameDescriptor), hwfc, + v4l2request_frame_alloc, NULL); + if (!ffhwframesctx(hwfc)->pool_internal) + return AVERROR(ENOMEM); + + av_log(hwfc, AV_LOG_VERBOSE, "Using CAPTURE buffer format %s (%dx%d)\n", + av_fourcc2str(pixelformat), hwfc->width, hwfc->height); + + return 0; +} + +static void v4l2request_frames_uninit(AVHWFramesContext *hwfc) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + + av_buffer_pool_uninit(&fctxi->capture.pool); + av_buffer_pool_uninit(&fctxi->output.pool); + + if (fctxi->video_fd >= 0) { + close(fctxi->video_fd); + fctxi->video_fd = -1; + } + + if (fctxi->media_fd) { + close(fctxi->media_fd); + fctxi->media_fd = -1; + } +} + +static int v4l2request_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame) +{ + V4L2RequestFrameDescriptor *desc; + + frame->buf[0] = av_buffer_pool_get(hwfc->pool); + if (!frame->buf[0]) + return AVERROR(ENOMEM); + + desc = (V4L2RequestFrameDescriptor *)frame->buf[0]->data; + frame->data[0] = (uint8_t *)&desc->base; + frame->data[1] = (uint8_t *)(uintptr_t)desc->index; + + frame->format = AV_PIX_FMT_DRM_PRIME; + frame->width = hwfc->width; + frame->height = hwfc->height; + + return 0; +} + +typedef struct V4L2RequestMapping { + // Address and length of each mmap()ed region. + int nb_regions; + int object[AV_DRM_MAX_PLANES]; + void *address[AV_DRM_MAX_PLANES]; + size_t length[AV_DRM_MAX_PLANES]; +} V4L2RequestMapping; + +static void v4l2request_unmap_frame(AVHWFramesContext *hwfc, + HWMapDescriptor *hwmap) +{ + V4L2RequestMapping *map = hwmap->priv; + + for (int i = 0; i < map->nb_regions; i++) { + struct dma_buf_sync sync = { + .flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ, + }; + ioctl(map->object[i], DMA_BUF_IOCTL_SYNC, &sync); + munmap(map->address[i], map->length[i]); + } + + av_free(map); +} + +static int v4l2request_map_frame(AVHWFramesContext *hwfc, + AVFrame *dst, const AVFrame *src) +{ + const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0]; + struct dma_buf_sync sync = { + .flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ, + }; + V4L2RequestMapping *map; + int ret, i, p, plane; + void *addr; + + map = av_mallocz(sizeof(*map)); + if (!map) + return AVERROR(ENOMEM); + + av_assert0(desc->nb_objects <= AV_DRM_MAX_PLANES); + for (i = 0; i < desc->nb_objects; i++) { + addr = mmap(NULL, desc->objects[i].size, PROT_READ, MAP_SHARED, + desc->objects[i].fd, 0); + if (addr == MAP_FAILED) { + ret = AVERROR(errno); + av_log(hwfc, AV_LOG_ERROR, "Failed to map DRM object %d to memory: %s (%d)\n", + desc->objects[i].fd, strerror(errno), errno); + goto fail; + } + + map->address[i] = addr; + map->length[i] = desc->objects[i].size; + map->object[i] = desc->objects[i].fd; + + /* + * We're not checking for errors here because the kernel may not + * support the ioctl, in which case its okay to carry on + */ + ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_SYNC, &sync); + } + map->nb_regions = i; + + plane = 0; + for (i = 0; i < desc->nb_layers; i++) { + const AVDRMLayerDescriptor *layer = &desc->layers[i]; + for (p = 0; p < layer->nb_planes; p++) { + dst->data[plane] = + (uint8_t *)map->address[layer->planes[p].object_index] + + layer->planes[p].offset; + dst->linesize[plane] = layer->planes[p].pitch; + ++plane; + } + } + av_assert0(plane <= AV_DRM_MAX_PLANES); + + dst->width = src->width; + dst->height = src->height; + + ret = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, + v4l2request_unmap_frame, map); + if (ret < 0) + goto fail; + + return 0; + +fail: + for (i = 0; i < desc->nb_objects; i++) { + if (map->address[i]) + munmap(map->address[i], map->length[i]); + } + av_free(map); + return ret; +} + +static int v4l2request_transfer_get_formats(AVHWFramesContext *hwfc, + enum AVHWFrameTransferDirection dir, + enum AVPixelFormat **formats) +{ + enum AVPixelFormat *fmts; + + if (dir == AV_HWFRAME_TRANSFER_DIRECTION_TO) + return AVERROR(ENOSYS); + + fmts = av_malloc_array(2, sizeof(*fmts)); + if (!fmts) + return AVERROR(ENOMEM); + + fmts[0] = hwfc->sw_format; + fmts[1] = AV_PIX_FMT_NONE; + + if (hwfc->sw_format == AV_PIX_FMT_YUV420P || + hwfc->sw_format == AV_PIX_FMT_YUV420P10 || + hwfc->sw_format == AV_PIX_FMT_YUV422P10) + fmts[0] = AV_PIX_FMT_NONE; + + *formats = fmts; + return 0; +} + +static int v4l2request_transfer_data_from(AVHWFramesContext *hwfc, + AVFrame *dst, const AVFrame *src) +{ + AVFrame *map; + int ret; + + if (dst->width > hwfc->width || dst->height > hwfc->height) + return AVERROR(EINVAL); + + map = av_frame_alloc(); + if (!map) + return AVERROR(ENOMEM); + map->format = dst->format; + + ret = v4l2request_map_frame(hwfc, map, src); + if (ret) + goto fail; + + map->width = dst->width; + map->height = dst->height; + + ret = av_frame_copy(dst, map); + if (ret) + goto fail; + + ret = 0; +fail: + av_frame_free(&map); + return ret; +} + +static int v4l2request_map_from(AVHWFramesContext *hwfc, AVFrame *dst, + const AVFrame *src, int flags) +{ + int ret; + + if (!(flags & AV_HWFRAME_MAP_READ)) + return AVERROR(ENOSYS); + + if (hwfc->sw_format == AV_PIX_FMT_NONE || + hwfc->sw_format == AV_PIX_FMT_YUV420P || + hwfc->sw_format == AV_PIX_FMT_YUV420P10 || + hwfc->sw_format == AV_PIX_FMT_YUV422P10) + return AVERROR(ENOSYS); + else if (dst->format == AV_PIX_FMT_NONE) + dst->format = hwfc->sw_format; + else if (hwfc->sw_format != dst->format) + return AVERROR(ENOSYS); + + ret = v4l2request_map_frame(hwfc, dst, src); + if (ret) + return ret; + + return av_frame_copy_props(dst, src); +} + +const HWContextType ff_hwcontext_type_v4l2request = { + .type = AV_HWDEVICE_TYPE_V4L2REQUEST, + .name = "V4L2 Request API", + + .device_hwctx_size = sizeof(V4L2RequestDeviceContext), + .device_create = v4l2request_device_create, + .device_uninit = v4l2request_device_uninit, + + .frames_hwctx_size = sizeof(V4L2RequestFramesContext), + .frames_init = v4l2request_frames_init, + .frames_uninit = v4l2request_frames_uninit, + .frames_get_buffer = v4l2request_get_buffer, + .transfer_get_formats = v4l2request_transfer_get_formats, + .transfer_data_from = v4l2request_transfer_data_from, + .map_from = v4l2request_map_from, + + .pix_fmts = (const enum AVPixelFormat[]) { + AV_PIX_FMT_DRM_PRIME, + AV_PIX_FMT_NONE + }, +}; diff --git a/libavutil/hwcontext_v4l2request.h b/libavutil/hwcontext_v4l2request.h new file mode 100644 index 0000000000..ec6beadd64 --- /dev/null +++ b/libavutil/hwcontext_v4l2request.h @@ -0,0 +1,70 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_HWCONTEXT_V4L2REQUEST_H +#define AVUTIL_HWCONTEXT_V4L2REQUEST_H + +#include +#include + +/** + * @file + * An API-specific header for AV_HWDEVICE_TYPE_V4L2REQUEST. + */ + +typedef struct AVV4L2RequestFramesContextInternal AVV4L2RequestFramesContextInternal; + +/** + * V4L2 Request API frames context. + * + * This struct is allocated as AVHWFramesContext.hwctx + */ +typedef struct AVV4L2RequestFramesContext { + /** + * Internal context for the initialized V4L2 stateless decoder/encoder session. + */ + AVV4L2RequestFramesContextInternal *internal; + + /** + * V4L2_PIX_FMT_* coded pixel format to set on the OUTPUT queue (decoders) + * or the CAPTURE queue (encoders) during initialization. + * + * This field must be set by caller before av_hwframe_ctx_init() is called. + */ + uint32_t pixelformat; + + /** + * Optional bit depth of the frame pixel format, e.g. 8 or 10. + * + * This field should be set by caller before av_hwframe_ctx_init() is called, + * the field will be updated to match the selected frame pixel format after + * successful initialization. + */ + uint32_t bit_depth; + + /** + * Optional codec-specific extended controls to be set during initialization. + * + * These fields should be set by caller before av_hwframe_ctx_init() is called, + * fields are reset to NULL and 0 after successful initialization. + */ + struct v4l2_ext_control *init_controls; + int nb_init_controls; +} AVV4L2RequestFramesContext; + +#endif /* AVUTIL_HWCONTEXT_V4L2REQUEST_H */ diff --git a/libavutil/hwcontext_v4l2request_internal.h b/libavutil/hwcontext_v4l2request_internal.h new file mode 100644 index 0000000000..20436b2b2d --- /dev/null +++ b/libavutil/hwcontext_v4l2request_internal.h @@ -0,0 +1,67 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_HWCONTEXT_V4L2REQUEST_INTERNAL_H +#define AVUTIL_HWCONTEXT_V4L2REQUEST_INTERNAL_H + +#include "buffer.h" +#include "hwcontext_v4l2request.h" + +/** + * @file + * FFmpeg internal API-specific header for AV_HWDEVICE_TYPE_V4L2REQUEST. + */ + +/** + * Internal context for the initialized V4L2 stateless decoder/encoder session. + */ +struct AVV4L2RequestFramesContextInternal { + /** + * Media device file descriptor of the initialized session. + */ + int media_fd; + + /** + * Video device file descriptor of the initialized session. + */ + int video_fd; + + /** + * Details of the initialized CAPTURE and OUTPUT queues. + */ + struct { + /** + * V4L2 buffer format. + */ + struct v4l2_format format; + + /** + * V4L2 buffer capabilities flags. + */ + uint32_t capabilities; + + /** + * Buffer pool of allocated V4L2 buffers. + * + * AVBufferRef.data points to a struct v4l2_buffer for the created buffer. + */ + AVBufferPool *pool; + } capture, output; +}; + +#endif /* AVUTIL_HWCONTEXT_V4L2REQUEST_INTERNAL_H */ -- 2.53.0 From f71de0fcf0e468d3ce832609b0abfeaa5c86930b Mon Sep 17 00:00:00 2001 From: Jonas Karlman Date: Tue, 6 Aug 2024 09:06:02 +0000 Subject: [PATCH 02/13] avutil/hwcontext_v4l2request: Probe for a capable media and video device Probe all media devices and its linked video devices to locate a video device that support stateless decoding of the specific codec using the V4L2 Request API. A video device is deemed capable when all tests pass, e.g. kernel drivers support the coded pixel format and the frame size is supported. Basic flow for initialization follows the kernel Memory-to-memory Stateless Video Decoder Interface > Initialization [1]. [1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#initialization Co-developed-by: Jernej Skrabec Signed-off-by: Jernej Skrabec Co-developed-by: Alex Bee Signed-off-by: Alex Bee Signed-off-by: Jonas Karlman --- configure | 9 +- libavutil/hwcontext_v4l2request.c | 506 +++++++++++++++++++++++++++++- 2 files changed, 510 insertions(+), 5 deletions(-) diff --git a/configure b/configure index 85abceeea7..d63b8a5abf 100755 --- a/configure +++ b/configure @@ -1968,6 +1968,7 @@ EXTERNAL_LIBRARY_LIST=" libtorch libtwolame libuavs3d + libudev libv4l2 libvmaf libvorbis @@ -3152,8 +3153,8 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32" ffnvcodec_deps_any="libdl LoadLibrary" mediacodec_deps="android mediandk" nvdec_deps="ffnvcodec" -v4l2_request_deps="linux_media_h v4l2_timeval_to_ns libdrm" -v4l2_request_suggest="libdrm" +v4l2_request_deps="linux_media_h v4l2_timeval_to_ns libdrm libudev" +v4l2_request_suggest="libdrm libudev" vaapi_x11_deps="xlib_x11" videotoolbox_hwaccel_deps="videotoolbox pthreads" videotoolbox_hwaccel_extralibs="-framework QuartzCore" @@ -4052,7 +4053,7 @@ swscale_suggest="libm stdatomic" avcodec_extralibs="pthreads_extralibs iconv_extralibs dxva2_extralibs lcms2_extralibs" avfilter_extralibs="pthreads_extralibs" -avutil_extralibs="d3d11va_extralibs d3d12va_extralibs mediacodec_extralibs nanosleep_extralibs pthreads_extralibs vaapi_drm_extralibs vaapi_x11_extralibs vaapi_win32_extralibs vdpau_x11_extralibs" +avutil_extralibs="d3d11va_extralibs d3d12va_extralibs mediacodec_extralibs nanosleep_extralibs pthreads_extralibs v4l2_request_extralibs vaapi_drm_extralibs vaapi_x11_extralibs vaapi_win32_extralibs vdpau_x11_extralibs" # programs ffmpeg_deps="avcodec avfilter avformat threads" @@ -7224,6 +7225,8 @@ fi if enabled v4l2_request; then check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns + check_pkg_config libudev libudev libudev.h udev_new + v4l2_request_extralibs="$libudev_extralibs" fi check_headers sys/videoio.h diff --git a/libavutil/hwcontext_v4l2request.c b/libavutil/hwcontext_v4l2request.c index 56d3fef8e2..f316f57826 100644 --- a/libavutil/hwcontext_v4l2request.c +++ b/libavutil/hwcontext_v4l2request.c @@ -26,6 +26,7 @@ #include #include +#include #include "avassert.h" #include "hwcontext_drm.h" @@ -33,7 +34,16 @@ #include "hwcontext_v4l2request_internal.h" #include "mem.h" +typedef struct V4L2RequestVideoDecoder { + dev_t media_dev; + dev_t video_dev; + uint32_t *pixelformats; + int nb_pixelformats; +} V4L2RequestVideoDecoder; + typedef struct V4L2RequestDeviceContext { + V4L2RequestVideoDecoder *decoders; + int nb_decoders; } V4L2RequestDeviceContext; typedef struct V4L2RequestFramesContext { @@ -119,25 +129,517 @@ static int v4l2request_set_drm_descriptor(AVDRMFrameDescriptor *desc, static void v4l2request_device_uninit(AVHWDeviceContext *hwdev) { + V4L2RequestDeviceContext *hwctx = hwdev->hwctx; + + av_freep(&hwctx->decoders); + hwctx->nb_decoders = 0; } static int v4l2request_device_create(AVHWDeviceContext *hwdev, const char *device, AVDictionary *opts, int flags) { + V4L2RequestDeviceContext *hwctx = hwdev->hwctx; + + hwctx->decoders = NULL; + hwctx->nb_decoders = 0; + + // TODO: enumerate V4L2 Request API capable video decoders + // and fill hwctx->decoders and hwctx->nb_decoders, + // limit to decoders for the media 'device' when specified + return 0; } +static int v4l2request_set_format(AVHWFramesContext *hwfc, + enum v4l2_buf_type type, + uint32_t pixelformat, + uint32_t buffersize) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + struct v4l2_format format = { + .type = type, + }; + + if (V4L2_TYPE_IS_MULTIPLANAR(type)) { + format.fmt.pix_mp.width = hwfc->width; + format.fmt.pix_mp.height = hwfc->height; + format.fmt.pix_mp.pixelformat = pixelformat; + format.fmt.pix_mp.plane_fmt[0].sizeimage = buffersize; + format.fmt.pix_mp.num_planes = 1; + } else { + format.fmt.pix.width = hwfc->width; + format.fmt.pix.height = hwfc->height; + format.fmt.pix.pixelformat = pixelformat; + format.fmt.pix.sizeimage = buffersize; + } + + if (ioctl(fctxi->video_fd, VIDIOC_S_FMT, &format) < 0) + return AVERROR(errno); + + return 0; +} + +static int v4l2request_select_capture_format(AVHWFramesContext *hwfc) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + enum v4l2_buf_type type = fctxi->capture.format.type; + struct v4l2_format format = { + .type = type, + }; + struct v4l2_fmtdesc fmtdesc = { + .index = 0, + .type = type, + }; + uint32_t pixelformat; + + // Get the driver preferred (or default) format + if (ioctl(fctxi->video_fd, VIDIOC_G_FMT, &format) < 0) + return AVERROR(errno); + + pixelformat = V4L2_TYPE_IS_MULTIPLANAR(type) ? + format.fmt.pix_mp.pixelformat : + format.fmt.pix.pixelformat; + + // Use the driver preferred format when it is supported + for (int i = 0; i < FF_ARRAY_ELEMS(v4l2request_capture_pixelformats); i++) { + if (pixelformat == v4l2request_capture_pixelformats[i].pixelformat && + (fctx->bit_depth == v4l2request_capture_pixelformats[i].bit_depth || + !fctx->bit_depth)) + return v4l2request_set_format(hwfc, type, pixelformat, 0); + } + + // Otherwise, use first format that is supported + while (ioctl(fctxi->video_fd, VIDIOC_ENUM_FMT, &fmtdesc) >= 0) { + for (int i = 0; i < FF_ARRAY_ELEMS(v4l2request_capture_pixelformats); i++) { + if (fmtdesc.pixelformat == v4l2request_capture_pixelformats[i].pixelformat && + (fctx->bit_depth == v4l2request_capture_pixelformats[i].bit_depth || + !fctx->bit_depth)) + return v4l2request_set_format(hwfc, type, fmtdesc.pixelformat, 0); + } + + fmtdesc.index++; + } + + return AVERROR(errno); +} + +static int v4l2request_try_framesize(AVHWFramesContext *hwfc, + uint32_t pixelformat) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + struct v4l2_frmsizeenum frmsize = { + .index = 0, + .pixel_format = pixelformat, + }; + + // Enumerate and check if frame size is supported + while (ioctl(fctxi->video_fd, VIDIOC_ENUM_FRAMESIZES, &frmsize) >= 0) { + if (frmsize.type == V4L2_FRMSIZE_TYPE_DISCRETE && + hwfc->width == frmsize.discrete.width && + hwfc->height == frmsize.discrete.height) { + return 0; + } else if ((frmsize.type == V4L2_FRMSIZE_TYPE_STEPWISE || + frmsize.type == V4L2_FRMSIZE_TYPE_CONTINUOUS) && + hwfc->width >= frmsize.stepwise.min_width && + hwfc->height >= frmsize.stepwise.min_height && + hwfc->width <= frmsize.stepwise.max_width && + hwfc->height <= frmsize.stepwise.max_height) { + return 0; + } + + frmsize.index++; + } + + return AVERROR(errno); +} + +static int v4l2request_try_format(AVHWFramesContext *hwfc, + enum v4l2_buf_type type, + uint32_t pixelformat) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + struct v4l2_fmtdesc fmtdesc = { + .index = 0, + .type = type, + }; + + // Enumerate and check if format is supported + while (ioctl(fctxi->video_fd, VIDIOC_ENUM_FMT, &fmtdesc) >= 0) { + if (fmtdesc.pixelformat == pixelformat) + return 0; + + fmtdesc.index++; + } + + return AVERROR(errno); +} + +static int v4l2request_set_controls(AVHWFramesContext *hwfc, + struct v4l2_ext_control *control, int count) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + struct v4l2_ext_controls controls = { + .controls = control, + .count = count, + }; + + if (!control || !count) + return 0; + + if (ioctl(fctxi->video_fd, VIDIOC_S_EXT_CTRLS, &controls) < 0) + return AVERROR(errno); + + return 0; +} + +static int v4l2request_probe_video_device(AVHWFramesContext *hwfc, + const char *path, + uint32_t pixelformat, + uint32_t buffersize) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + struct v4l2_capability capability; + struct v4l2_create_buffers buffers; + unsigned int capabilities; + int ret; + + /* + * Open video device in non-blocking mode to support decoding using + * multiple queued requests, required for e.g. multi stage decoding. + */ + fctxi->video_fd = open(path, O_RDWR | O_NONBLOCK); + if (fctxi->video_fd < 0) { + ret = AVERROR(errno); + av_log(hwfc, AV_LOG_ERROR, "Failed to open video device %s: %s (%d)\n", + path, strerror(errno), errno); + return ret; + } + + // Query capabilities of the video device + if (ioctl(fctxi->video_fd, VIDIOC_QUERYCAP, &capability) < 0) { + ret = AVERROR(errno); + av_log(hwfc, AV_LOG_ERROR, "Failed to query capabilities of %s: %s (%d)\n", + path, strerror(errno), errno); + goto fail; + } + + // Use device capabilities of the opened device when supported + capabilities = (capability.capabilities & V4L2_CAP_DEVICE_CAPS) ? + capability.device_caps : capability.capabilities; + + // Ensure streaming is supported on the video device + if ((capabilities & V4L2_CAP_STREAMING) != V4L2_CAP_STREAMING) { + ret = AVERROR(EINVAL); + av_log(hwfc, AV_LOG_VERBOSE, "Device %s is missing streaming capability\n", path); + goto fail; + } + + // Ensure multi- or single-planar API can be used + if ((capabilities & V4L2_CAP_VIDEO_M2M_MPLANE) == V4L2_CAP_VIDEO_M2M_MPLANE) { + fctxi->output.format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + fctxi->capture.format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + } else if ((capabilities & V4L2_CAP_VIDEO_M2M) == V4L2_CAP_VIDEO_M2M) { + fctxi->output.format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT; + fctxi->capture.format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + } else { + ret = AVERROR(EINVAL); + av_log(hwfc, AV_LOG_VERBOSE, "Device %s is missing mem2mem capability\n", path); + goto fail; + } + + // Query OUTPUT buffer capabilities + buffers = (struct v4l2_create_buffers) { + .count = 0, + .memory = V4L2_MEMORY_MMAP, + .format.type = fctxi->output.format.type, + }; + if (ioctl(fctxi->video_fd, VIDIOC_CREATE_BUFS, &buffers) < 0) { + ret = AVERROR(errno); + av_log(hwfc, AV_LOG_ERROR, + "Failed to query OUTPUT buffer capabilities of %s: %s (%d)\n", + path, strerror(errno), errno); + goto fail; + } + fctxi->output.capabilities = buffers.capabilities; + + // Ensure requests can be used + if ((buffers.capabilities & V4L2_BUF_CAP_SUPPORTS_REQUESTS) != + V4L2_BUF_CAP_SUPPORTS_REQUESTS) { + ret = AVERROR(EINVAL); + av_log(hwfc, AV_LOG_VERBOSE, "Device %s is missing support for requests\n", path); + goto fail; + } + + // Ensure the codec pixelformat can be used + ret = v4l2request_try_format(hwfc, fctxi->output.format.type, pixelformat); + if (ret < 0) { + av_log(hwfc, AV_LOG_VERBOSE, "Device %s is missing support for pixelformat %s\n", + path, av_fourcc2str(pixelformat)); + goto fail; + } + + // Ensure frame size is supported, when driver support ENUM_FRAMESIZES + ret = v4l2request_try_framesize(hwfc, pixelformat); + if (ret < 0 && ret != AVERROR(ENOTTY)) { + av_log(hwfc, AV_LOG_VERBOSE, + "Device %s is missing support for frame size %dx%d of pixelformat %s\n", + path, hwfc->width, hwfc->height, av_fourcc2str(pixelformat)); + goto fail; + } + + // Set the codec pixelformat and OUTPUT buffersize to be used + ret = v4l2request_set_format(hwfc, fctxi->output.format.type, pixelformat, buffersize); + if (ret < 0) { + av_log(hwfc, AV_LOG_ERROR, + "Failed to set OUTPUT pixelformat %s of %s: %s (%d)\n", + av_fourcc2str(pixelformat), path, strerror(errno), errno); + goto fail; + } + + // Get format details for OUTPUT buffers + if (ioctl(fctxi->video_fd, VIDIOC_G_FMT, &fctxi->output.format) < 0) { + ret = AVERROR(errno); + av_log(hwfc, AV_LOG_ERROR, "Failed to get OUTPUT format: %s (%d)\n", + strerror(errno), errno); + goto fail; + } + + /* + * Set any codec specific controls that can help assist the driver + * make a decision on what CAPTURE buffer format can be used. + */ + ret = v4l2request_set_controls(hwfc, fctx->init_controls, fctx->nb_init_controls); + if (ret < 0) { + av_log(hwfc, AV_LOG_VERBOSE, + "Failed to set %d control(s): %s (%d)\n", + fctx->nb_init_controls, strerror(errno), errno); + goto fail; + } + + // Select a supported CAPTURE buffer format + ret = v4l2request_select_capture_format(hwfc); + if (ret < 0) { + av_log(hwfc, AV_LOG_VERBOSE, + "Failed to select a CAPTURE format %s of %s: %s (%d)\n", + av_fourcc2str(pixelformat), path, strerror(errno), errno); + goto fail; + } + + // Query CAPTURE buffer capabilities + buffers = (struct v4l2_create_buffers) { + .count = 0, + .memory = V4L2_MEMORY_MMAP, + .format.type = fctxi->capture.format.type, + }; + if (ioctl(fctxi->video_fd, VIDIOC_CREATE_BUFS, &buffers) < 0) { + ret = AVERROR(errno); + av_log(hwfc, AV_LOG_ERROR, + "Failed to query CAPTURE buffer capabilities of %s: %s (%d)\n", + path, strerror(errno), errno); + goto fail; + } + fctxi->capture.capabilities = buffers.capabilities; + + // Get format details for CAPTURE buffers + if (ioctl(fctxi->video_fd, VIDIOC_G_FMT, &fctxi->capture.format) < 0) { + ret = AVERROR(errno); + av_log(hwfc, AV_LOG_ERROR, "Failed to get CAPTURE format: %s (%d)\n", + strerror(errno), errno); + goto fail; + } + + // All tests passed, video device should be capable + return 0; + +fail: + if (fctxi->video_fd >= 0) { + close(fctxi->video_fd); + fctxi->video_fd = -1; + } + return ret; +} + +static int v4l2request_probe_video_devices(AVHWFramesContext *hwfc, + struct udev *udev, + uint32_t pixelformat, + uint32_t buffersize) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + struct media_device_info device_info; + struct media_v2_topology topology = {0}; + struct media_v2_interface *interfaces; + struct udev_device *device; + const char *path; + dev_t devnum; + int ret; + + if (ioctl(fctxi->media_fd, MEDIA_IOC_DEVICE_INFO, &device_info) < 0) + return AVERROR(errno); + + if (ioctl(fctxi->media_fd, MEDIA_IOC_G_TOPOLOGY, &topology) < 0) { + ret = AVERROR(errno); + av_log(hwfc, AV_LOG_ERROR, "Failed to get media topology: %s (%d)\n", + strerror(errno), errno); + return ret; + } + + if (!topology.num_interfaces) + return AVERROR(ENOENT); + + interfaces = av_calloc(topology.num_interfaces, sizeof(struct media_v2_interface)); + if (!interfaces) + return AVERROR(ENOMEM); + + topology.ptr_interfaces = (__u64)(uintptr_t)interfaces; + if (ioctl(fctxi->media_fd, MEDIA_IOC_G_TOPOLOGY, &topology) < 0) { + ret = AVERROR(errno); + av_log(hwfc, AV_LOG_ERROR, "Failed to get media topology: %s (%d)\n", + strerror(errno), errno); + goto fail; + } + + ret = AVERROR(ENOENT); + for (int i = 0; i < topology.num_interfaces; i++) { + if (interfaces[i].intf_type != MEDIA_INTF_T_V4L_VIDEO) + continue; + + devnum = makedev(interfaces[i].devnode.major, interfaces[i].devnode.minor); + device = udev_device_new_from_devnum(udev, 'c', devnum); + if (!device) + continue; + + path = udev_device_get_devnode(device); + if (path) + ret = v4l2request_probe_video_device(hwfc, path, pixelformat, buffersize); + udev_device_unref(device); + + // Stop when we have found a capable video device + if (!ret) { + av_log(hwfc, AV_LOG_INFO, + "Using V4L2 media driver %s (%u.%u.%u) for %s\n", + device_info.driver, + device_info.driver_version >> 16, + (device_info.driver_version >> 8) & 0xff, + device_info.driver_version & 0xff, + av_fourcc2str(pixelformat)); + break; + } + } + +fail: + av_free(interfaces); + return ret; +} + +static int v4l2request_probe_media_device(AVHWFramesContext *hwfc, + struct udev_device *device, + uint32_t pixelformat, + uint32_t buffersize) +{ + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + AVV4L2RequestFramesContextInternal *fctxi = fctx->internal; + const char *path; + int ret; + + path = udev_device_get_devnode(device); + if (!path) + return AVERROR(ENODEV); + + // Open enumerated media device + fctxi->media_fd = open(path, O_RDWR); + if (fctxi->media_fd < 0) { + ret = AVERROR(errno); + av_log(hwfc, AV_LOG_ERROR, "Failed to open media device %s: %s (%d)\n", + path, strerror(errno), errno); + return ret; + } + + // Probe video devices of current media device + ret = v4l2request_probe_video_devices(hwfc, udev_device_get_udev(device), + pixelformat, buffersize); + + // Cleanup when no capable video device was found + if (ret < 0) { + close(fctxi->media_fd); + fctxi->media_fd = -1; + } + + return ret; +} + +static int v4l2request_probe_media_devices(AVHWFramesContext *hwfc, + struct udev *udev, + uint32_t pixelformat, + uint32_t buffersize) +{ + struct udev_enumerate *enumerate; + struct udev_list_entry *devices; + struct udev_list_entry *entry; + struct udev_device *device; + int ret; + + enumerate = udev_enumerate_new(udev); + if (!enumerate) + return AVERROR(ENOMEM); + + udev_enumerate_add_match_subsystem(enumerate, "media"); + udev_enumerate_scan_devices(enumerate); + devices = udev_enumerate_get_list_entry(enumerate); + + ret = AVERROR(ENOENT); + udev_list_entry_foreach(entry, devices) { + const char *path = udev_list_entry_get_name(entry); + if (!path) + continue; + + device = udev_device_new_from_syspath(udev, path); + if (!device) + continue; + + // Probe media device for a capable video device + ret = v4l2request_probe_media_device(hwfc, device, pixelformat, buffersize); + udev_device_unref(device); + + // Stop when we have found a capable media and video device + if (!ret) + break; + } + + udev_enumerate_unref(enumerate); + return ret; +} + static int v4l2request_open_decoder(AVHWFramesContext *hwfc) { AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + uint32_t buffersize; + struct udev *udev; + int ret; // Ensure codec pixelformat is set if (!fctx->pixelformat) return AVERROR(EINVAL); - // TODO: locate a decoder supporting the requested pixelformat + // FIXME: locate a decoder using hwdevice context decoders - return AVERROR(ENOSYS); + udev = udev_new(); + if (!udev) + return AVERROR(ENOMEM); + + buffersize = FFMAX(hwfc->width * hwfc->height * 3 / 2, 256 * 1024); + + // Probe all media devices (auto-detection) + ret = v4l2request_probe_media_devices(hwfc, udev, fctx->pixelformat, buffersize); + + udev_unref(udev); + return ret; } static AVBufferRef *v4l2request_v4l2_buffer_alloc(AVHWFramesContext *hwfc, -- 2.53.0 From d47e6fdf152bfc97948a2f6540af492e56eff4c9 Mon Sep 17 00:00:00 2001 From: Jonas Karlman Date: Tue, 6 Aug 2024 09:06:01 +0000 Subject: [PATCH 03/13] avcodec: Add common V4L2 Request API code Add initial helpers for supporting V4L2 Request API hwaccels. Basic flow for initialization follows the kernel Memory-to-memory Stateless Video Decoder Interface > Initialization [1]. In init a hw frame ctx is created and initialized to start a new V4L2 video decoding session, initial CAPTURE buffers is pre-allocated, OUTPUT buffers and request objects are allocated for a circular queue. In frame_params the codec pixel format and optional codec-specific extended controls is configured before the hw frame ctx is initialized. In uninit any pending request is flushed before resources are released. [1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#initialization Co-developed-by: Jernej Skrabec Signed-off-by: Jernej Skrabec Co-developed-by: Alex Bee Signed-off-by: Alex Bee Signed-off-by: Jonas Karlman --- libavcodec/Makefile | 2 + libavcodec/v4l2_request.c | 249 ++++++++++++++++++++++++++++++++++++++ libavcodec/v4l2_request.h | 64 ++++++++++ 3 files changed, 315 insertions(+) create mode 100644 libavcodec/v4l2_request.c create mode 100644 libavcodec/v4l2_request.h diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 153a9e3881..55069167ab 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -176,6 +176,7 @@ OBJS-$(CONFIG_VP3DSP) += vp3dsp.o OBJS-$(CONFIG_VP56DSP) += vp56dsp.o OBJS-$(CONFIG_VP8DSP) += vp8dsp.o OBJS-$(CONFIG_V4L2_M2M) += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o +OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o OBJS-$(CONFIG_WMA_FREQS) += wma_freqs.o OBJS-$(CONFIG_WMV2DSP) += wmv2dsp.o @@ -1291,6 +1292,7 @@ SKIPHEADERS-$(CONFIG_VDPAU) += vdpau.h vdpau_internal.h SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += videotoolbox.h vt_internal.h SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_video.h vulkan_encode.h vulkan_decode.h SKIPHEADERS-$(CONFIG_V4L2_M2M) += v4l2_buffers.h v4l2_context.h v4l2_m2m.h +SKIPHEADERS-$(CONFIG_V4L2_REQUEST) += v4l2_request.h SKIPHEADERS-$(CONFIG_ZLIB) += zlib_wrapper.h TESTPROGS = avcodec \ diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c new file mode 100644 index 0000000000..dfec0e8906 --- /dev/null +++ b/libavcodec/v4l2_request.c @@ -0,0 +1,249 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "libavutil/hwcontext_v4l2request_internal.h" +#include "libavutil/mem.h" +#include "decode.h" +#include "internal.h" +#include "v4l2_request.h" + +static const AVClass v4l2_request_context_class = { + .class_name = "V4L2RequestContext", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, +}; + +static inline V4L2RequestContext *v4l2_request_context(AVCodecContext *avctx) +{ + return (V4L2RequestContext *)avctx->internal->hwaccel_priv_data; +} + +static void v4l2_request_output_buffer_uninit(V4L2RequestOutputBuffer *output) +{ + // Close the request associated with the OUTPUT buffer + if (output->request_fd >= 0) { + close(output->request_fd); + output->request_fd = -1; + } + + // Umap the OUTPUT buffer memory + if (output->addr) { + munmap(output->addr, output->size); + output->addr = NULL; + } + + // Return the OUTPUT buffer to the frames context OUTPUT pool + av_buffer_unref(&output->ref); +} + +static int v4l2_request_output_buffer_init(V4L2RequestContext *ctx, + V4L2RequestOutputBuffer *output) +{ + struct v4l2_format *format = &ctx->fctxi->output.format; + struct v4l2_buffer *buffer; + off_t offset; + void *addr; + int ret; + + // Get an OUTPUT buffer from frames context OUTPUT pool + output->ref = av_buffer_pool_get(ctx->fctxi->output.pool); + if (!output->ref) + return AVERROR(ENOMEM); + + buffer = (struct v4l2_buffer *)output->ref->data; + output->index = buffer->index; + output->size = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? + format->fmt.pix_mp.plane_fmt[0].sizeimage : + format->fmt.pix.sizeimage; + output->bytesused = 0; + + // Map the OUTPUT buffer memory, raw bitstream data is written into it + offset = V4L2_TYPE_IS_MULTIPLANAR(buffer->type) ? + buffer->m.planes[0].m.mem_offset : + buffer->m.offset; + addr = mmap(NULL, output->size, PROT_READ | PROT_WRITE, MAP_SHARED, + ctx->fctxi->video_fd, offset); + if (addr == MAP_FAILED) { + ret = AVERROR(errno); + av_log(ctx, AV_LOG_ERROR, "Failed to map OUTPUT buffer %d: %s (%d)\n", + output->index, strerror(errno), errno); + goto fail; + } + output->addr = addr; + + // Allocate and associated a request for the OUTPUT buffer + if (ioctl(ctx->fctxi->media_fd, MEDIA_IOC_REQUEST_ALLOC, &output->request_fd) < 0) { + ret = AVERROR(errno); + av_log(ctx, AV_LOG_ERROR, "Failed to allocate request for OUTPUT buffer %d: %s (%d)\n", + output->index, strerror(errno), errno); + goto fail; + } + + return 0; + +fail: + v4l2_request_output_buffer_uninit(output); + return ret; +} + +int ff_v4l2_request_frame_params(AVCodecContext *avctx, + AVBufferRef *hw_frames_ctx, + uint32_t pixelformat, + uint8_t bit_depth) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + AVHWFramesContext *hwfc = (AVHWFramesContext *)hw_frames_ctx->data; + AVV4L2RequestFramesContext *fctx = hwfc->hwctx; + + // Set parameters used during frames context initialization + fctx->pixelformat = pixelformat; + fctx->bit_depth = bit_depth; + if (ctx) { + fctx->init_controls = ctx->init_controls; + fctx->nb_init_controls = ctx->nb_init_controls; + } + + hwfc->format = AV_PIX_FMT_DRM_PRIME; + hwfc->sw_format = AV_PIX_FMT_NONE; + hwfc->width = avctx->coded_width; + hwfc->height = avctx->coded_height; + + // Pre-allocate CAPTURE buffers to ensure CAPTURE queue can be started + hwfc->initial_pool_size = 1; + + return 0; +} + +int ff_v4l2_request_uninit(AVCodecContext *avctx) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + enum v4l2_buf_type type; + + if (ctx->fctxi) { + // TODO: Flush and wait on all pending requests + + // Stop streaming on OUTPUT queue + type = ctx->fctxi->output.format.type; + if (ioctl(ctx->fctxi->video_fd, VIDIOC_STREAMOFF, &type) < 0) + av_log(ctx, AV_LOG_WARNING, "Failed to stop OUTPUT streaming: %s (%d)\n", + strerror(errno), errno); + + // Stop streaming on CAPTURE queue + type = ctx->fctxi->capture.format.type; + if (ioctl(ctx->fctxi->video_fd, VIDIOC_STREAMOFF, &type) < 0) + av_log(ctx, AV_LOG_WARNING, "Failed to stop CAPTURE streaming: %s (%d)\n", + strerror(errno), errno); + + // Release OUTPUT buffers and requests + for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) + v4l2_request_output_buffer_uninit(&ctx->output[i]); + + ctx->fctxi = NULL; + } + + av_buffer_unref(&ctx->frames_ref); + ff_mutex_destroy(&ctx->mutex); + + return 0; +} + +int ff_v4l2_request_init(AVCodecContext *avctx, + struct v4l2_ext_control *control, int count, + int (*post_frames_ctx)(AVCodecContext *avctx)) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + AVHWFramesContext *hwfc; + AVV4L2RequestFramesContext *fctx; + enum v4l2_buf_type type; + int ret; + + // Set initial default values + ctx->av_class = &v4l2_request_context_class; + ctx->init_controls = control; + ctx->nb_init_controls = count; + ff_mutex_init(&ctx->mutex, NULL); + for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) { + ctx->output[i].index = i; + ctx->output[i].request_fd = -1; + } + + // Create frames context and allocate initial CAPTURE buffers + ret = ff_decode_get_hw_frames_ctx(avctx, AV_HWDEVICE_TYPE_V4L2REQUEST); + if (ret < 0) + goto fail; + + ctx->frames_ref = av_buffer_ref(avctx->hw_frames_ctx); + if (!ctx->frames_ref) { + ret = AVERROR(ENOMEM); + goto fail; + } + + // Get internal hwctx from frames context + hwfc = (AVHWFramesContext *)ctx->frames_ref->data; + fctx = hwfc->hwctx; + ctx->fctxi = fctx->internal; + + // Reset init controls after successful frames context initialization + ctx->init_controls = NULL; + ctx->nb_init_controls = 0; + + // Check codec-specific controls, e.g. profile and level + if (post_frames_ctx) { + ret = post_frames_ctx(avctx); + if (ret < 0) + goto fail; + } + + // Allocate OUTPUT buffers and requests for circular queue + for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) { + ret = v4l2_request_output_buffer_init(ctx, &ctx->output[i]); + if (ret < 0) + goto fail; + } + + // Start streaming on OUTPUT queue + type = ctx->fctxi->output.format.type; + if (ioctl(ctx->fctxi->video_fd, VIDIOC_STREAMON, &type) < 0) { + ret = AVERROR(errno); + av_log(ctx, AV_LOG_ERROR, "Failed to start OUTPUT streaming: %s (%d)\n", + strerror(errno), errno); + goto fail; + } + + // Start streaming on CAPTURE queue + type = ctx->fctxi->capture.format.type; + if (ioctl(ctx->fctxi->video_fd, VIDIOC_STREAMON, &type) < 0) { + ret = AVERROR(errno); + av_log(ctx, AV_LOG_ERROR, "Failed to start CAPTURE streaming: %s (%d)\n", + strerror(errno), errno); + goto fail; + } + + return 0; + +fail: + ff_v4l2_request_uninit(avctx); + return ret; +} diff --git a/libavcodec/v4l2_request.h b/libavcodec/v4l2_request.h new file mode 100644 index 0000000000..d09bbf7344 --- /dev/null +++ b/libavcodec/v4l2_request.h @@ -0,0 +1,64 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_V4L2_REQUEST_H +#define AVCODEC_V4L2_REQUEST_H + +#include +#include +#include + +#include "libavutil/buffer.h" +#include "libavutil/log.h" +#include "libavutil/thread.h" +#include "avcodec.h" + +typedef struct AVV4L2RequestFramesContextInternal AVV4L2RequestFramesContextInternal; + +typedef struct V4L2RequestOutputBuffer { + AVBufferRef *ref; + uint32_t index; + int request_fd; + uint8_t *addr; + uint32_t size; + uint32_t bytesused; + struct timeval timestamp; +} V4L2RequestOutputBuffer; + +typedef struct V4L2RequestContext { + const AVClass *av_class; + AVBufferRef *frames_ref; + AVV4L2RequestFramesContextInternal *fctxi; + AVMutex mutex; + V4L2RequestOutputBuffer output[4]; + struct v4l2_ext_control *init_controls; + int nb_init_controls; +} V4L2RequestContext; + +int ff_v4l2_request_frame_params(AVCodecContext *avctx, + AVBufferRef *hw_frames_ctx, + uint32_t pixelformat, + uint8_t bit_depth); + +int ff_v4l2_request_uninit(AVCodecContext *avctx); + +int ff_v4l2_request_init(AVCodecContext *avctx, + struct v4l2_ext_control *control, int count, + int (*post_frames_ctx)(AVCodecContext *avctx)); + +#endif /* AVCODEC_V4L2_REQUEST_H */ -- 2.53.0 From 190cc2137b4400a29052b59d7b1ca48f5a69dc32 Mon Sep 17 00:00:00 2001 From: Jonas Karlman Date: Tue, 6 Aug 2024 09:06:03 +0000 Subject: [PATCH 04/13] avcodec/v4l2request: Add common decode support for hwaccels Add common support for decoding using the V4L2 Request API. Basic flow for decoding follows the kernel Memory-to-memory Stateless Video Decoder Interface > Decoding [1]. A codec hwaccel typically handle decoding as follow: In start_frame next OUTPUT buffer and its related request object is picked from a circular queue and any codec specific CONTROLs are prepared. In decode_slice the slice bitstream data is appended to the OUTPUT buffer. In end_frame a CAPTURE buffer tied to the AVFrame is queued, it will be used as the decoding target by the driver / hw decoder. The prepared codec specific CONTROLs are queued as part of the request object. Finally, the request object is submitted for decoding. In post_process the AVFrame is only returned once the CAPTURE buffer tied to the AVFrame has been made available for dequeuing by the driver. For slice based hw decoders only the request for the final slice of the frame is submitted in end_frame, remaining is submitted in decode_slice. [1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#decoding Co-developed-by: Jernej Skrabec Signed-off-by: Jernej Skrabec Co-developed-by: Alex Bee Signed-off-by: Alex Bee Signed-off-by: Jonas Karlman --- configure | 3 +- libavcodec/hwconfig.h | 2 + libavcodec/v4l2_request.c | 535 +++++++++++++++++++++++++++++++++++++- libavcodec/v4l2_request.h | 41 +++ 4 files changed, 579 insertions(+), 2 deletions(-) diff --git a/configure b/configure index d63b8a5abf..f57c10bcb0 100755 --- a/configure +++ b/configure @@ -3153,7 +3153,7 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32" ffnvcodec_deps_any="libdl LoadLibrary" mediacodec_deps="android mediandk" nvdec_deps="ffnvcodec" -v4l2_request_deps="linux_media_h v4l2_timeval_to_ns libdrm libudev" +v4l2_request_deps="linux_media_h v4l2_timeval_to_ns v4l2_m2m_hold_capture_buf libdrm libudev" v4l2_request_suggest="libdrm libudev" vaapi_x11_deps="xlib_x11" videotoolbox_hwaccel_deps="videotoolbox pthreads" @@ -7224,6 +7224,7 @@ if enabled v4l2_m2m; then fi if enabled v4l2_request; then + check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns check_pkg_config libudev libudev libudev.h udev_new v4l2_request_extralibs="$libudev_extralibs" diff --git a/libavcodec/hwconfig.h b/libavcodec/hwconfig.h index ee29ca631d..159064a1f1 100644 --- a/libavcodec/hwconfig.h +++ b/libavcodec/hwconfig.h @@ -79,6 +79,8 @@ void ff_hwaccel_uninit(AVCodecContext *avctx); HW_CONFIG_HWACCEL(0, 0, 1, D3D11VA_VLD, NONE, ff_ ## codec ## _d3d11va_hwaccel) #define HWACCEL_D3D12VA(codec) \ HW_CONFIG_HWACCEL(1, 1, 0, D3D12, D3D12VA, ff_ ## codec ## _d3d12va_hwaccel) +#define HWACCEL_V4L2REQUEST(codec) \ + HW_CONFIG_HWACCEL(1, 0, 0, DRM_PRIME, V4L2REQUEST, ff_ ## codec ## _v4l2request_hwaccel) #define HW_CONFIG_ENCODER(device, frames, ad_hoc, format, device_type_) \ &(const AVCodecHWConfigInternal) { \ diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c index dfec0e8906..7d12c0ee65 100644 --- a/libavcodec/v4l2_request.c +++ b/libavcodec/v4l2_request.c @@ -19,6 +19,7 @@ #include "config.h" #include +#include #include #include #include @@ -40,6 +41,537 @@ static inline V4L2RequestContext *v4l2_request_context(AVCodecContext *avctx) return (V4L2RequestContext *)avctx->internal->hwaccel_priv_data; } +static inline uint32_t v4l2_request_frameindex(AVFrame *frame) +{ + return (uint32_t)(uintptr_t)frame->data[1]; +} + +uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame) +{ + /* + * The CAPTURE buffer index is used as a base for V4L2 frame reference. + * This works because frames are decoded into a CAPTURE buffer that is + * closely tied to an AVFrame. + */ + struct timeval timestamp = { + .tv_sec = 0, + .tv_usec = v4l2_request_frameindex(frame) + 1, + }; + return v4l2_timeval_to_ns(×tamp); +} + +int ff_v4l2_request_query_control(AVCodecContext *avctx, + struct v4l2_query_ext_ctrl *control) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + + if (ioctl(ctx->fctxi->video_fd, VIDIOC_QUERY_EXT_CTRL, control) < 0) { + int ret = AVERROR(errno); + // Skip error logging when driver does not support control id (EINVAL) + if (errno != EINVAL) + av_log(ctx, AV_LOG_ERROR, "Failed to query control %u: %s (%d)\n", + control->id, strerror(errno), errno); + return ret; + } + + return 0; +} + +int ff_v4l2_request_query_control_default_value(AVCodecContext *avctx, + uint32_t id) +{ + struct v4l2_query_ext_ctrl control = { + .id = id, + }; + int ret; + + ret = ff_v4l2_request_query_control(avctx, &control); + if (ret < 0) + return ret; + + return control.default_value; +} + +static int v4l2_request_set_controls(V4L2RequestContext *ctx, int request_fd, + struct v4l2_ext_control *control, int count) +{ + struct v4l2_ext_controls controls = { + .controls = control, + .count = count, + .request_fd = request_fd, + .which = (request_fd >= 0) ? V4L2_CTRL_WHICH_REQUEST_VAL : 0, + }; + + if (!control || !count) + return 0; + + if (ioctl(ctx->fctxi->video_fd, VIDIOC_S_EXT_CTRLS, &controls) < 0) + return AVERROR(errno); + + return 0; +} + +int ff_v4l2_request_set_controls(AVCodecContext *avctx, + struct v4l2_ext_control *control, int count) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + int ret; + + ret = v4l2_request_set_controls(ctx, -1, control, count); + if (ret < 0) + av_log(ctx, AV_LOG_ERROR, "Failed to set %d control(s): %s (%d)\n", + count, strerror(errno), errno); + + return ret; +} + +static int v4l2_request_queue_buffer(V4L2RequestContext *ctx, + struct v4l2_buffer *buffer) +{ + struct v4l2_plane planes[1] = {}; + + if (V4L2_TYPE_IS_MULTIPLANAR(buffer->type)) { + planes[0].bytesused = buffer->bytesused; + buffer->bytesused = 0; + buffer->length = 1; + buffer->m.planes = planes; + } + + // Queue the buffer + if (ioctl(ctx->fctxi->video_fd, VIDIOC_QBUF, buffer) < 0) + return AVERROR(errno); + + // Mark the buffer as queued + if (V4L2_TYPE_IS_OUTPUT(buffer->type)) + ctx->queued_output |= 1 << buffer->index; + else + ctx->queued_capture |= 1 << buffer->index; + + return 0; +} + +static int v4l2_request_queue_capture_buffer(V4L2RequestContext *ctx, + uint32_t index) +{ + struct v4l2_buffer buffer = { + .index = index, + .type = ctx->fctxi->capture.format.type, + .memory = V4L2_MEMORY_MMAP, + }; + return v4l2_request_queue_buffer(ctx, &buffer); +} + +static int v4l2_request_queue_output_buffer(V4L2RequestContext *ctx, + V4L2RequestOutputBuffer *output, + uint32_t flags) +{ + struct v4l2_buffer buffer = { + .index = output->index, + .type = ctx->fctxi->output.format.type, + .memory = V4L2_MEMORY_MMAP, + .timestamp = output->timestamp, + .bytesused = output->bytesused, + .request_fd = output->request_fd, + .flags = V4L2_BUF_FLAG_REQUEST_FD | flags, + }; + return v4l2_request_queue_buffer(ctx, &buffer); +} + +static int v4l2_request_dequeue_buffer(V4L2RequestContext *ctx, + enum v4l2_buf_type type) +{ + struct v4l2_plane planes[1] = {}; + struct v4l2_buffer buffer = { + .type = type, + .memory = V4L2_MEMORY_MMAP, + }; + + if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) { + buffer.length = 1; + buffer.m.planes = planes; + } + + // Dequeue next completed buffer + if (ioctl(ctx->fctxi->video_fd, VIDIOC_DQBUF, &buffer) < 0) + return AVERROR(errno); + + // Mark the buffer as dequeued + if (V4L2_TYPE_IS_OUTPUT(buffer.type)) + ctx->queued_output &= ~(1 << buffer.index); + else + ctx->queued_capture &= ~(1 << buffer.index); + + return 0; +} + +static inline int v4l2_request_dequeue_completed_buffers(V4L2RequestContext *ctx, + enum v4l2_buf_type type) +{ + int ret; + + do { + ret = v4l2_request_dequeue_buffer(ctx, type); + } while (!ret); + + return ret; +} + +static int v4l2_request_wait_on_capture(V4L2RequestContext *ctx, uint32_t index) +{ + enum v4l2_buf_type type = ctx->fctxi->capture.format.type; + struct pollfd pollfd = { + .fd = ctx->fctxi->video_fd, + .events = POLLIN, + }; + + ff_mutex_lock(&ctx->mutex); + + // Dequeue all completed CAPTURE buffers + if (ctx->queued_capture) + v4l2_request_dequeue_completed_buffers(ctx, type); + + // Wait on the specific CAPTURE buffer + while (ctx->queued_capture & (1 << index)) { + int ret = poll(&pollfd, 1, 2000); + if (ret <= 0) + goto fail; + + ret = v4l2_request_dequeue_buffer(ctx, type); + if (ret < 0 && ret != AVERROR(EAGAIN)) + goto fail; + } + + ff_mutex_unlock(&ctx->mutex); + return 0; + +fail: + ff_mutex_unlock(&ctx->mutex); + av_log(ctx, AV_LOG_ERROR, "Failed waiting on CAPTURE buffer %d\n", index); + return AVERROR(EINVAL); +} + +static V4L2RequestOutputBuffer *v4l2_request_next_output(V4L2RequestContext *ctx) +{ + enum v4l2_buf_type type = ctx->fctxi->output.format.type; + V4L2RequestOutputBuffer *output; + struct pollfd pollfd = { + .fd = ctx->fctxi->video_fd, + .events = POLLOUT, + }; + uint8_t index; + + ff_mutex_lock(&ctx->mutex); + + // Use next OUTPUT buffer in the circular queue + index = ctx->next_output; + output = &ctx->output[index]; + ctx->next_output = (index + 1) % FF_ARRAY_ELEMS(ctx->output); + + // Dequeue all completed OUTPUT buffers + if (ctx->queued_output) + v4l2_request_dequeue_completed_buffers(ctx, type); + + // Wait on the specific OUTPUT buffer + while (ctx->queued_output & (1 << output->index)) { + int ret = poll(&pollfd, 1, 2000); + if (ret <= 0) + goto fail; + + ret = v4l2_request_dequeue_buffer(ctx, type); + if (ret < 0 && ret != AVERROR(EAGAIN)) + goto fail; + } + + ff_mutex_unlock(&ctx->mutex); + + // Reset bytesused state + output->bytesused = 0; + + return output; + +fail: + ff_mutex_unlock(&ctx->mutex); + av_log(ctx, AV_LOG_ERROR, "Failed waiting on OUTPUT buffer %d\n", + output->index); + return NULL; +} + +static int v4l2_request_wait_on_request(V4L2RequestContext *ctx, + V4L2RequestOutputBuffer *output) +{ + struct pollfd pollfd = { + .fd = output->request_fd, + .events = POLLPRI, + }; + + // Wait on the specific request to complete + while (ctx->queued_request & (1 << output->index)) { + int ret = poll(&pollfd, 1, 2000); + if (ret <= 0) + break; + + // Mark request as dequeued + if (pollfd.revents & (POLLPRI | POLLERR)) { + ctx->queued_request &= ~(1 << output->index); + break; + } + } + + // Reinit the request object + if (ioctl(output->request_fd, MEDIA_REQUEST_IOC_REINIT) < 0) { + int ret = AVERROR(errno); + av_log(ctx, AV_LOG_ERROR, "Failed to reinit request object %d: %s (%d)\n", + output->request_fd, strerror(errno), errno); + return ret; + } + + // Ensure request is marked as dequeued + ctx->queued_request &= ~(1 << output->index); + + return 0; +} + +int ff_v4l2_request_append_output(AVCodecContext *avctx, + V4L2RequestPictureContext *pic, + const uint8_t *data, uint32_t size) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + + // Append data to OUTPUT buffer and ensure there is enough space for padding + if (pic->output->bytesused + size + AV_INPUT_BUFFER_PADDING_SIZE <= pic->output->size) { + memcpy(pic->output->addr + pic->output->bytesused, data, size); + pic->output->bytesused += size; + return 0; + } else { + av_log(ctx, AV_LOG_ERROR, + "Failed to append %u bytes data to OUTPUT buffer %d (%u of %u used)\n", + size, pic->output->index, pic->output->bytesused, pic->output->size); + return AVERROR(ENOMEM); + } +} + +static int v4l2_request_queue_decode(AVCodecContext *avctx, + V4L2RequestPictureContext *pic, + struct v4l2_ext_control *control, int count, + bool first_slice, bool last_slice) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + uint32_t flags; + int ret; + + if (first_slice) { + /* + * Wait on dequeue of the target CAPTURE buffer. Otherwise V4L2 decoder + * may use a different CAPTURE buffer than hwaccel expects. + * + * Normally decoding has already completed when a CAPTURE buffer is + * reused so this is more or less a no-op, however in some situations + * FFmpeg may reuse an AVFrame early, i.e. when no output frame was + * produced prior time, and a synchronization is necessary. + */ + ret = v4l2_request_wait_on_capture(ctx, pic->capture_index); + if (ret < 0) + return ret; + } + + ff_mutex_lock(&ctx->mutex); + + /* + * The OUTPUT buffer tied to prior use of current request object can + * independently be dequeued before the full decode request has been + * completed. This may happen when a decoder use multi stage decoding, + * e.g. rpi-hevc-dec. In such case we can start reusing the OUTPUT buffer, + * however we must wait on the prior request to fully complete before we + * can reuse the request object, and a synchronization is necessary. + */ + ret = v4l2_request_wait_on_request(ctx, pic->output); + if (ret < 0) + goto fail; + + /* + * Dequeue any completed OUTPUT buffers, this is strictly not necessary, + * however if a synchronization was necessary for the CAPTURE and/or request + * there is more than likely one or more OUTPUT buffers that can be dequeued. + */ + if (ctx->queued_output) + v4l2_request_dequeue_completed_buffers(ctx, ctx->fctxi->output.format.type); + + // Set codec controls for current request + ret = v4l2_request_set_controls(ctx, pic->output->request_fd, control, count); + if (ret < 0) { + av_log(ctx, AV_LOG_ERROR, "Failed to set %d control(s) for request %d: %s (%d)\n", + count, pic->output->request_fd, strerror(errno), errno); + goto fail; + } + + // Ensure there is zero padding at the end of bitstream data + memset(pic->output->addr + pic->output->bytesused, 0, AV_INPUT_BUFFER_PADDING_SIZE); + + /* + * Use CAPTURE buffer index as base for V4L2 frame reference. + * This works because a CAPTURE buffer is closely tied to a AVFrame + * and FFmpeg handle all frame reference tracking for us. + */ + pic->output->timestamp = (struct timeval) { + .tv_sec = 0, + .tv_usec = pic->capture_index + 1, + }; + + /* + * Queue the OUTPUT buffer of current request. The CAPTURE buffer may be + * hold by the V4L2 decoder unless this is the last slice of a frame. + */ + flags = last_slice ? 0 : V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF; + ret = v4l2_request_queue_output_buffer(ctx, pic->output, flags); + if (ret < 0) { + av_log(ctx, AV_LOG_ERROR, "Failed to queue OUTPUT buffer %d for request %d: %s (%d)\n", + pic->output->index, pic->output->request_fd, strerror(errno), errno); + goto fail; + } + + if (first_slice) { + /* + * Queue the target CAPTURE buffer, hwaccel expect and depend on that + * this specific CAPTURE buffer will be used as decode target for + * current request, otherwise frames may be output in wrong order or + * wrong CAPTURE buffer could get used as a reference frame. + */ + ret = v4l2_request_queue_capture_buffer(ctx, pic->capture_index); + if (ret < 0) { + av_log(ctx, AV_LOG_ERROR, "Failed to queue CAPTURE buffer %d for request %d: %s (%d)\n", + pic->capture_index, pic->output->request_fd, strerror(errno), errno); + goto fail; + } + } + + // Queue current request + ret = ioctl(pic->output->request_fd, MEDIA_REQUEST_IOC_QUEUE); + if (ret < 0) { + ret = AVERROR(errno); + av_log(ctx, AV_LOG_ERROR, "Failed to queue request object %d: %s (%d)\n", + pic->output->request_fd, strerror(errno), errno); + goto fail; + } + + // Mark current request as queued + ctx->queued_request |= 1 << pic->output->index; + + ret = 0; +fail: + ff_mutex_unlock(&ctx->mutex); + return ret; +} + +int ff_v4l2_request_decode_slice(AVCodecContext *avctx, + V4L2RequestPictureContext *pic, + struct v4l2_ext_control *control, int count, + bool first_slice, bool last_slice) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + + /* + * Fallback to queue each slice as a full frame when holding CAPTURE + * buffers is not supported by the driver. + */ + if ((ctx->fctxi->output.capabilities & V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF) != + V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF) + return v4l2_request_queue_decode(avctx, pic, control, count, true, true); + + return v4l2_request_queue_decode(avctx, pic, control, count, + first_slice, last_slice); +} + +int ff_v4l2_request_decode_frame(AVCodecContext *avctx, + V4L2RequestPictureContext *pic, + struct v4l2_ext_control *control, int count) +{ + return v4l2_request_queue_decode(avctx, pic, control, count, true, true); +} + +static int v4l2_request_post_process(void *logctx, AVFrame *frame) +{ + uint32_t index = v4l2_request_frameindex(frame); + FrameDecodeData *fdd = (FrameDecodeData *)frame->private_ref->data; + V4L2RequestContext *ctx = fdd->hwaccel_priv; + + // Wait on CAPTURE buffer before returning the frame to application + return v4l2_request_wait_on_capture(ctx, index); +} + +int ff_v4l2_request_reset_picture(AVCodecContext *avctx, V4L2RequestPictureContext *pic) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + + // Get and wait on next OUTPUT buffer from circular queue + pic->output = v4l2_request_next_output(ctx); + if (!pic->output) + return AVERROR(EINVAL); + + return 0; +} + +int ff_v4l2_request_start_frame(AVCodecContext *avctx, + V4L2RequestPictureContext *pic, + AVFrame *frame) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + uint32_t index = v4l2_request_frameindex(frame); + FrameDecodeData *fdd = (FrameDecodeData *)frame->private_ref->data; + int ret; + + // Get next OUTPUT buffer from circular queue + ret = ff_v4l2_request_reset_picture(avctx, pic); + if (ret) + return ret; + + // Ensure CAPTURE buffer is dequeued before reuse + ret = v4l2_request_wait_on_capture(ctx, index); + if (ret) + return ret; + + // Wait on CAPTURE buffer in post_process() before returning to application + fdd->hwaccel_priv = ctx; + fdd->post_process = v4l2_request_post_process; + + // CAPTURE buffer used for current frame + pic->capture_index = index; + + return 0; +} + +void ff_v4l2_request_flush(AVCodecContext *avctx) +{ + V4L2RequestContext *ctx = v4l2_request_context(avctx); + enum v4l2_buf_type type = ctx->fctxi->output.format.type; + struct pollfd pollfd = { + .fd = ctx->fctxi->video_fd, + .events = POLLOUT, + }; + + ff_mutex_lock(&ctx->mutex); + + // Dequeue all completed OUTPUT buffers + if (ctx->queued_output) + v4l2_request_dequeue_completed_buffers(ctx, type); + + // Wait on any remaining OUTPUT buffer + while (ctx->queued_output) { + int ret = poll(&pollfd, 1, 2000); + if (ret <= 0) + break; + + ret = v4l2_request_dequeue_buffer(ctx, type); + if (ret < 0 && ret != AVERROR(EAGAIN)) + break; + } + + // Dequeue all completed CAPTURE buffers + if (ctx->queued_capture) + v4l2_request_dequeue_completed_buffers(ctx, ctx->fctxi->capture.format.type); + + ff_mutex_unlock(&ctx->mutex); +} + static void v4l2_request_output_buffer_uninit(V4L2RequestOutputBuffer *output) { // Close the request associated with the OUTPUT buffer @@ -142,7 +674,8 @@ int ff_v4l2_request_uninit(AVCodecContext *avctx) enum v4l2_buf_type type; if (ctx->fctxi) { - // TODO: Flush and wait on all pending requests + // Flush and wait on all pending requests + ff_v4l2_request_flush(avctx); // Stop streaming on OUTPUT queue type = ctx->fctxi->output.format.type; diff --git a/libavcodec/v4l2_request.h b/libavcodec/v4l2_request.h index d09bbf7344..53759f20bb 100644 --- a/libavcodec/v4l2_request.h +++ b/libavcodec/v4l2_request.h @@ -46,10 +46,51 @@ typedef struct V4L2RequestContext { AVV4L2RequestFramesContextInternal *fctxi; AVMutex mutex; V4L2RequestOutputBuffer output[4]; + uint8_t next_output; + uint32_t queued_output; + uint32_t queued_request; + uint64_t queued_capture; struct v4l2_ext_control *init_controls; int nb_init_controls; } V4L2RequestContext; +typedef struct V4L2RequestPictureContext { + V4L2RequestOutputBuffer *output; + uint32_t capture_index; +} V4L2RequestPictureContext; + +uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame); + +int ff_v4l2_request_query_control(AVCodecContext *avctx, + struct v4l2_query_ext_ctrl *control); + +int ff_v4l2_request_query_control_default_value(AVCodecContext *avctx, + uint32_t id); + +int ff_v4l2_request_set_controls(AVCodecContext *avctx, + struct v4l2_ext_control *control, int count); + +int ff_v4l2_request_append_output(AVCodecContext *avctx, + V4L2RequestPictureContext *pic, + const uint8_t *data, uint32_t size); + +int ff_v4l2_request_decode_slice(AVCodecContext *avctx, + V4L2RequestPictureContext *pic, + struct v4l2_ext_control *control, int count, + bool first_slice, bool last_slice); + +int ff_v4l2_request_decode_frame(AVCodecContext *avctx, + V4L2RequestPictureContext *pic, + struct v4l2_ext_control *control, int count); + +int ff_v4l2_request_reset_picture(AVCodecContext *avctx, + V4L2RequestPictureContext *pic); + +int ff_v4l2_request_start_frame(AVCodecContext *avctx, + V4L2RequestPictureContext *pic, AVFrame *frame); + +void ff_v4l2_request_flush(AVCodecContext *avctx); + int ff_v4l2_request_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx, uint32_t pixelformat, -- 2.53.0 From 44d96f760674b4e62dcbf52864057d829aae7eb9 Mon Sep 17 00:00:00 2001 From: Jonas Karlman Date: Tue, 6 Aug 2024 09:06:04 +0000 Subject: [PATCH 05/13] avcodec: Add V4L2 Request API mpeg2 hwaccel Add a V4L2 Request API hwaccel for MPEG2. Support for MPEG2 is enabled when Linux kernel headers declare the control id V4L2_CID_STATELESS_MPEG2_SEQUENCE, added in v5.14. This also change v4l2_request hwaccel to use autodetect in configure. Signed-off-by: Jonas Karlman --- configure | 7 +- libavcodec/Makefile | 1 + libavcodec/hwaccels.h | 1 + libavcodec/mpeg12dec.c | 6 ++ libavcodec/v4l2_request_mpeg2.c | 183 ++++++++++++++++++++++++++++++++ 5 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 libavcodec/v4l2_request_mpeg2.c diff --git a/configure b/configure index f57c10bcb0..01daad1e66 100755 --- a/configure +++ b/configure @@ -359,7 +359,7 @@ External library support: --enable-omx-rpi enable OpenMAX IL code for Raspberry Pi [no] --enable-rkmpp enable Rockchip Media Process Platform code [no] --disable-v4l2-m2m disable V4L2 mem2mem code [autodetect] - --enable-v4l2-request enable V4L2 Request API code [no] + --disable-v4l2-request disable V4L2 Request API code [autodetect] --disable-vaapi disable Video Acceleration API (mainly Unix/Intel) code [autodetect] --disable-vdpau disable Nvidia Video Decode and Presentation API for Unix code [autodetect] --disable-videotoolbox disable VideoToolbox code [autodetect] @@ -2008,6 +2008,7 @@ HWACCEL_AUTODETECT_LIBRARY_LIST=" videotoolbox vulkan v4l2_m2m + v4l2_request " # catchall list of things that require external libs to link @@ -2029,7 +2030,6 @@ HWACCEL_LIBRARY_LIST=" mmal omx opencl - v4l2_request " DOCUMENT_LIST=" @@ -3237,6 +3237,8 @@ mpeg2_dxva2_hwaccel_deps="dxva2" mpeg2_dxva2_hwaccel_select="mpeg2video_decoder" mpeg2_nvdec_hwaccel_deps="nvdec" mpeg2_nvdec_hwaccel_select="mpeg2video_decoder" +mpeg2_v4l2request_hwaccel_deps="v4l2_request mpeg2_v4l2_request" +mpeg2_v4l2request_hwaccel_select="mpeg2video_decoder" mpeg2_vaapi_hwaccel_deps="vaapi" mpeg2_vaapi_hwaccel_select="mpeg2video_decoder" mpeg2_vdpau_hwaccel_deps="vdpau" @@ -7224,6 +7226,7 @@ if enabled v4l2_m2m; then fi if enabled v4l2_request; then + check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns check_pkg_config libudev libudev libudev.h udev_new diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 55069167ab..6017380587 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1039,6 +1039,7 @@ OBJS-$(CONFIG_MPEG2_DXVA2_HWACCEL) += dxva2_mpeg2.o OBJS-$(CONFIG_MPEG2_D3D12VA_HWACCEL) += dxva2_mpeg2.o d3d12va_mpeg2.o OBJS-$(CONFIG_MPEG2_NVDEC_HWACCEL) += nvdec_mpeg12.o OBJS-$(CONFIG_MPEG2_QSV_HWACCEL) += qsvdec.o +OBJS-$(CONFIG_MPEG2_V4L2REQUEST_HWACCEL) += v4l2_request_mpeg2.o OBJS-$(CONFIG_MPEG2_VAAPI_HWACCEL) += vaapi_mpeg2.o OBJS-$(CONFIG_MPEG2_VDPAU_HWACCEL) += vdpau_mpeg12.o OBJS-$(CONFIG_MPEG2_VIDEOTOOLBOX_HWACCEL) += videotoolbox.o diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h index 5171e4c7d7..0cba7c71be 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h @@ -57,6 +57,7 @@ extern const struct FFHWAccel ff_mpeg2_d3d11va2_hwaccel; extern const struct FFHWAccel ff_mpeg2_d3d12va_hwaccel; extern const struct FFHWAccel ff_mpeg2_dxva2_hwaccel; extern const struct FFHWAccel ff_mpeg2_nvdec_hwaccel; +extern const struct FFHWAccel ff_mpeg2_v4l2request_hwaccel; extern const struct FFHWAccel ff_mpeg2_vaapi_hwaccel; extern const struct FFHWAccel ff_mpeg2_vdpau_hwaccel; extern const struct FFHWAccel ff_mpeg2_videotoolbox_hwaccel; diff --git a/libavcodec/mpeg12dec.c b/libavcodec/mpeg12dec.c index 4f784611de..af8ffa5976 100644 --- a/libavcodec/mpeg12dec.c +++ b/libavcodec/mpeg12dec.c @@ -839,6 +839,9 @@ static const enum AVPixelFormat mpeg2_hwaccel_pixfmt_list_420[] = { #endif #if CONFIG_MPEG2_VIDEOTOOLBOX_HWACCEL AV_PIX_FMT_VIDEOTOOLBOX, +#endif +#if CONFIG_MPEG2_V4L2REQUEST_HWACCEL + AV_PIX_FMT_DRM_PRIME, #endif AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE @@ -2681,6 +2684,9 @@ const FFCodec ff_mpeg2video_decoder = { #endif #if CONFIG_MPEG2_VIDEOTOOLBOX_HWACCEL HWACCEL_VIDEOTOOLBOX(mpeg2), +#endif +#if CONFIG_MPEG2_V4L2REQUEST_HWACCEL + HWACCEL_V4L2REQUEST(mpeg2), #endif NULL }, diff --git a/libavcodec/v4l2_request_mpeg2.c b/libavcodec/v4l2_request_mpeg2.c new file mode 100644 index 0000000000..6ebbfb914a --- /dev/null +++ b/libavcodec/v4l2_request_mpeg2.c @@ -0,0 +1,183 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include "hwaccel_internal.h" +#include "hwconfig.h" +#include "mathops.h" +#include "mpegvideo.h" +#include "v4l2_request.h" + +typedef struct V4L2RequestControlsMPEG2 { + V4L2RequestPictureContext pic; + struct v4l2_ctrl_mpeg2_sequence sequence; + struct v4l2_ctrl_mpeg2_picture picture; + struct v4l2_ctrl_mpeg2_quantisation quantisation; +} V4L2RequestControlsMPEG2; + +static int v4l2_request_mpeg2_start_frame(AVCodecContext *avctx, + av_unused const uint8_t *buffer, + av_unused uint32_t size) +{ + const MpegEncContext *s = avctx->priv_data; + V4L2RequestControlsMPEG2 *controls = s->cur_pic.ptr->hwaccel_picture_private; + int ret; + + ret = ff_v4l2_request_start_frame(avctx, &controls->pic, s->cur_pic.ptr->f); + if (ret) + return ret; + + controls->sequence = (struct v4l2_ctrl_mpeg2_sequence) { + /* ISO/IEC 13818-2, ITU-T Rec. H.262: Sequence header */ + .horizontal_size = s->width, + .vertical_size = s->height, + .vbv_buffer_size = controls->pic.output->size, + + /* ISO/IEC 13818-2, ITU-T Rec. H.262: Sequence extension */ + .profile_and_level_indication = 0, + .chroma_format = s->chroma_format, + }; + + if (s->progressive_sequence) + controls->sequence.flags |= V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE; + + controls->picture = (struct v4l2_ctrl_mpeg2_picture) { + /* ISO/IEC 13818-2, ITU-T Rec. H.262: Picture header */ + .picture_coding_type = s->pict_type, + + /* ISO/IEC 13818-2, ITU-T Rec. H.262: Picture coding extension */ + .f_code[0][0] = s->mpeg_f_code[0][0], + .f_code[0][1] = s->mpeg_f_code[0][1], + .f_code[1][0] = s->mpeg_f_code[1][0], + .f_code[1][1] = s->mpeg_f_code[1][1], + .picture_structure = s->picture_structure, + .intra_dc_precision = s->intra_dc_precision, + }; + + if (s->top_field_first) + controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST; + + if (s->frame_pred_frame_dct) + controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT; + + if (s->concealment_motion_vectors) + controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV; + + if (s->intra_vlc_format) + controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_INTRA_VLC; + + if (s->q_scale_type) + controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE; + + if (s->alternate_scan) + controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_ALT_SCAN; + + if (s->repeat_first_field) + controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_REPEAT_FIRST; + + if (s->progressive_frame) + controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_PROGRESSIVE; + + switch (s->pict_type) { + case AV_PICTURE_TYPE_B: + if (s->next_pic.ptr) + controls->picture.backward_ref_ts = + ff_v4l2_request_get_capture_timestamp(s->next_pic.ptr->f); + // fall-through + case AV_PICTURE_TYPE_P: + if (s->last_pic.ptr) + controls->picture.forward_ref_ts = + ff_v4l2_request_get_capture_timestamp(s->last_pic.ptr->f); + } + + for (int i = 0; i < 64; i++) { + int n = s->idsp.idct_permutation[ff_zigzag_direct[i]]; + controls->quantisation.intra_quantiser_matrix[i] = s->intra_matrix[n]; + controls->quantisation.non_intra_quantiser_matrix[i] = s->inter_matrix[n]; + controls->quantisation.chroma_intra_quantiser_matrix[i] = s->chroma_intra_matrix[n]; + controls->quantisation.chroma_non_intra_quantiser_matrix[i] = s->chroma_inter_matrix[n]; + } + + return 0; +} + +static int v4l2_request_mpeg2_decode_slice(AVCodecContext *avctx, + const uint8_t *buffer, uint32_t size) +{ + const MpegEncContext *s = avctx->priv_data; + V4L2RequestControlsMPEG2 *controls = s->cur_pic.ptr->hwaccel_picture_private; + + return ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); +} + +static int v4l2_request_mpeg2_end_frame(AVCodecContext *avctx) +{ + const MpegEncContext *s = avctx->priv_data; + V4L2RequestControlsMPEG2 *controls = s->cur_pic.ptr->hwaccel_picture_private; + + struct v4l2_ext_control control[] = { + { + .id = V4L2_CID_STATELESS_MPEG2_SEQUENCE, + .ptr = &controls->sequence, + .size = sizeof(controls->sequence), + }, + { + .id = V4L2_CID_STATELESS_MPEG2_PICTURE, + .ptr = &controls->picture, + .size = sizeof(controls->picture), + }, + { + .id = V4L2_CID_STATELESS_MPEG2_QUANTISATION, + .ptr = &controls->quantisation, + .size = sizeof(controls->quantisation), + }, + }; + + return ff_v4l2_request_decode_frame(avctx, &controls->pic, + control, FF_ARRAY_ELEMS(control)); +} + +static int v4l2_request_mpeg2_init(AVCodecContext *avctx) +{ + return ff_v4l2_request_init(avctx, NULL, 0, NULL); +} + +static int v4l2_request_mpeg2_frame_params(AVCodecContext *avctx, + AVBufferRef *hw_frames_ctx) +{ + return ff_v4l2_request_frame_params(avctx, hw_frames_ctx, + V4L2_PIX_FMT_MPEG2_SLICE, 8); +} + +const FFHWAccel ff_mpeg2_v4l2request_hwaccel = { + .p.name = "mpeg2_v4l2request", + .p.type = AVMEDIA_TYPE_VIDEO, + .p.id = AV_CODEC_ID_MPEG2VIDEO, + .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, + .start_frame = v4l2_request_mpeg2_start_frame, + .decode_slice = v4l2_request_mpeg2_decode_slice, + .end_frame = v4l2_request_mpeg2_end_frame, + .flush = ff_v4l2_request_flush, + .frame_priv_data_size = sizeof(V4L2RequestControlsMPEG2), + .init = v4l2_request_mpeg2_init, + .uninit = ff_v4l2_request_uninit, + .priv_data_size = sizeof(V4L2RequestContext), + .frame_params = v4l2_request_mpeg2_frame_params, + .caps_internal = HWACCEL_CAP_ASYNC_SAFE, +}; -- 2.53.0 From a516e28ae65617b14d0d44247420633f4a61af97 Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Tue, 6 Aug 2024 09:06:05 +0000 Subject: [PATCH 06/13] avcodec/h264dec: add ref_pic_marking and pic_order_cnt bit_size to slice context The V4L2_CID_STATELESS_H264_DECODE_PARAMS control require following: - dec_ref_pic_marking_bit_size Size in bits of the dec_ref_pic_marking() syntax element. - pic_order_cnt_bit_size Combined size in bits of the picture order count related syntax elements: pic_order_cnt_lsb, delta_pic_order_cnt_bottom, delta_pic_order_cnt0, and delta_pic_order_cnt1. Save the bit sizes while parsing for later use in hwaccel, similar to short/long_term_ref_pic_set_size in hevcdec. Signed-off-by: Boris Brezillon Signed-off-by: Jonas Karlman --- libavcodec/h264_slice.c | 6 +++++- libavcodec/h264dec.h | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c index 01c43a652e..4ee61f2f7a 100644 --- a/libavcodec/h264_slice.c +++ b/libavcodec/h264_slice.c @@ -1700,7 +1700,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, unsigned int slice_type, tmp, i; int field_pic_flag, bottom_field_flag; int first_slice = sl == h->slice_ctx && !h->current_slice; - int picture_structure; + int picture_structure, pos; if (first_slice) av_assert0(!h->setup_finished); @@ -1791,6 +1791,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, sl->poc_lsb = 0; sl->delta_poc_bottom = 0; + pos = get_bits_left(&sl->gb); if (sps->poc_type == 0) { sl->poc_lsb = get_bits(&sl->gb, sps->log2_max_poc_lsb); @@ -1805,6 +1806,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, if (pps->pic_order_present == 1 && picture_structure == PICT_FRAME) sl->delta_poc[1] = get_se_golomb(&sl->gb); } + sl->pic_order_cnt_bit_size = pos - get_bits_left(&sl->gb); sl->redundant_pic_count = 0; if (pps->redundant_pic_cnt_present) @@ -1844,9 +1846,11 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, sl->explicit_ref_marking = 0; if (nal->ref_idc) { + pos = get_bits_left(&sl->gb); ret = ff_h264_decode_ref_pic_marking(sl, &sl->gb, nal, h->avctx); if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE)) return AVERROR_INVALIDDATA; + sl->ref_pic_marking_bit_size = pos - get_bits_left(&sl->gb); } if (sl->slice_type_nos != AV_PICTURE_TYPE_I && pps->cabac) { diff --git a/libavcodec/h264dec.h b/libavcodec/h264dec.h index ccd7583bf4..71cbb0e0cc 100644 --- a/libavcodec/h264dec.h +++ b/libavcodec/h264dec.h @@ -324,6 +324,7 @@ typedef struct H264SliceContext { MMCO mmco[H264_MAX_MMCO_COUNT]; int nb_mmco; int explicit_ref_marking; + int ref_pic_marking_bit_size; int frame_num; int idr_pic_id; @@ -332,6 +333,7 @@ typedef struct H264SliceContext { int delta_poc[2]; int curr_pic_num; int max_pic_num; + int pic_order_cnt_bit_size; } H264SliceContext; /** -- 2.53.0 From 201477325d1c51d166e0f85f187561920d768951 Mon Sep 17 00:00:00 2001 From: Jernej Skrabec Date: Tue, 6 Aug 2024 09:06:06 +0000 Subject: [PATCH 07/13] avcodec: Add V4L2 Request API h264 hwaccel Add a V4L2 Request API hwaccel for H.264, supporting both slice and frame decoding modes. Support for H.264 is enabled when Linux kernel headers declare the control id V4L2_CID_STATELESS_H264_DECODE_MODE, added in v5.11. Signed-off-by: Jernej Skrabec Co-developed-by: Jonas Karlman Signed-off-by: Jonas Karlman --- configure | 3 + libavcodec/Makefile | 1 + libavcodec/h264_slice.c | 7 + libavcodec/h264dec.c | 3 + libavcodec/hwaccels.h | 1 + libavcodec/v4l2_request_h264.c | 531 +++++++++++++++++++++++++++++++++ 6 files changed, 546 insertions(+) create mode 100644 libavcodec/v4l2_request_h264.c diff --git a/configure b/configure index 01daad1e66..28253b3e2b 100755 --- a/configure +++ b/configure @@ -3191,6 +3191,8 @@ h264_dxva2_hwaccel_deps="dxva2" h264_dxva2_hwaccel_select="h264_decoder" h264_nvdec_hwaccel_deps="nvdec" h264_nvdec_hwaccel_select="h264_decoder" +h264_v4l2request_hwaccel_deps="v4l2_request h264_v4l2_request" +h264_v4l2request_hwaccel_select="h264_decoder" h264_vaapi_hwaccel_deps="vaapi" h264_vaapi_hwaccel_select="h264_decoder" h264_vdpau_hwaccel_deps="vdpau" @@ -7226,6 +7228,7 @@ if enabled v4l2_m2m; then fi if enabled v4l2_request; then + check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_H264_DECODE_MODE" check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 6017380587..74303ad674 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1017,6 +1017,7 @@ OBJS-$(CONFIG_H264_DXVA2_HWACCEL) += dxva2_h264.o OBJS-$(CONFIG_H264_D3D12VA_HWACCEL) += dxva2_h264.o d3d12va_h264.o OBJS-$(CONFIG_H264_NVDEC_HWACCEL) += nvdec_h264.o OBJS-$(CONFIG_H264_QSV_HWACCEL) += qsvdec.o +OBJS-$(CONFIG_H264_V4L2REQUEST_HWACCEL) += v4l2_request_h264.o OBJS-$(CONFIG_H264_VAAPI_HWACCEL) += vaapi_h264.o OBJS-$(CONFIG_H264_VDPAU_HWACCEL) += vdpau_h264.o OBJS-$(CONFIG_H264_VIDEOTOOLBOX_HWACCEL) += videotoolbox.o diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c index 4ee61f2f7a..d36203234f 100644 --- a/libavcodec/h264_slice.c +++ b/libavcodec/h264_slice.c @@ -791,6 +791,7 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) (CONFIG_H264_D3D11VA_HWACCEL * 2) + \ CONFIG_H264_D3D12VA_HWACCEL + \ CONFIG_H264_NVDEC_HWACCEL + \ + CONFIG_H264_V4L2REQUEST_HWACCEL + \ CONFIG_H264_VAAPI_HWACCEL + \ CONFIG_H264_VIDEOTOOLBOX_HWACCEL + \ CONFIG_H264_VDPAU_HWACCEL + \ @@ -816,6 +817,9 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) #endif #if CONFIG_H264_VULKAN_HWACCEL *fmt++ = AV_PIX_FMT_VULKAN; +#endif +#if CONFIG_H264_V4L2REQUEST_HWACCEL + *fmt++ = AV_PIX_FMT_DRM_PRIME; #endif if (CHROMA444(h)) { if (h->avctx->colorspace == AVCOL_SPC_RGB) { @@ -872,6 +876,9 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) #if CONFIG_H264_VIDEOTOOLBOX_HWACCEL if (h->avctx->colorspace != AVCOL_SPC_RGB) *fmt++ = AV_PIX_FMT_VIDEOTOOLBOX; +#endif +#if CONFIG_H264_V4L2REQUEST_HWACCEL + *fmt++ = AV_PIX_FMT_DRM_PRIME; #endif if (CHROMA444(h)) { if (h->avctx->colorspace == AVCOL_SPC_RGB) diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c index 0154fe17b6..a5e89ebddd 100644 --- a/libavcodec/h264dec.c +++ b/libavcodec/h264dec.c @@ -1160,6 +1160,9 @@ const FFCodec ff_h264_decoder = { #endif #if CONFIG_H264_VULKAN_HWACCEL HWACCEL_VULKAN(h264), +#endif +#if CONFIG_H264_V4L2REQUEST_HWACCEL + HWACCEL_V4L2REQUEST(h264), #endif NULL }, diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h index 0cba7c71be..024e021d73 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h @@ -34,6 +34,7 @@ extern const struct FFHWAccel ff_h264_d3d11va2_hwaccel; extern const struct FFHWAccel ff_h264_d3d12va_hwaccel; extern const struct FFHWAccel ff_h264_dxva2_hwaccel; extern const struct FFHWAccel ff_h264_nvdec_hwaccel; +extern const struct FFHWAccel ff_h264_v4l2request_hwaccel; extern const struct FFHWAccel ff_h264_vaapi_hwaccel; extern const struct FFHWAccel ff_h264_vdpau_hwaccel; extern const struct FFHWAccel ff_h264_videotoolbox_hwaccel; diff --git a/libavcodec/v4l2_request_h264.c b/libavcodec/v4l2_request_h264.c new file mode 100644 index 0000000000..9d06e9c757 --- /dev/null +++ b/libavcodec/v4l2_request_h264.c @@ -0,0 +1,531 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include "h264dec.h" +#include "hwaccel_internal.h" +#include "hwconfig.h" +#include "internal.h" +#include "v4l2_request.h" + +typedef struct V4L2RequestContextH264 { + V4L2RequestContext base; + enum v4l2_stateless_h264_decode_mode decode_mode; + enum v4l2_stateless_h264_start_code start_code; +} V4L2RequestContextH264; + +typedef struct V4L2RequestControlsH264 { + V4L2RequestPictureContext pic; + struct v4l2_ctrl_h264_sps sps; + struct v4l2_ctrl_h264_pps pps; + struct v4l2_ctrl_h264_scaling_matrix scaling_matrix; + struct v4l2_ctrl_h264_decode_params decode_params; + struct v4l2_ctrl_h264_slice_params slice_params; + struct v4l2_ctrl_h264_pred_weights pred_weights; + bool pred_weights_required; + bool first_slice; + int num_slices; +} V4L2RequestControlsH264; + +static uint8_t nalu_slice_start_code[] = { 0x00, 0x00, 0x01 }; + +static void fill_weight_factors(struct v4l2_h264_weight_factors *weight_factors, + int list, const H264SliceContext *sl) +{ + for (int i = 0; i < sl->ref_count[list]; i++) { + if (sl->pwt.luma_weight_flag[list]) { + weight_factors->luma_weight[i] = sl->pwt.luma_weight[i][list][0]; + weight_factors->luma_offset[i] = sl->pwt.luma_weight[i][list][1]; + } else { + weight_factors->luma_weight[i] = 1 << sl->pwt.luma_log2_weight_denom; + weight_factors->luma_offset[i] = 0; + } + for (int j = 0; j < 2; j++) { + if (sl->pwt.chroma_weight_flag[list]) { + weight_factors->chroma_weight[i][j] = sl->pwt.chroma_weight[i][list][j][0]; + weight_factors->chroma_offset[i][j] = sl->pwt.chroma_weight[i][list][j][1]; + } else { + weight_factors->chroma_weight[i][j] = 1 << sl->pwt.chroma_log2_weight_denom; + weight_factors->chroma_offset[i][j] = 0; + } + } + } +} + +static void fill_dpb_entry(struct v4l2_h264_dpb_entry *entry, + const H264Picture *pic, int long_idx) +{ + entry->reference_ts = ff_v4l2_request_get_capture_timestamp(pic->f); + entry->pic_num = pic->pic_id; + entry->frame_num = pic->long_ref ? long_idx : pic->frame_num; + entry->fields = pic->reference & V4L2_H264_FRAME_REF; + entry->flags = V4L2_H264_DPB_ENTRY_FLAG_VALID; + if (entry->fields) + entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_ACTIVE; + if (pic->long_ref) + entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM; + if (pic->field_picture) + entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_FIELD; + if (pic->field_poc[0] != INT_MAX) + entry->top_field_order_cnt = pic->field_poc[0]; + if (pic->field_poc[1] != INT_MAX) + entry->bottom_field_order_cnt = pic->field_poc[1]; +} + +static void fill_dpb(struct v4l2_ctrl_h264_decode_params *decode_params, + const H264Context *h) +{ + int entries = 0; + + for (int i = 0; i < h->short_ref_count; i++) { + const H264Picture *pic = h->short_ref[i]; + if (pic && (pic->field_poc[0] != INT_MAX || pic->field_poc[1] != INT_MAX)) + fill_dpb_entry(&decode_params->dpb[entries++], pic, pic->pic_id); + } + + if (!h->long_ref_count) + return; + + for (int i = 0; i < FF_ARRAY_ELEMS(h->long_ref); i++) { + const H264Picture *pic = h->long_ref[i]; + if (pic && (pic->field_poc[0] != INT_MAX || pic->field_poc[1] != INT_MAX)) + fill_dpb_entry(&decode_params->dpb[entries++], pic, i); + } +} + +static void fill_ref_list(struct v4l2_h264_reference *reference, + struct v4l2_ctrl_h264_decode_params *decode_params, + const H264Ref *ref) +{ + uint64_t timestamp; + + if (!ref->parent) + return; + + timestamp = ff_v4l2_request_get_capture_timestamp(ref->parent->f); + + for (uint8_t i = 0; i < FF_ARRAY_ELEMS(decode_params->dpb); i++) { + struct v4l2_h264_dpb_entry *entry = &decode_params->dpb[i]; + if ((entry->flags & V4L2_H264_DPB_ENTRY_FLAG_VALID) && + entry->reference_ts == timestamp) { + reference->fields = ref->reference & V4L2_H264_FRAME_REF; + reference->index = i; + return; + } + } +} + +static void fill_sps(struct v4l2_ctrl_h264_sps *ctrl, const H264Context *h) +{ + const SPS *sps = h->ps.sps; + + *ctrl = (struct v4l2_ctrl_h264_sps) { + .profile_idc = sps->profile_idc, + .constraint_set_flags = sps->constraint_set_flags, + .level_idc = sps->level_idc, + .seq_parameter_set_id = sps->sps_id, + .chroma_format_idc = sps->chroma_format_idc, + .bit_depth_luma_minus8 = sps->bit_depth_luma - 8, + .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8, + .log2_max_frame_num_minus4 = sps->log2_max_frame_num - 4, + .pic_order_cnt_type = sps->poc_type, + .log2_max_pic_order_cnt_lsb_minus4 = sps->log2_max_poc_lsb - 4, + .max_num_ref_frames = sps->ref_frame_count, + .num_ref_frames_in_pic_order_cnt_cycle = sps->poc_cycle_length, + .offset_for_non_ref_pic = sps->offset_for_non_ref_pic, + .offset_for_top_to_bottom_field = sps->offset_for_top_to_bottom_field, + .pic_width_in_mbs_minus1 = h->mb_width - 1, + .pic_height_in_map_units_minus1 = sps->frame_mbs_only_flag ? + h->mb_height - 1 : h->mb_height / 2 - 1, + }; + + if (sps->poc_cycle_length > 0 && sps->poc_cycle_length <= 255) + memcpy(ctrl->offset_for_ref_frame, sps->offset_for_ref_frame, + sps->poc_cycle_length * sizeof(ctrl->offset_for_ref_frame[0])); + + if (sps->residual_color_transform_flag) + ctrl->flags |= V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; + + if (sps->transform_bypass) + ctrl->flags |= V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS; + + if (sps->delta_pic_order_always_zero_flag) + ctrl->flags |= V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO; + + if (sps->gaps_in_frame_num_allowed_flag) + ctrl->flags |= V4L2_H264_SPS_FLAG_GAPS_IN_FRAME_NUM_VALUE_ALLOWED; + + if (sps->frame_mbs_only_flag) + ctrl->flags |= V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY; + + if (sps->mb_aff) + ctrl->flags |= V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD; + + if (sps->direct_8x8_inference_flag) + ctrl->flags |= V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE; +} + +static void fill_pps(struct v4l2_ctrl_h264_pps *ctrl, const H264Context *h) +{ + const SPS *sps = h->ps.sps; + const PPS *pps = h->ps.pps; + const H264SliceContext *sl = &h->slice_ctx[0]; + int qp_bd_offset = 6 * (sps->bit_depth_luma - 8); + + *ctrl = (struct v4l2_ctrl_h264_pps) { + .pic_parameter_set_id = sl->pps_id, + .seq_parameter_set_id = pps->sps_id, + .num_slice_groups_minus1 = pps->slice_group_count - 1, + .num_ref_idx_l0_default_active_minus1 = pps->ref_count[0] - 1, + .num_ref_idx_l1_default_active_minus1 = pps->ref_count[1] - 1, + .weighted_bipred_idc = pps->weighted_bipred_idc, + .pic_init_qp_minus26 = pps->init_qp - 26 - qp_bd_offset, + .pic_init_qs_minus26 = pps->init_qs - 26 - qp_bd_offset, + .chroma_qp_index_offset = pps->chroma_qp_index_offset[0], + .second_chroma_qp_index_offset = pps->chroma_qp_index_offset[1], + }; + + if (pps->cabac) + ctrl->flags |= V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE; + + if (pps->pic_order_present) + ctrl->flags |= V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT; + + if (pps->weighted_pred) + ctrl->flags |= V4L2_H264_PPS_FLAG_WEIGHTED_PRED; + + if (pps->deblocking_filter_parameters_present) + ctrl->flags |= V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT; + + if (pps->constrained_intra_pred) + ctrl->flags |= V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED; + + if (pps->redundant_pic_cnt_present) + ctrl->flags |= V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT; + + if (pps->transform_8x8_mode) + ctrl->flags |= V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE; + + /* FFmpeg always provide a scaling matrix */ + ctrl->flags |= V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT; +} + +static int v4l2_request_h264_start_frame(AVCodecContext *avctx, + av_unused const uint8_t *buffer, + av_unused uint32_t size) +{ + const H264Context *h = avctx->priv_data; + const PPS *pps = h->ps.pps; + const SPS *sps = h->ps.sps; + const H264SliceContext *sl = &h->slice_ctx[0]; + V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; + int ret; + + ret = ff_v4l2_request_start_frame(avctx, &controls->pic, h->cur_pic_ptr->f); + if (ret) + return ret; + + fill_sps(&controls->sps, h); + fill_pps(&controls->pps, h); + + memcpy(controls->scaling_matrix.scaling_list_4x4, pps->scaling_matrix4, + sizeof(controls->scaling_matrix.scaling_list_4x4)); + memcpy(controls->scaling_matrix.scaling_list_8x8[0], pps->scaling_matrix8[0], + sizeof(controls->scaling_matrix.scaling_list_8x8[0])); + memcpy(controls->scaling_matrix.scaling_list_8x8[1], pps->scaling_matrix8[3], + sizeof(controls->scaling_matrix.scaling_list_8x8[1])); + + if (sps->chroma_format_idc == 3) { + memcpy(controls->scaling_matrix.scaling_list_8x8[2], pps->scaling_matrix8[1], + sizeof(controls->scaling_matrix.scaling_list_8x8[2])); + memcpy(controls->scaling_matrix.scaling_list_8x8[3], pps->scaling_matrix8[4], + sizeof(controls->scaling_matrix.scaling_list_8x8[3])); + memcpy(controls->scaling_matrix.scaling_list_8x8[4], pps->scaling_matrix8[2], + sizeof(controls->scaling_matrix.scaling_list_8x8[4])); + memcpy(controls->scaling_matrix.scaling_list_8x8[5], pps->scaling_matrix8[5], + sizeof(controls->scaling_matrix.scaling_list_8x8[5])); + } + + controls->decode_params = (struct v4l2_ctrl_h264_decode_params) { + .nal_ref_idc = h->nal_ref_idc, + .frame_num = h->poc.frame_num, + .top_field_order_cnt = h->cur_pic_ptr->field_poc[0] != INT_MAX ? + h->cur_pic_ptr->field_poc[0] : 0, + .bottom_field_order_cnt = h->cur_pic_ptr->field_poc[1] != INT_MAX ? + h->cur_pic_ptr->field_poc[1] : 0, + .idr_pic_id = sl->idr_pic_id, + .pic_order_cnt_lsb = sl->poc_lsb, + .delta_pic_order_cnt_bottom = sl->delta_poc_bottom, + .delta_pic_order_cnt0 = sl->delta_poc[0], + .delta_pic_order_cnt1 = sl->delta_poc[1], + /* Size in bits of dec_ref_pic_marking() syntax element. */ + .dec_ref_pic_marking_bit_size = sl->ref_pic_marking_bit_size, + /* Size in bits of pic order count syntax. */ + .pic_order_cnt_bit_size = sl->pic_order_cnt_bit_size, + .slice_group_change_cycle = 0, /* slice group not supported by FFmpeg */ + }; + + if (h->picture_idr) + controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC; + + if (FIELD_PICTURE(h)) + controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC; + + if (h->picture_structure == PICT_BOTTOM_FIELD) + controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD; + +#if defined(V4L2_H264_DECODE_PARAM_FLAG_PFRAME) + if (sl->slice_type_nos == AV_PICTURE_TYPE_P) + controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_PFRAME; +#endif + +#if defined(V4L2_H264_DECODE_PARAM_FLAG_BFRAME) + if (sl->slice_type_nos == AV_PICTURE_TYPE_B) + controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_BFRAME; +#endif + + fill_dpb(&controls->decode_params, h); + + controls->first_slice = true; + controls->num_slices = 0; + + return 0; +} + +static int v4l2_request_h264_queue_decode(AVCodecContext *avctx, bool last_slice) +{ + const H264Context *h = avctx->priv_data; + V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; + V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; + + struct v4l2_ext_control control[] = { + { + .id = V4L2_CID_STATELESS_H264_SPS, + .ptr = &controls->sps, + .size = sizeof(controls->sps), + }, + { + .id = V4L2_CID_STATELESS_H264_PPS, + .ptr = &controls->pps, + .size = sizeof(controls->pps), + }, + { + .id = V4L2_CID_STATELESS_H264_SCALING_MATRIX, + .ptr = &controls->scaling_matrix, + .size = sizeof(controls->scaling_matrix), + }, + { + .id = V4L2_CID_STATELESS_H264_DECODE_PARAMS, + .ptr = &controls->decode_params, + .size = sizeof(controls->decode_params), + }, + { + .id = V4L2_CID_STATELESS_H264_SLICE_PARAMS, + .ptr = &controls->slice_params, + .size = sizeof(controls->slice_params), + }, + { + .id = V4L2_CID_STATELESS_H264_PRED_WEIGHTS, + .ptr = &controls->pred_weights, + .size = sizeof(controls->pred_weights), + }, + }; + + if (ctx->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED) { + int count = FF_ARRAY_ELEMS(control) - (controls->pred_weights_required ? 0 : 1); + return ff_v4l2_request_decode_slice(avctx, &controls->pic, control, count, + controls->first_slice, last_slice); + } + + return ff_v4l2_request_decode_frame(avctx, &controls->pic, + control, FF_ARRAY_ELEMS(control) - 2); +} + +static int v4l2_request_h264_decode_slice(AVCodecContext *avctx, + const uint8_t *buffer, uint32_t size) +{ + const H264Context *h = avctx->priv_data; + const PPS *pps = h->ps.pps; + const H264SliceContext *sl = &h->slice_ctx[0]; + V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; + V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; + int i, ret, count; + + if (ctx->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED && + controls->num_slices) { + ret = v4l2_request_h264_queue_decode(avctx, false); + if (ret) + return ret; + + ff_v4l2_request_reset_picture(avctx, &controls->pic); + controls->first_slice = 0; + } + + if (ctx->start_code == V4L2_STATELESS_H264_START_CODE_ANNEX_B) { + ret = ff_v4l2_request_append_output(avctx, &controls->pic, + nalu_slice_start_code, 3); + if (ret) + return ret; + } + + ret = ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); + if (ret) + return ret; + + if (ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED) + return 0; + + controls->slice_params = (struct v4l2_ctrl_h264_slice_params) { + /* Offset in bits to slice_data() from the beginning of this slice. */ + .header_bit_size = get_bits_count(&sl->gb), + + .first_mb_in_slice = sl->first_mb_addr, + + .slice_type = ff_h264_get_slice_type(sl), + .colour_plane_id = 0, /* separate colour plane not supported by FFmpeg */ + .redundant_pic_cnt = sl->redundant_pic_count, + .cabac_init_idc = sl->cabac_init_idc, + .slice_qp_delta = sl->qscale - pps->init_qp, + .slice_qs_delta = 0, /* not implemented by FFmpeg */ + .disable_deblocking_filter_idc = sl->deblocking_filter < 2 ? + !sl->deblocking_filter : + sl->deblocking_filter, + .slice_alpha_c0_offset_div2 = sl->slice_alpha_c0_offset / 2, + .slice_beta_offset_div2 = sl->slice_beta_offset / 2, + .num_ref_idx_l0_active_minus1 = sl->list_count > 0 ? sl->ref_count[0] - 1 : 0, + .num_ref_idx_l1_active_minus1 = sl->list_count > 1 ? sl->ref_count[1] - 1 : 0, + }; + + if (sl->slice_type == AV_PICTURE_TYPE_B && sl->direct_spatial_mv_pred) + controls->slice_params.flags |= V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED; + + /* V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH: not implemented by FFmpeg */ + + controls->pred_weights_required = + V4L2_H264_CTRL_PRED_WEIGHTS_REQUIRED(&controls->pps, &controls->slice_params); + if (controls->pred_weights_required) { + controls->pred_weights.chroma_log2_weight_denom = sl->pwt.chroma_log2_weight_denom; + controls->pred_weights.luma_log2_weight_denom = sl->pwt.luma_log2_weight_denom; + } + + count = sl->list_count > 0 ? sl->ref_count[0] : 0; + for (i = 0; i < count; i++) + fill_ref_list(&controls->slice_params.ref_pic_list0[i], + &controls->decode_params, &sl->ref_list[0][i]); + if (count && controls->pred_weights_required) + fill_weight_factors(&controls->pred_weights.weight_factors[0], 0, sl); + + count = sl->list_count > 1 ? sl->ref_count[1] : 0; + for (i = 0; i < count; i++) + fill_ref_list(&controls->slice_params.ref_pic_list1[i], + &controls->decode_params, &sl->ref_list[1][i]); + if (count && controls->pred_weights_required) + fill_weight_factors(&controls->pred_weights.weight_factors[1], 1, sl); + + controls->num_slices++; + return 0; +} + +static int v4l2_request_h264_end_frame(AVCodecContext *avctx) +{ + return v4l2_request_h264_queue_decode(avctx, true); +} + +static int v4l2_request_h264_post_frames_ctx(AVCodecContext *avctx) +{ + V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; + + struct v4l2_ext_control control[] = { + { .id = V4L2_CID_STATELESS_H264_DECODE_MODE, }, + { .id = V4L2_CID_STATELESS_H264_START_CODE, }, + }; + + ctx->decode_mode = ff_v4l2_request_query_control_default_value(avctx, + V4L2_CID_STATELESS_H264_DECODE_MODE); + if (ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED && + ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED) { + av_log(ctx, AV_LOG_VERBOSE, "Unsupported decode mode: %d\n", + ctx->decode_mode); + return AVERROR(EINVAL); + } + + ctx->start_code = ff_v4l2_request_query_control_default_value(avctx, + V4L2_CID_STATELESS_H264_START_CODE); + if (ctx->start_code != V4L2_STATELESS_H264_START_CODE_NONE && + ctx->start_code != V4L2_STATELESS_H264_START_CODE_ANNEX_B) { + av_log(ctx, AV_LOG_VERBOSE, "Unsupported start code: %d\n", + ctx->start_code); + return AVERROR(EINVAL); + } + + // TODO: check V4L2_CID_MPEG_VIDEO_H264_PROFILE control + // TODO: check V4L2_CID_MPEG_VIDEO_H264_LEVEL control + + control[0].value = ctx->decode_mode; + control[1].value = ctx->start_code; + + return ff_v4l2_request_set_controls(avctx, control, FF_ARRAY_ELEMS(control)); +} + +static int v4l2_request_h264_init(AVCodecContext *avctx) +{ + const H264Context *h = avctx->priv_data; + struct v4l2_ctrl_h264_sps sps; + + struct v4l2_ext_control control[] = { + { + .id = V4L2_CID_STATELESS_H264_SPS, + .ptr = &sps, + .size = sizeof(sps), + }, + }; + + fill_sps(&sps, h); + + return ff_v4l2_request_init(avctx, control, FF_ARRAY_ELEMS(control), + v4l2_request_h264_post_frames_ctx); +} + +static int v4l2_request_h264_frame_params(AVCodecContext *avctx, + AVBufferRef *hw_frames_ctx) +{ + const H264Context *h = avctx->priv_data; + const SPS *sps = h ? h->ps.sps : NULL; + uint8_t bit_depth = sps ? sps->bit_depth_luma : 0; + + return ff_v4l2_request_frame_params(avctx, hw_frames_ctx, + V4L2_PIX_FMT_H264_SLICE, bit_depth); +} + +const FFHWAccel ff_h264_v4l2request_hwaccel = { + .p.name = "h264_v4l2request", + .p.type = AVMEDIA_TYPE_VIDEO, + .p.id = AV_CODEC_ID_H264, + .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, + .start_frame = v4l2_request_h264_start_frame, + .decode_slice = v4l2_request_h264_decode_slice, + .end_frame = v4l2_request_h264_end_frame, + .flush = ff_v4l2_request_flush, + .frame_priv_data_size = sizeof(V4L2RequestControlsH264), + .init = v4l2_request_h264_init, + .uninit = ff_v4l2_request_uninit, + .priv_data_size = sizeof(V4L2RequestContextH264), + .frame_params = v4l2_request_h264_frame_params, + .caps_internal = HWACCEL_CAP_ASYNC_SAFE, +}; -- 2.53.0 From c6b2a583c3faebfc02174b706fe506e464bdc13f Mon Sep 17 00:00:00 2001 From: Jernej Skrabec Date: Tue, 6 Aug 2024 09:06:07 +0000 Subject: [PATCH 08/13] avcodec: Add V4L2 Request API hevc hwaccel Add a V4L2 Request API hwaccel for HEVC, supporting both slice and frame decoding modes. Support for HEVC is enabled when Linux kernel headers declare the control id V4L2_CID_STATELESS_HEVC_SPS, added in v6.0. Co-developed-by: Benjamin Gaignard Signed-off-by: Benjamin Gaignard Co-developed-by: Alex Bee Signed-off-by: Alex Bee Signed-off-by: Jernej Skrabec Co-developed-by: Jonas Karlman Signed-off-by: Jonas Karlman --- configure | 5 + libavcodec/Makefile | 1 + libavcodec/hevc/hevcdec.c | 10 + libavcodec/hwaccels.h | 1 + libavcodec/v4l2_request_hevc.c | 754 +++++++++++++++++++++++++++++++++ 5 files changed, 771 insertions(+) create mode 100644 libavcodec/v4l2_request_hevc.c diff --git a/configure b/configure index 28253b3e2b..a56fd0a16c 100755 --- a/configure +++ b/configure @@ -2494,6 +2494,7 @@ TYPES_LIST=" struct_sockaddr_sa_len struct_sockaddr_storage struct_stat_st_mtim_tv_nsec + struct_v4l2_ctrl_hevc_decode_params_num_delta_pocs_of_ref_rps_idx struct_v4l2_frmivalenum_discrete struct_mfxConfigInterface " @@ -3211,6 +3212,8 @@ hevc_dxva2_hwaccel_deps="dxva2 DXVA_PicParams_HEVC" hevc_dxva2_hwaccel_select="hevc_decoder" hevc_nvdec_hwaccel_deps="nvdec" hevc_nvdec_hwaccel_select="hevc_decoder" +hevc_v4l2request_hwaccel_deps="v4l2_request hevc_v4l2_request" +hevc_v4l2request_hwaccel_select="hevc_decoder" hevc_vaapi_hwaccel_deps="vaapi VAPictureParameterBufferHEVC" hevc_vaapi_hwaccel_select="hevc_decoder" hevc_vdpau_hwaccel_deps="vdpau VdpPictureInfoHEVC" @@ -7229,10 +7232,12 @@ fi if enabled v4l2_request; then check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_H264_DECODE_MODE" + check_cc hevc_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_HEVC_SPS" check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns check_pkg_config libudev libudev libudev.h udev_new + check_struct linux/videodev2.h "struct v4l2_ctrl_hevc_decode_params" num_delta_pocs_of_ref_rps_idx v4l2_request_extralibs="$libudev_extralibs" fi diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 74303ad674..d2b5831694 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1027,6 +1027,7 @@ OBJS-$(CONFIG_HEVC_DXVA2_HWACCEL) += dxva2_hevc.o OBJS-$(CONFIG_HEVC_D3D12VA_HWACCEL) += dxva2_hevc.o d3d12va_hevc.o OBJS-$(CONFIG_HEVC_NVDEC_HWACCEL) += nvdec_hevc.o OBJS-$(CONFIG_HEVC_QSV_HWACCEL) += qsvdec.o +OBJS-$(CONFIG_HEVC_V4L2REQUEST_HWACCEL) += v4l2_request_hevc.o OBJS-$(CONFIG_HEVC_VAAPI_HWACCEL) += vaapi_hevc.o h265_profile_level.o OBJS-$(CONFIG_HEVC_VDPAU_HWACCEL) += vdpau_hevc.o h265_profile_level.o OBJS-$(CONFIG_HEVC_VULKAN_HWACCEL) += vulkan_decode.o vulkan_hevc.o diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c index 7bd3a706a7..21498a5dfc 100644 --- a/libavcodec/hevc/hevcdec.c +++ b/libavcodec/hevc/hevcdec.c @@ -528,6 +528,7 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps) CONFIG_HEVC_D3D11VA_HWACCEL * 2 + \ CONFIG_HEVC_D3D12VA_HWACCEL + \ CONFIG_HEVC_NVDEC_HWACCEL + \ + CONFIG_HEVC_V4L2REQUEST_HWACCEL + \ CONFIG_HEVC_VAAPI_HWACCEL + \ CONFIG_HEVC_VIDEOTOOLBOX_HWACCEL + \ CONFIG_HEVC_VDPAU_HWACCEL + \ @@ -562,6 +563,9 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps) #endif #if CONFIG_HEVC_VULKAN_HWACCEL *fmt++ = AV_PIX_FMT_VULKAN; +#endif +#if CONFIG_HEVC_V4L2REQUEST_HWACCEL + *fmt++ = AV_PIX_FMT_DRM_PRIME; #endif break; case AV_PIX_FMT_YUV420P10: @@ -589,6 +593,9 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps) #endif #if CONFIG_HEVC_NVDEC_HWACCEL *fmt++ = AV_PIX_FMT_CUDA; +#endif +#if CONFIG_HEVC_V4L2REQUEST_HWACCEL + *fmt++ = AV_PIX_FMT_DRM_PRIME; #endif break; case AV_PIX_FMT_YUV444P: @@ -4136,6 +4143,9 @@ const FFCodec ff_hevc_decoder = { #endif #if CONFIG_HEVC_VULKAN_HWACCEL HWACCEL_VULKAN(hevc), +#endif +#if CONFIG_HEVC_V4L2REQUEST_HWACCEL + HWACCEL_V4L2REQUEST(hevc), #endif NULL }, diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h index 024e021d73..ad145fc07c 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h @@ -44,6 +44,7 @@ extern const struct FFHWAccel ff_hevc_d3d11va2_hwaccel; extern const struct FFHWAccel ff_hevc_d3d12va_hwaccel; extern const struct FFHWAccel ff_hevc_dxva2_hwaccel; extern const struct FFHWAccel ff_hevc_nvdec_hwaccel; +extern const struct FFHWAccel ff_hevc_v4l2request_hwaccel; extern const struct FFHWAccel ff_hevc_vaapi_hwaccel; extern const struct FFHWAccel ff_hevc_vdpau_hwaccel; extern const struct FFHWAccel ff_hevc_videotoolbox_hwaccel; diff --git a/libavcodec/v4l2_request_hevc.c b/libavcodec/v4l2_request_hevc.c new file mode 100644 index 0000000000..c32166c1e3 --- /dev/null +++ b/libavcodec/v4l2_request_hevc.c @@ -0,0 +1,754 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include "libavutil/mem.h" +#include "hevc/hevcdec.h" +#include "hwaccel_internal.h" +#include "hwconfig.h" +#include "internal.h" +#include "v4l2_request.h" + +#define V4L2_HEVC_CONTROLS_MAX 6 + +typedef struct V4L2RequestContextHEVC { + V4L2RequestContext base; + enum v4l2_stateless_hevc_decode_mode decode_mode; + enum v4l2_stateless_hevc_start_code start_code; + unsigned int max_slice_params; + unsigned int max_entry_point_offsets; + bool has_scaling_matrix; +} V4L2RequestContextHEVC; + +typedef struct V4L2RequestControlsHEVC { + V4L2RequestPictureContext pic; + struct v4l2_ctrl_hevc_sps sps; + struct v4l2_ctrl_hevc_pps pps; + struct v4l2_ctrl_hevc_decode_params decode_params; + struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix; + struct v4l2_ctrl_hevc_slice_params slice_params; + struct v4l2_ctrl_hevc_slice_params *frame_slice_params; + unsigned int allocated_slice_params; + unsigned int num_slice_params; + uint32_t *entry_point_offsets; + unsigned int allocated_entry_point_offsets; + unsigned int num_entry_point_offsets; + bool first_slice; +} V4L2RequestControlsHEVC; + +static uint8_t nalu_slice_start_code[] = { 0x00, 0x00, 0x01 }; + +static void fill_pred_weight_table(struct v4l2_hevc_pred_weight_table *table, + const HEVCContext *h) +{ + int32_t luma_weight_denom, chroma_weight_denom; + const SliceHeader *sh = &h->sh; + + if (sh->slice_type == HEVC_SLICE_I || + (sh->slice_type == HEVC_SLICE_P && !h->pps->weighted_pred_flag) || + (sh->slice_type == HEVC_SLICE_B && !h->pps->weighted_bipred_flag)) + return; + + table->luma_log2_weight_denom = sh->luma_log2_weight_denom; + + if (h->pps->sps->chroma_format_idc) + table->delta_chroma_log2_weight_denom = sh->chroma_log2_weight_denom - + sh->luma_log2_weight_denom; + + luma_weight_denom = (1 << sh->luma_log2_weight_denom); + chroma_weight_denom = (1 << sh->chroma_log2_weight_denom); + + for (int i = 0; i < 15 && i < sh->nb_refs[L0]; i++) { + table->delta_luma_weight_l0[i] = sh->luma_weight_l0[i] - luma_weight_denom; + table->luma_offset_l0[i] = sh->luma_offset_l0[i]; + table->delta_chroma_weight_l0[i][0] = sh->chroma_weight_l0[i][0] - chroma_weight_denom; + table->delta_chroma_weight_l0[i][1] = sh->chroma_weight_l0[i][1] - chroma_weight_denom; + table->chroma_offset_l0[i][0] = sh->chroma_offset_l0[i][0]; + table->chroma_offset_l0[i][1] = sh->chroma_offset_l0[i][1]; + } + + if (sh->slice_type != HEVC_SLICE_B) + return; + + for (int i = 0; i < 15 && i < sh->nb_refs[L1]; i++) { + table->delta_luma_weight_l1[i] = sh->luma_weight_l1[i] - luma_weight_denom; + table->luma_offset_l1[i] = sh->luma_offset_l1[i]; + table->delta_chroma_weight_l1[i][0] = sh->chroma_weight_l1[i][0] - chroma_weight_denom; + table->delta_chroma_weight_l1[i][1] = sh->chroma_weight_l1[i][1] - chroma_weight_denom; + table->chroma_offset_l1[i][0] = sh->chroma_offset_l1[i][0]; + table->chroma_offset_l1[i][1] = sh->chroma_offset_l1[i][1]; + } +} + +static uint8_t get_ref_pic_index(const HEVCContext *h, const HEVCFrame *frame, + struct v4l2_ctrl_hevc_decode_params *decode_params) +{ + uint64_t timestamp; + + if (!frame || !frame->f) + return 0; + + timestamp = ff_v4l2_request_get_capture_timestamp(frame->f); + + for (uint8_t i = 0; i < decode_params->num_active_dpb_entries; i++) { + struct v4l2_hevc_dpb_entry *entry = &decode_params->dpb[i]; + if (entry->timestamp == timestamp) + return i; + } + + return 0; +} + +static void fill_decode_params(struct v4l2_ctrl_hevc_decode_params *decode_params, + const HEVCContext *h) +{ + const HEVCFrame *pic = h->cur_frame; + const HEVCLayerContext *l = &h->layers[h->cur_layer]; + const SliceHeader *sh = &h->sh; + int i, entries = 0; + + *decode_params = (struct v4l2_ctrl_hevc_decode_params) { + .pic_order_cnt_val = h->poc, + .short_term_ref_pic_set_size = sh->short_term_ref_pic_set_size, + .long_term_ref_pic_set_size = sh->long_term_ref_pic_set_size, + .num_poc_st_curr_before = h->rps[ST_CURR_BEF].nb_refs, + .num_poc_st_curr_after = h->rps[ST_CURR_AFT].nb_refs, + .num_poc_lt_curr = h->rps[LT_CURR].nb_refs, + }; + +#if HAVE_STRUCT_V4L2_CTRL_HEVC_DECODE_PARAMS_NUM_DELTA_POCS_OF_REF_RPS_IDX + if (h->sh.short_term_ref_pic_set_sps_flag == 0 && h->sh.short_term_rps) + decode_params->num_delta_pocs_of_ref_rps_idx = + h->sh.short_term_rps->rps_idx_num_delta_pocs; +#endif + + for (i = 0; i < FF_ARRAY_ELEMS(l->DPB); i++) { + const HEVCFrame *frame = &l->DPB[i]; + if (frame != pic && + (frame->flags & (HEVC_FRAME_FLAG_LONG_REF | HEVC_FRAME_FLAG_SHORT_REF))) { + struct v4l2_hevc_dpb_entry *entry = &decode_params->dpb[entries++]; + + entry->timestamp = ff_v4l2_request_get_capture_timestamp(frame->f); + entry->field_pic = !!(frame->f->flags & AV_FRAME_FLAG_INTERLACED); + entry->flags = 0; + if (frame->flags & HEVC_FRAME_FLAG_LONG_REF) + entry->flags |= V4L2_HEVC_DPB_ENTRY_LONG_TERM_REFERENCE; + + entry->pic_order_cnt_val = frame->poc; + } + } + + decode_params->num_active_dpb_entries = entries; + + if (IS_IRAP(h)) + decode_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_IRAP_PIC; + + if (IS_IDR(h)) + decode_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_IDR_PIC; + + if (sh->no_output_of_prior_pics_flag) + decode_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_NO_OUTPUT_OF_PRIOR; + + for (i = 0; i < V4L2_HEVC_DPB_ENTRIES_NUM_MAX; i++) { + decode_params->poc_st_curr_before[i] = + get_ref_pic_index(h, h->rps[ST_CURR_BEF].ref[i], decode_params); + decode_params->poc_st_curr_after[i] = + get_ref_pic_index(h, h->rps[ST_CURR_AFT].ref[i], decode_params); + decode_params->poc_lt_curr[i] = + get_ref_pic_index(h, h->rps[LT_CURR].ref[i], decode_params); + } +} + +static int fill_slice_params(V4L2RequestControlsHEVC *controls, int slice, + bool max_entry_point_offsets, const HEVCContext *h) +{ + struct v4l2_ctrl_hevc_slice_params *slice_params = &controls->frame_slice_params[slice]; + struct v4l2_ctrl_hevc_decode_params *decode_params = &controls->decode_params; + const SliceHeader *sh = &h->sh; + RefPicList *rpl; + int i, offsets; + + *slice_params = (struct v4l2_ctrl_hevc_slice_params) { + .bit_size = 0, + .data_byte_offset = controls->pic.output->bytesused + sh->data_offset, + .num_entry_point_offsets = sh->num_entry_point_offsets, + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ + .nal_unit_type = h->nal_unit_type, + .nuh_temporal_id_plus1 = h->temporal_id + 1, + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ + .slice_type = sh->slice_type, + .colour_plane_id = sh->colour_plane_id, + .slice_pic_order_cnt = sh->poc, + .num_ref_idx_l0_active_minus1 = sh->nb_refs[L0] ? sh->nb_refs[L0] - 1 : 0, + .num_ref_idx_l1_active_minus1 = sh->nb_refs[L1] ? sh->nb_refs[L1] - 1 : 0, + .collocated_ref_idx = sh->slice_temporal_mvp_enabled_flag ? + sh->collocated_ref_idx : 0, + .five_minus_max_num_merge_cand = sh->slice_type == HEVC_SLICE_I ? + 0 : 5 - sh->max_num_merge_cand, + .slice_qp_delta = sh->slice_qp_delta, + .slice_cb_qp_offset = sh->slice_cb_qp_offset, + .slice_cr_qp_offset = sh->slice_cr_qp_offset, + .slice_act_y_qp_offset = 0, + .slice_act_cb_qp_offset = 0, + .slice_act_cr_qp_offset = 0, + .slice_beta_offset_div2 = sh->beta_offset / 2, + .slice_tc_offset_div2 = sh->tc_offset / 2, + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ + .pic_struct = h->sei.picture_timing.picture_struct, + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ + .slice_segment_addr = sh->slice_segment_addr, + .short_term_ref_pic_set_size = sh->short_term_ref_pic_set_size, + .long_term_ref_pic_set_size = sh->long_term_ref_pic_set_size, + }; + + if (h->pps->pps_slice_act_qp_offsets_present_flag) { + slice_params->slice_act_y_qp_offset = sh->slice_act_y_qp_offset; + slice_params->slice_act_cb_qp_offset = sh->slice_act_cb_qp_offset; + slice_params->slice_act_cr_qp_offset = sh->slice_act_cr_qp_offset; + } + + if (sh->slice_sample_adaptive_offset_flag[0]) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_SAO_LUMA; + + if (sh->slice_sample_adaptive_offset_flag[1]) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_SAO_CHROMA; + + if (sh->slice_temporal_mvp_enabled_flag) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_TEMPORAL_MVP_ENABLED; + + if (sh->mvd_l1_zero_flag) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_MVD_L1_ZERO; + + if (sh->cabac_init_flag) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_CABAC_INIT; + + if (sh->collocated_list == L0) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_COLLOCATED_FROM_L0; + + if (sh->use_integer_mv_flag) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_USE_INTEGER_MV; + + if (sh->disable_deblocking_filter_flag) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_DEBLOCKING_FILTER_DISABLED; + + if (sh->slice_loop_filter_across_slices_enabled_flag) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED; + + if (sh->dependent_slice_segment_flag) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_DEPENDENT_SLICE_SEGMENT; + + if (sh->slice_type != HEVC_SLICE_I) { + rpl = &h->cur_frame->refPicList[0]; + for (i = 0; i < rpl->nb_refs; i++) + slice_params->ref_idx_l0[i] = get_ref_pic_index(h, rpl->ref[i], decode_params); + } + + if (sh->slice_type == HEVC_SLICE_B) { + rpl = &h->cur_frame->refPicList[1]; + for (i = 0; i < rpl->nb_refs; i++) + slice_params->ref_idx_l1[i] = get_ref_pic_index(h, rpl->ref[i], decode_params); + } + + fill_pred_weight_table(&slice_params->pred_weight_table, h); + + if (!max_entry_point_offsets) + return 0; + + if (controls->allocated_entry_point_offsets < controls->num_entry_point_offsets + sh->num_entry_point_offsets) { + void *entry_point_offsets = controls->entry_point_offsets; + offsets = controls->allocated_entry_point_offsets == 0 ? 128 : controls->allocated_entry_point_offsets * 2; + while (controls->num_entry_point_offsets + sh->num_entry_point_offsets > offsets) + offsets *= 2; + entry_point_offsets = av_realloc_array(entry_point_offsets, offsets, sizeof(*controls->entry_point_offsets)); + if (!entry_point_offsets) + return AVERROR(ENOMEM); + controls->entry_point_offsets = entry_point_offsets; + controls->allocated_entry_point_offsets = offsets; + } + + for (i = 0, offsets = controls->num_entry_point_offsets; i < sh->num_entry_point_offsets; i++) + controls->entry_point_offsets[offsets + i] = sh->entry_point_offset[i]; + controls->num_entry_point_offsets += sh->num_entry_point_offsets; + + return 0; +} + +static void fill_sps(struct v4l2_ctrl_hevc_sps *ctrl, const HEVCContext *h) +{ + const HEVCPPS *pps = h->pps; + const HEVCSPS *sps = pps->sps; + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Sequence parameter set */ + *ctrl = (struct v4l2_ctrl_hevc_sps) { + .video_parameter_set_id = sps->vps_id, + .seq_parameter_set_id = pps->sps_id, + .pic_width_in_luma_samples = sps->width, + .pic_height_in_luma_samples = sps->height, + .bit_depth_luma_minus8 = sps->bit_depth - 8, + .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8, + .log2_max_pic_order_cnt_lsb_minus4 = sps->log2_max_poc_lsb - 4, + .sps_max_dec_pic_buffering_minus1 = + sps->temporal_layer[sps->max_sub_layers - 1].max_dec_pic_buffering - 1, + .sps_max_num_reorder_pics = + sps->temporal_layer[sps->max_sub_layers - 1].num_reorder_pics, + .sps_max_latency_increase_plus1 = + sps->temporal_layer[sps->max_sub_layers - 1].max_latency_increase + 1, + .log2_min_luma_coding_block_size_minus3 = sps->log2_min_cb_size - 3, + .log2_diff_max_min_luma_coding_block_size = + sps->log2_diff_max_min_coding_block_size, + .log2_min_luma_transform_block_size_minus2 = sps->log2_min_tb_size - 2, + .log2_diff_max_min_luma_transform_block_size = + sps->log2_max_trafo_size - sps->log2_min_tb_size, + .max_transform_hierarchy_depth_inter = sps->max_transform_hierarchy_depth_inter, + .max_transform_hierarchy_depth_intra = sps->max_transform_hierarchy_depth_intra, + .pcm_sample_bit_depth_luma_minus1 = sps->pcm.bit_depth - 1, + .pcm_sample_bit_depth_chroma_minus1 = sps->pcm.bit_depth_chroma - 1, + .log2_min_pcm_luma_coding_block_size_minus3 = sps->pcm.log2_min_pcm_cb_size - 3, + .log2_diff_max_min_pcm_luma_coding_block_size = + sps->pcm.log2_max_pcm_cb_size - sps->pcm.log2_min_pcm_cb_size, + .num_short_term_ref_pic_sets = sps->nb_st_rps, + .num_long_term_ref_pics_sps = sps->num_long_term_ref_pics_sps, + .chroma_format_idc = sps->chroma_format_idc, + .sps_max_sub_layers_minus1 = sps->max_sub_layers - 1, + }; + + if (sps->separate_colour_plane) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_SEPARATE_COLOUR_PLANE; + + if (sps->scaling_list_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_SCALING_LIST_ENABLED; + + if (sps->amp_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_AMP_ENABLED; + + if (sps->sao_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_SAMPLE_ADAPTIVE_OFFSET; + + if (sps->pcm_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_PCM_ENABLED; + + if (sps->pcm_loop_filter_disabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_PCM_LOOP_FILTER_DISABLED; + + if (sps->long_term_ref_pics_present) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_LONG_TERM_REF_PICS_PRESENT; + + if (sps->temporal_mvp_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_SPS_TEMPORAL_MVP_ENABLED; + + if (sps->strong_intra_smoothing_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_STRONG_INTRA_SMOOTHING_ENABLED; +} + +static int v4l2_request_hevc_start_frame(AVCodecContext *avctx, + av_unused const uint8_t *buffer, + av_unused uint32_t size) +{ + const HEVCContext *h = avctx->priv_data; + const HEVCPPS *pps = h->pps; + const HEVCSPS *sps = pps->sps; + V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data; + V4L2RequestControlsHEVC *controls = h->cur_frame->hwaccel_picture_private; + const SliceHeader *sh = &h->sh; + int ret; + + ret = ff_v4l2_request_start_frame(avctx, &controls->pic, h->cur_frame->f); + if (ret) + return ret; + + fill_sps(&controls->sps, h); + fill_decode_params(&controls->decode_params, h); + + if (ctx->has_scaling_matrix) { + const ScalingList *sl = pps->scaling_list_data_present_flag ? + &pps->scaling_list : + sps->scaling_list_enabled ? + &sps->scaling_list : NULL; + if (sl) { + for (int i = 0; i < 6; i++) { + for (int j = 0; j < 16; j++) + controls->scaling_matrix.scaling_list_4x4[i][j] = sl->sl[0][i][j]; + for (int j = 0; j < 64; j++) { + controls->scaling_matrix.scaling_list_8x8[i][j] = sl->sl[1][i][j]; + controls->scaling_matrix.scaling_list_16x16[i][j] = sl->sl[2][i][j]; + if (i < 2) + controls->scaling_matrix.scaling_list_32x32[i][j] = sl->sl[3][i * 3][j]; + } + controls->scaling_matrix.scaling_list_dc_coef_16x16[i] = sl->sl_dc[0][i]; + if (i < 2) + controls->scaling_matrix.scaling_list_dc_coef_32x32[i] = sl->sl_dc[1][i * 3]; + } + } + } + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture parameter set */ + controls->pps = (struct v4l2_ctrl_hevc_pps) { + .pic_parameter_set_id = sh->pps_id, + .num_extra_slice_header_bits = pps->num_extra_slice_header_bits, + .num_ref_idx_l0_default_active_minus1 = pps->num_ref_idx_l0_default_active - 1, + .num_ref_idx_l1_default_active_minus1 = pps->num_ref_idx_l1_default_active - 1, + .init_qp_minus26 = pps->pic_init_qp_minus26, + .diff_cu_qp_delta_depth = pps->diff_cu_qp_delta_depth, + .pps_cb_qp_offset = pps->cb_qp_offset, + .pps_cr_qp_offset = pps->cr_qp_offset, + .pps_beta_offset_div2 = pps->beta_offset / 2, + .pps_tc_offset_div2 = pps->tc_offset / 2, + .log2_parallel_merge_level_minus2 = pps->log2_parallel_merge_level - 2, + }; + + if (pps->dependent_slice_segments_enabled_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_DEPENDENT_SLICE_SEGMENT_ENABLED; + + if (pps->output_flag_present_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_OUTPUT_FLAG_PRESENT; + + if (pps->sign_data_hiding_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_SIGN_DATA_HIDING_ENABLED; + + if (pps->cabac_init_present_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_CABAC_INIT_PRESENT; + + if (pps->constrained_intra_pred_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_CONSTRAINED_INTRA_PRED; + + if (pps->transform_skip_enabled_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_TRANSFORM_SKIP_ENABLED; + + if (pps->cu_qp_delta_enabled_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_CU_QP_DELTA_ENABLED; + + if (pps->pic_slice_level_chroma_qp_offsets_present_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT; + + if (pps->weighted_pred_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_WEIGHTED_PRED; + + if (pps->weighted_bipred_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_WEIGHTED_BIPRED; + + if (pps->transquant_bypass_enable_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_TRANSQUANT_BYPASS_ENABLED; + + if (pps->tiles_enabled_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_TILES_ENABLED; + + if (pps->entropy_coding_sync_enabled_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_ENTROPY_CODING_SYNC_ENABLED; + + if (pps->loop_filter_across_tiles_enabled_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED; + + if (pps->seq_loop_filter_across_slices_enabled_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED; + + if (pps->deblocking_filter_override_enabled_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_DEBLOCKING_FILTER_OVERRIDE_ENABLED; + + if (pps->disable_dbf) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_PPS_DISABLE_DEBLOCKING_FILTER; + + if (pps->lists_modification_present_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_LISTS_MODIFICATION_PRESENT; + + if (pps->slice_header_extension_present_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_SLICE_SEGMENT_HEADER_EXTENSION_PRESENT; + + if (pps->deblocking_filter_control_present_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT; + + if (pps->uniform_spacing_flag) + controls->pps.flags |= V4L2_HEVC_PPS_FLAG_UNIFORM_SPACING; + + if (pps->tiles_enabled_flag) { + controls->pps.num_tile_columns_minus1 = pps->num_tile_columns - 1; + controls->pps.num_tile_rows_minus1 = pps->num_tile_rows - 1; + + for (int i = 0; i < pps->num_tile_columns; i++) + controls->pps.column_width_minus1[i] = pps->column_width[i] - 1; + + for (int i = 0; i < pps->num_tile_rows; i++) + controls->pps.row_height_minus1[i] = pps->row_height[i] - 1; + } + + controls->first_slice = true; + controls->frame_slice_params = &controls->slice_params; + controls->allocated_slice_params = 0; + controls->num_slice_params = 0; + controls->allocated_entry_point_offsets = 0; + controls->num_entry_point_offsets = 0; + + return 0; +} + +static int v4l2_request_hevc_queue_decode(AVCodecContext *avctx, bool last_slice) +{ + const HEVCContext *h = avctx->priv_data; + V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data; + V4L2RequestControlsHEVC *controls = h->cur_frame->hwaccel_picture_private; + int count = 0; + + struct v4l2_ext_control control[V4L2_HEVC_CONTROLS_MAX] = {}; + + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_SPS, + .ptr = &controls->sps, + .size = sizeof(controls->sps), + }; + + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_PPS, + .ptr = &controls->pps, + .size = sizeof(controls->pps), + }; + + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_DECODE_PARAMS, + .ptr = &controls->decode_params, + .size = sizeof(controls->decode_params), + }; + + if (ctx->has_scaling_matrix) { + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX, + .ptr = &controls->scaling_matrix, + .size = sizeof(controls->scaling_matrix), + }; + } + + if (ctx->max_slice_params && controls->num_slice_params) { + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS, + .ptr = controls->frame_slice_params, + .size = sizeof(*controls->frame_slice_params) * + FFMIN(controls->num_slice_params, ctx->max_slice_params), + }; + } + + if (ctx->max_entry_point_offsets && controls->num_entry_point_offsets) { + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS, + .ptr = controls->entry_point_offsets, + .size = sizeof(*controls->entry_point_offsets) * + FFMIN(controls->num_entry_point_offsets, + ctx->max_entry_point_offsets), + }; + } + + if (ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED) + return ff_v4l2_request_decode_slice(avctx, &controls->pic, control, count, + controls->first_slice, last_slice); + + return ff_v4l2_request_decode_frame(avctx, &controls->pic, control, count); +} + +static int v4l2_request_hevc_decode_slice(AVCodecContext *avctx, + const uint8_t *buffer, uint32_t size) +{ + const HEVCContext *h = avctx->priv_data; + V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data; + V4L2RequestControlsHEVC *controls = h->cur_frame->hwaccel_picture_private; + const SliceHeader *sh = &h->sh; + int ret, slice = controls->num_slice_params; + uint32_t extra_size = 0; + + if (ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED && + (slice >= ctx->max_slice_params || (ctx->max_entry_point_offsets && + (controls->num_entry_point_offsets + sh->num_entry_point_offsets > ctx->max_entry_point_offsets)))) { + ret = v4l2_request_hevc_queue_decode(avctx, false); + if (ret) + return ret; + + ff_v4l2_request_reset_picture(avctx, &controls->pic); + slice = controls->num_slice_params = 0; + controls->num_entry_point_offsets = 0; + controls->first_slice = false; + } + + if (ctx->start_code == V4L2_STATELESS_HEVC_START_CODE_ANNEX_B) { + ret = ff_v4l2_request_append_output(avctx, &controls->pic, + nalu_slice_start_code, 3); + if (ret) + return ret; + extra_size = 3; + } + + if (ctx->max_slice_params) { + if (slice && controls->allocated_slice_params < slice + 1) { + void *slice_params = controls->allocated_slice_params == 0 ? NULL : controls->frame_slice_params; + int slices = controls->allocated_slice_params == 0 ? 8 : controls->allocated_slice_params * 2; + slice_params = av_realloc_array(slice_params, slices, sizeof(*controls->frame_slice_params)); + if (!slice_params) + return AVERROR(ENOMEM); + if (controls->allocated_slice_params == 0) + memcpy(slice_params, controls->frame_slice_params, sizeof(*controls->frame_slice_params)); + controls->frame_slice_params = slice_params; + controls->allocated_slice_params = slices; + } + + ret = fill_slice_params(controls, slice, !!ctx->max_entry_point_offsets, h); + if (ret) + return ret; + } + + ret = ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); + if (ret) + return ret; + + if (ctx->max_slice_params) + controls->frame_slice_params[slice].bit_size = (size + extra_size) * 8; + + controls->num_slice_params++; + return 0; +} + +static int v4l2_request_hevc_end_frame(AVCodecContext *avctx) +{ + return v4l2_request_hevc_queue_decode(avctx, true); +} + +static void v4l2_request_hevc_free_frame_priv(FFRefStructOpaque hwctx, void *data) +{ + V4L2RequestControlsHEVC *controls = data; + + if (controls->allocated_slice_params) + av_freep(&controls->frame_slice_params); + + av_freep(&controls->entry_point_offsets); +} + +static int v4l2_request_hevc_post_frames_ctx(AVCodecContext *avctx) +{ + V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data; + int ret; + + struct v4l2_ext_control control[] = { + { .id = V4L2_CID_STATELESS_HEVC_DECODE_MODE, }, + { .id = V4L2_CID_STATELESS_HEVC_START_CODE, }, + }; + struct v4l2_query_ext_ctrl scaling_matrix = { + .id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX, + }; + struct v4l2_query_ext_ctrl entry_point_offsets = { + .id = V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS, + }; + struct v4l2_query_ext_ctrl slice_params = { + .id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS, + }; + + ctx->decode_mode = ff_v4l2_request_query_control_default_value(avctx, + V4L2_CID_STATELESS_HEVC_DECODE_MODE); + if (ctx->decode_mode != V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED && + ctx->decode_mode != V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED) { + av_log(ctx, AV_LOG_VERBOSE, "Unsupported decode mode: %d\n", + ctx->decode_mode); + return AVERROR(EINVAL); + } + + ctx->start_code = ff_v4l2_request_query_control_default_value(avctx, + V4L2_CID_STATELESS_HEVC_START_CODE); + if (ctx->start_code != V4L2_STATELESS_HEVC_START_CODE_NONE && + ctx->start_code != V4L2_STATELESS_HEVC_START_CODE_ANNEX_B) { + av_log(ctx, AV_LOG_VERBOSE, "Unsupported start code: %d\n", + ctx->start_code); + return AVERROR(EINVAL); + } + + // TODO: check V4L2_CID_MPEG_VIDEO_HEVC_PROFILE control + // TODO: check V4L2_CID_MPEG_VIDEO_HEVC_LEVEL control + + ret = ff_v4l2_request_query_control(avctx, &scaling_matrix); + if (!ret) + ctx->has_scaling_matrix = true; + else + ctx->has_scaling_matrix = false; + + ret = ff_v4l2_request_query_control(avctx, &entry_point_offsets); + if (!ret) + ctx->max_entry_point_offsets = FFMAX(entry_point_offsets.dims[0], 1); + else + ctx->max_entry_point_offsets = 0; + + ret = ff_v4l2_request_query_control(avctx, &slice_params); + if (!ret) + ctx->max_slice_params = FFMAX(slice_params.dims[0], 1); + else + ctx->max_slice_params = 0; + + av_log(ctx, AV_LOG_VERBOSE, "%s-based decoder with SLICE_PARAMS=%u, " + "ENTRY_POINT_OFFSETS=%u and SCALING_MATRIX=%d controls\n", + ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED ? "slice" : "frame", + ctx->max_slice_params, ctx->max_entry_point_offsets, ctx->has_scaling_matrix); + + control[0].value = ctx->decode_mode; + control[1].value = ctx->start_code; + + return ff_v4l2_request_set_controls(avctx, control, FF_ARRAY_ELEMS(control)); +} + +static int v4l2_request_hevc_init(AVCodecContext *avctx) +{ + const HEVCContext *h = avctx->priv_data; + struct v4l2_ctrl_hevc_sps sps; + + struct v4l2_ext_control control[] = { + { + .id = V4L2_CID_STATELESS_HEVC_SPS, + .ptr = &sps, + .size = sizeof(sps), + }, + }; + + fill_sps(&sps, h); + + return ff_v4l2_request_init(avctx, control, FF_ARRAY_ELEMS(control), + v4l2_request_hevc_post_frames_ctx); +} + +static int v4l2_request_hevc_frame_params(AVCodecContext *avctx, + AVBufferRef *hw_frames_ctx) +{ + const HEVCContext *h = avctx->priv_data; + const HEVCSPS *sps = h && h->pps ? h->pps->sps : NULL; + uint8_t bit_depth = sps ? sps->bit_depth : 0; + + return ff_v4l2_request_frame_params(avctx, hw_frames_ctx, + V4L2_PIX_FMT_HEVC_SLICE, bit_depth); +} + +const FFHWAccel ff_hevc_v4l2request_hwaccel = { + .p.name = "hevc_v4l2request", + .p.type = AVMEDIA_TYPE_VIDEO, + .p.id = AV_CODEC_ID_HEVC, + .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, + .start_frame = v4l2_request_hevc_start_frame, + .decode_slice = v4l2_request_hevc_decode_slice, + .end_frame = v4l2_request_hevc_end_frame, + .flush = ff_v4l2_request_flush, + .free_frame_priv = v4l2_request_hevc_free_frame_priv, + .frame_priv_data_size = sizeof(V4L2RequestControlsHEVC), + .init = v4l2_request_hevc_init, + .uninit = ff_v4l2_request_uninit, + .priv_data_size = sizeof(V4L2RequestContextHEVC), + .frame_params = v4l2_request_hevc_frame_params, + .caps_internal = HWACCEL_CAP_ASYNC_SAFE, +}; -- 2.53.0 From b53b0767bb9826422e77fecb26dd434ce333803f Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Wed, 22 May 2019 14:46:58 +0200 Subject: [PATCH 09/13] avcodec: Add V4L2 Request API vp8 hwaccel Add a V4L2 Request API hwaccel for VP8. Support for VP8 is enabled when Linux kernel headers declare the control id V4L2_CID_STATELESS_VP8_FRAME, added in v5.13. Co-developed-by: Ezequiel Garcia Signed-off-by: Ezequiel Garcia Signed-off-by: Boris Brezillon Co-developed-by: Jonas Karlman Signed-off-by: Jonas Karlman --- configure | 3 + libavcodec/Makefile | 1 + libavcodec/hwaccels.h | 1 + libavcodec/v4l2_request_vp8.c | 241 ++++++++++++++++++++++++++++++++++ libavcodec/vp8.c | 6 + 5 files changed, 252 insertions(+) create mode 100644 libavcodec/v4l2_request_vp8.c diff --git a/configure b/configure index a56fd0a16c..df6cbcfd55 100755 --- a/configure +++ b/configure @@ -3276,6 +3276,8 @@ vc1_vdpau_hwaccel_deps="vdpau" vc1_vdpau_hwaccel_select="vc1_decoder" vp8_nvdec_hwaccel_deps="nvdec" vp8_nvdec_hwaccel_select="vp8_decoder" +vp8_v4l2request_hwaccel_deps="v4l2_request vp8_v4l2_request" +vp8_v4l2request_hwaccel_select="vp8_decoder" vp8_vaapi_hwaccel_deps="vaapi" vp8_vaapi_hwaccel_select="vp8_decoder" vp9_d3d11va_hwaccel_deps="d3d11va DXVA_PicParams_VP9" @@ -7235,6 +7237,7 @@ if enabled v4l2_request; then check_cc hevc_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_HEVC_SPS" check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" + check_cc vp8_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_VP8_FRAME" check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns check_pkg_config libudev libudev libudev.h udev_new check_struct linux/videodev2.h "struct v4l2_ctrl_hevc_decode_params" num_delta_pocs_of_ref_rps_idx diff --git a/libavcodec/Makefile b/libavcodec/Makefile index d2b5831694..35aabc3ba5 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1057,6 +1057,7 @@ OBJS-$(CONFIG_VC1_QSV_HWACCEL) += qsvdec.o OBJS-$(CONFIG_VC1_VAAPI_HWACCEL) += vaapi_vc1.o OBJS-$(CONFIG_VC1_VDPAU_HWACCEL) += vdpau_vc1.o OBJS-$(CONFIG_VP8_NVDEC_HWACCEL) += nvdec_vp8.o +OBJS-$(CONFIG_VP8_V4L2REQUEST_HWACCEL) += v4l2_request_vp8.o OBJS-$(CONFIG_VP8_VAAPI_HWACCEL) += vaapi_vp8.o OBJS-$(CONFIG_VP9_D3D11VA_HWACCEL) += dxva2_vp9.o OBJS-$(CONFIG_VP9_DXVA2_HWACCEL) += dxva2_vp9.o diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h index ad145fc07c..cb2cdb3893 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h @@ -76,6 +76,7 @@ extern const struct FFHWAccel ff_vc1_nvdec_hwaccel; extern const struct FFHWAccel ff_vc1_vaapi_hwaccel; extern const struct FFHWAccel ff_vc1_vdpau_hwaccel; extern const struct FFHWAccel ff_vp8_nvdec_hwaccel; +extern const struct FFHWAccel ff_vp8_v4l2request_hwaccel; extern const struct FFHWAccel ff_vp8_vaapi_hwaccel; extern const struct FFHWAccel ff_vp9_d3d11va_hwaccel; extern const struct FFHWAccel ff_vp9_d3d11va2_hwaccel; diff --git a/libavcodec/v4l2_request_vp8.c b/libavcodec/v4l2_request_vp8.c new file mode 100644 index 0000000000..5c7b588360 --- /dev/null +++ b/libavcodec/v4l2_request_vp8.c @@ -0,0 +1,241 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include "hwaccel_internal.h" +#include "hwconfig.h" +#include "v4l2_request.h" +#include "vp8.h" + +typedef struct V4L2RequestControlsVP8 { + V4L2RequestPictureContext pic; + struct v4l2_ctrl_vp8_frame frame; +} V4L2RequestControlsVP8; + +static int v4l2_request_vp8_start_frame(AVCodecContext *avctx, + const uint8_t *buffer, + av_unused uint32_t size) +{ + const VP8Context *s = avctx->priv_data; + V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private; + struct v4l2_ctrl_vp8_frame *ctrl = &controls->frame; + unsigned int header_size = 3 + 7 * s->keyframe; + const uint8_t *data = buffer + header_size; + int ret, i, j, k; + + ret = ff_v4l2_request_start_frame(avctx, &controls->pic, + s->framep[VP8_FRAME_CURRENT]->tf.f); + if (ret) + return ret; + + *ctrl = (struct v4l2_ctrl_vp8_frame) { + .lf = { + .sharpness_level = s->filter.sharpness, + .level = s->filter.level, + }, + + .quant = { + .y_ac_qi = s->quant.yac_qi, + .y_dc_delta = s->quant.ydc_delta, + .y2_dc_delta = s->quant.y2dc_delta, + .y2_ac_delta = s->quant.y2ac_delta, + .uv_dc_delta = s->quant.uvdc_delta, + .uv_ac_delta = s->quant.uvac_delta, + }, + + .coder_state = { + .range = s->coder_state_at_header_end.range, + .value = s->coder_state_at_header_end.value, + .bit_count = s->coder_state_at_header_end.bit_count, + }, + + .width = avctx->width, + .height = avctx->height, + + .horizontal_scale = 0, /* scale not supported by FFmpeg */ + .vertical_scale = 0, /* scale not supported by FFmpeg */ + + .version = s->profile & 0x3, + .prob_skip_false = s->prob->mbskip, + .prob_intra = s->prob->intra, + .prob_last = s->prob->last, + .prob_gf = s->prob->golden, + .num_dct_parts = s->num_coeff_partitions, + + .first_part_size = s->header_partition_size, + .first_part_header_bits = (8 * (s->coder_state_at_header_end.input - data) - + s->coder_state_at_header_end.bit_count - 8), + }; + + for (i = 0; i < 4; i++) { + ctrl->segment.quant_update[i] = s->segmentation.base_quant[i]; + ctrl->segment.lf_update[i] = s->segmentation.filter_level[i]; + } + + for (i = 0; i < 3; i++) + ctrl->segment.segment_probs[i] = s->prob->segmentid[i]; + + if (s->segmentation.enabled) + ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_ENABLED; + + if (s->segmentation.update_map) + ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_UPDATE_MAP; + + if (s->segmentation.update_feature_data) + ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_UPDATE_FEATURE_DATA; + + if (!s->segmentation.absolute_vals) + ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_DELTA_VALUE_MODE; + + for (i = 0; i < 4; i++) { + ctrl->lf.ref_frm_delta[i] = s->lf_delta.ref[i]; + ctrl->lf.mb_mode_delta[i] = s->lf_delta.mode[i + MODE_I4x4]; + } + + if (s->lf_delta.enabled) + ctrl->lf.flags |= V4L2_VP8_LF_ADJ_ENABLE; + + if (s->lf_delta.update) + ctrl->lf.flags |= V4L2_VP8_LF_DELTA_UPDATE; + + if (s->filter.simple) + ctrl->lf.flags |= V4L2_VP8_LF_FILTER_TYPE_SIMPLE; + + if (s->keyframe) { + static const uint8_t keyframe_y_mode_probs[4] = { + 145, 156, 163, 128 + }; + static const uint8_t keyframe_uv_mode_probs[3] = { + 142, 114, 183 + }; + + memcpy(ctrl->entropy.y_mode_probs, keyframe_y_mode_probs, 4); + memcpy(ctrl->entropy.uv_mode_probs, keyframe_uv_mode_probs, 3); + } else { + for (i = 0; i < 4; i++) + ctrl->entropy.y_mode_probs[i] = s->prob->pred16x16[i]; + for (i = 0; i < 3; i++) + ctrl->entropy.uv_mode_probs[i] = s->prob->pred8x8c[i]; + } + for (i = 0; i < 2; i++) + for (j = 0; j < 19; j++) + ctrl->entropy.mv_probs[i][j] = s->prob->mvc[i][j]; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 8; j++) { + static const int coeff_bands_inverse[8] = { + 0, 1, 2, 3, 5, 6, 4, 15 + }; + int coeff_pos = coeff_bands_inverse[j]; + + for (k = 0; k < 3; k++) { + memcpy(ctrl->entropy.coeff_probs[i][j][k], + s->prob->token[i][coeff_pos][k], 11); + } + } + } + + for (i = 0; i < 8; i++) + ctrl->dct_part_sizes[i] = s->coeff_partition_size[i]; + + if (s->framep[VP8_FRAME_PREVIOUS]) + ctrl->last_frame_ts = + ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_PREVIOUS]->tf.f); + if (s->framep[VP8_FRAME_GOLDEN]) + ctrl->golden_frame_ts = + ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_GOLDEN]->tf.f); + if (s->framep[VP8_FRAME_ALTREF]) + ctrl->alt_frame_ts = + ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_ALTREF]->tf.f); + + if (s->keyframe) + ctrl->flags |= V4L2_VP8_FRAME_FLAG_KEY_FRAME; + + if (s->profile & 0x4) + ctrl->flags |= V4L2_VP8_FRAME_FLAG_EXPERIMENTAL; + + if (!s->invisible) + ctrl->flags |= V4L2_VP8_FRAME_FLAG_SHOW_FRAME; + + if (s->mbskip_enabled) + ctrl->flags |= V4L2_VP8_FRAME_FLAG_MB_NO_SKIP_COEFF; + + if (s->sign_bias[VP8_FRAME_GOLDEN]) + ctrl->flags |= V4L2_VP8_FRAME_FLAG_SIGN_BIAS_GOLDEN; + + if (s->sign_bias[VP8_FRAME_ALTREF]) + ctrl->flags |= V4L2_VP8_FRAME_FLAG_SIGN_BIAS_ALT; + + return 0; +} + +static int v4l2_request_vp8_decode_slice(AVCodecContext *avctx, + const uint8_t *buffer, uint32_t size) +{ + const VP8Context *s = avctx->priv_data; + V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private; + + return ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); +} + +static int v4l2_request_vp8_end_frame(AVCodecContext *avctx) +{ + const VP8Context *s = avctx->priv_data; + V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private; + + struct v4l2_ext_control control[] = { + { + .id = V4L2_CID_STATELESS_VP8_FRAME, + .ptr = &controls->frame, + .size = sizeof(controls->frame), + }, + }; + + return ff_v4l2_request_decode_frame(avctx, &controls->pic, + control, FF_ARRAY_ELEMS(control)); +} + +static int v4l2_request_vp8_init(AVCodecContext *avctx) +{ + return ff_v4l2_request_init(avctx, NULL, 0, NULL); +} + +static int v4l2_request_vp8_frame_params(AVCodecContext *avctx, + AVBufferRef *hw_frames_ctx) +{ + return ff_v4l2_request_frame_params(avctx, hw_frames_ctx, + V4L2_PIX_FMT_VP8_FRAME, 8); +} + +const FFHWAccel ff_vp8_v4l2request_hwaccel = { + .p.name = "vp8_v4l2request", + .p.type = AVMEDIA_TYPE_VIDEO, + .p.id = AV_CODEC_ID_VP8, + .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, + .start_frame = v4l2_request_vp8_start_frame, + .decode_slice = v4l2_request_vp8_decode_slice, + .end_frame = v4l2_request_vp8_end_frame, + .flush = ff_v4l2_request_flush, + .frame_priv_data_size = sizeof(V4L2RequestControlsVP8), + .init = v4l2_request_vp8_init, + .uninit = ff_v4l2_request_uninit, + .priv_data_size = sizeof(V4L2RequestContext), + .frame_params = v4l2_request_vp8_frame_params, + .caps_internal = HWACCEL_CAP_ASYNC_SAFE, +}; diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c index d6df018655..04b16cc2e3 100644 --- a/libavcodec/vp8.c +++ b/libavcodec/vp8.c @@ -184,6 +184,9 @@ static enum AVPixelFormat get_pixel_format(VP8Context *s) #endif #if CONFIG_VP8_NVDEC_HWACCEL AV_PIX_FMT_CUDA, +#endif +#if CONFIG_VP8_V4L2REQUEST_HWACCEL + AV_PIX_FMT_DRM_PRIME, #endif AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE, @@ -2980,6 +2983,9 @@ const FFCodec ff_vp8_decoder = { #endif #if CONFIG_VP8_NVDEC_HWACCEL HWACCEL_NVDEC(vp8), +#endif +#if CONFIG_VP8_V4L2REQUEST_HWACCEL + HWACCEL_V4L2REQUEST(vp8), #endif NULL }, -- 2.53.0 From 531d793c1e17d3a73cf0081ea83ae9fc6aef1e40 Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Tue, 9 Dec 2025 20:30:05 +0100 Subject: [PATCH 10/13] avcodec: Add V4L2 Request API vp9 hwaccel Add a V4L2 Request API hwaccel for VP9. Support for VP9 is enabled when Linux kernel headers declare the control id V4L2_CID_STATELESS_VP9_FRAME, added in v5.17. Signed-off-by: Boris Brezillon Co-developed-by: Jernej Skrabec Signed-off-by: Jernej Skrabec Co-developed-by: Jonas Karlman Signed-off-by: Jonas Karlman --- configure | 3 + libavcodec/Makefile | 1 + libavcodec/hwaccels.h | 1 + libavcodec/v4l2_request_vp9.c | 529 ++++++++++++++++++++++++++++++++++ libavcodec/vp9.c | 10 +- libavcodec/vp9shared.h | 1 + 6 files changed, 544 insertions(+), 1 deletion(-) create mode 100644 libavcodec/v4l2_request_vp9.c diff --git a/configure b/configure index df6cbcfd55..7ce04a1604 100755 --- a/configure +++ b/configure @@ -3290,6 +3290,8 @@ vp9_dxva2_hwaccel_deps="dxva2 DXVA_PicParams_VP9" vp9_dxva2_hwaccel_select="vp9_decoder" vp9_nvdec_hwaccel_deps="nvdec" vp9_nvdec_hwaccel_select="vp9_decoder" +vp9_v4l2request_hwaccel_deps="v4l2_request vp9_v4l2_request" +vp9_v4l2request_hwaccel_select="vp9_decoder" vp9_vaapi_hwaccel_deps="vaapi VADecPictureParameterBufferVP9_bit_depth" vp9_vaapi_hwaccel_select="vp9_decoder" vp9_vdpau_hwaccel_deps="vdpau VdpPictureInfoVP9" @@ -7238,6 +7240,7 @@ if enabled v4l2_request; then check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" check_cc vp8_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_VP8_FRAME" + check_cc vp9_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_VP9_FRAME" check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns check_pkg_config libudev libudev libudev.h udev_new check_struct linux/videodev2.h "struct v4l2_ctrl_hevc_decode_params" num_delta_pocs_of_ref_rps_idx diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 35aabc3ba5..d3225ecbef 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1063,6 +1063,7 @@ OBJS-$(CONFIG_VP9_D3D11VA_HWACCEL) += dxva2_vp9.o OBJS-$(CONFIG_VP9_DXVA2_HWACCEL) += dxva2_vp9.o OBJS-$(CONFIG_VP9_D3D12VA_HWACCEL) += dxva2_vp9.o d3d12va_vp9.o OBJS-$(CONFIG_VP9_NVDEC_HWACCEL) += nvdec_vp9.o +OBJS-$(CONFIG_VP9_V4L2REQUEST_HWACCEL) += v4l2_request_vp9.o OBJS-$(CONFIG_VP9_VAAPI_HWACCEL) += vaapi_vp9.o OBJS-$(CONFIG_VP9_VDPAU_HWACCEL) += vdpau_vp9.o OBJS-$(CONFIG_VP9_VIDEOTOOLBOX_HWACCEL) += videotoolbox_vp9.o diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h index cb2cdb3893..8571cc7e8b 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h @@ -83,6 +83,7 @@ extern const struct FFHWAccel ff_vp9_d3d11va2_hwaccel; extern const struct FFHWAccel ff_vp9_d3d12va_hwaccel; extern const struct FFHWAccel ff_vp9_dxva2_hwaccel; extern const struct FFHWAccel ff_vp9_nvdec_hwaccel; +extern const struct FFHWAccel ff_vp9_v4l2request_hwaccel; extern const struct FFHWAccel ff_vp9_vaapi_hwaccel; extern const struct FFHWAccel ff_vp9_vdpau_hwaccel; extern const struct FFHWAccel ff_vp9_videotoolbox_hwaccel; diff --git a/libavcodec/v4l2_request_vp9.c b/libavcodec/v4l2_request_vp9.c new file mode 100644 index 0000000000..e7063b7839 --- /dev/null +++ b/libavcodec/v4l2_request_vp9.c @@ -0,0 +1,529 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include "hwaccel_internal.h" +#include "hwconfig.h" +#include "internal.h" +#include "v4l2_request.h" +#include "vp89_rac.h" +#include "vp9dec.h" + +#define V4L2_VP9_CONTROLS_MAX 2 + +typedef struct V4L2RequestContextVP9 { + V4L2RequestContext base; + bool has_compressed_hdr; +} V4L2RequestContextVP9; + +typedef struct V4L2RequestControlsVP9 { + V4L2RequestPictureContext pic; + struct v4l2_ctrl_vp9_frame frame; + struct v4l2_ctrl_vp9_compressed_hdr compressed_hdr; +} V4L2RequestControlsVP9; + +// differential forward probability updates +static int read_prob_delta(VPXRangeCoder *c) +{ + static const uint8_t inv_map_table[255] = { + 7, 20, 33, 46, 59, 72, 85, 98, 111, 124, 137, 150, 163, 176, + 189, 202, 215, 228, 241, 254, 1, 2, 3, 4, 5, 6, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 112, 113, 114, 115, + 116, 117, 118, 119, 120, 121, 122, 123, 125, 126, 127, 128, 129, 130, + 131, 132, 133, 134, 135, 136, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 147, 148, 149, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, + 161, 162, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, + 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 203, 204, 205, 206, + 207, 208, 209, 210, 211, 212, 213, 214, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 233, 234, 235, 236, + 237, 238, 239, 240, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, + 252, 253, 253, + }; + int d; + + /* This code is trying to do a differential probability update. For a + * current probability A in the range [1, 255], the difference to a new + * probability of any value can be expressed differentially as 1-A, 255-A + * where some part of this (absolute range) exists both in positive as + * well as the negative part, whereas another part only exists in one + * half. We're trying to code this shared part differentially, i.e. + * times two where the value of the lowest bit specifies the sign, and + * the single part is then coded on top of this. This absolute difference + * then again has a value of [0, 254], but a bigger value in this range + * indicates that we're further away from the original value A, so we + * can code this as a VLC code, since higher values are increasingly + * unlikely. The first 20 values in inv_map_table[] allow 'cheap, rough' + * updates vs. the 'fine, exact' updates further down the range, which + * adds one extra dimension to this differential update model. */ + + if (!vp89_rac_get(c)) { + d = vp89_rac_get_uint(c, 4) + 0; + } else if (!vp89_rac_get(c)) { + d = vp89_rac_get_uint(c, 4) + 16; + } else if (!vp89_rac_get(c)) { + d = vp89_rac_get_uint(c, 5) + 32; + } else { + d = vp89_rac_get_uint(c, 7); + if (d >= 65) + d = (d << 1) - 65 + vp89_rac_get(c); + d += 64; + av_assert2(d < FF_ARRAY_ELEMS(inv_map_table)); + } + + return inv_map_table[d]; +} + +static void fill_compressed_hdr(struct v4l2_ctrl_vp9_compressed_hdr *ctrl, + const uint8_t *buffer, uint32_t size, + AVCodecContext *avctx) +{ + const VP9Context *s = avctx->priv_data; + enum CompPredMode comppredmode; + int ret, i, j, k, l, m, n; + VPXRangeCoder c; + + ret = ff_vpx_init_range_decoder(&c, buffer + s->s.h.uncompressed_header_size, + s->s.h.compressed_header_size); + if (ret < 0) + return; + + if (vpx_rac_get_prob_branchy(&c, 128)) // marker bit + return; + + // txfm updates + if (s->s.h.lossless) { + ctrl->tx_mode = V4L2_VP9_TX_MODE_ONLY_4X4; + } else { + ctrl->tx_mode = vp89_rac_get_uint(&c, 2); + if (ctrl->tx_mode == V4L2_VP9_TX_MODE_ALLOW_32X32) + ctrl->tx_mode += vp89_rac_get(&c); + + if (ctrl->tx_mode == V4L2_VP9_TX_MODE_SELECT) { + for (i = 0; i < 2; i++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->tx8[i][0] = read_prob_delta(&c); + for (i = 0; i < 2; i++) + for (j = 0; j < 2; j++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->tx16[i][j] = read_prob_delta(&c); + for (i = 0; i < 2; i++) + for (j = 0; j < 3; j++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->tx32[i][j] = read_prob_delta(&c); + } + } + + // coef updates + for (i = 0; i < 4; i++) { + if (vp89_rac_get(&c)) { + for (j = 0; j < 2; j++) + for (k = 0; k < 2; k++) + for (l = 0; l < 6; l++) + for (m = 0; m < 6; m++) { + if (m >= 3 && l == 0) // dc only has 3 pt + break; + for (n = 0; n < 3; n++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->coef[i][j][k][l][m][n] = + read_prob_delta(&c); + } + } + if (ctrl->tx_mode == i) + break; + } + + // mode updates + for (i = 0; i < 3; i++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->skip[i] = read_prob_delta(&c); + if (!s->s.h.keyframe && !s->s.h.intraonly) { + for (i = 0; i < 7; i++) + for (j = 0; j < 3; j++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->inter_mode[i][j] = read_prob_delta(&c); + + if (s->s.h.filtermode == FILTER_SWITCHABLE) + for (i = 0; i < 4; i++) + for (j = 0; j < 2; j++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->interp_filter[i][j] = read_prob_delta(&c); + + for (i = 0; i < 4; i++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->is_inter[i] = read_prob_delta(&c); + + if (s->s.h.allowcompinter) { + comppredmode = vp89_rac_get(&c); + if (comppredmode) + comppredmode += vp89_rac_get(&c); + if (comppredmode == PRED_SWITCHABLE) + for (i = 0; i < 5; i++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->comp_mode[i] = read_prob_delta(&c); + } else { + comppredmode = PRED_SINGLEREF; + } + + if (comppredmode != PRED_COMPREF) { + for (i = 0; i < 5; i++) { + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->single_ref[i][0] = read_prob_delta(&c); + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->single_ref[i][1] = read_prob_delta(&c); + } + } + + if (comppredmode != PRED_SINGLEREF) { + for (i = 0; i < 5; i++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->comp_ref[i] = read_prob_delta(&c); + } + + for (i = 0; i < 4; i++) + for (j = 0; j < 9; j++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->y_mode[i][j] = read_prob_delta(&c); + + for (i = 0; i < 4; i++) + for (j = 0; j < 4; j++) + for (k = 0; k < 3; k++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->partition[(i * 4) + j][k] = read_prob_delta(&c); + + // mv fields + for (i = 0; i < 3; i++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->mv.joint[i] = (vp89_rac_get_uint(&c, 7) << 1) | 1; + + for (i = 0; i < 2; i++) { + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->mv.sign[i] = (vp89_rac_get_uint(&c, 7) << 1) | 1; + + for (j = 0; j < 10; j++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->mv.classes[i][j] = (vp89_rac_get_uint(&c, 7) << 1) | 1; + + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->mv.class0_bit[i] = (vp89_rac_get_uint(&c, 7) << 1) | 1; + + for (j = 0; j < 10; j++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->mv.bits[i][j] = (vp89_rac_get_uint(&c, 7) << 1) | 1; + } + + for (i = 0; i < 2; i++) { + for (j = 0; j < 2; j++) + for (k = 0; k < 3; k++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->mv.class0_fr[i][j][k] = + (vp89_rac_get_uint(&c, 7) << 1) | 1; + + for (j = 0; j < 3; j++) + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->mv.fr[i][j] = (vp89_rac_get_uint(&c, 7) << 1) | 1; + } + + if (s->s.h.highprecisionmvs) { + for (i = 0; i < 2; i++) { + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->mv.class0_hp[i] = (vp89_rac_get_uint(&c, 7) << 1) | 1; + + if (vpx_rac_get_prob_branchy(&c, 252)) + ctrl->mv.hp[i] = (vp89_rac_get_uint(&c, 7) << 1) | 1; + } + } + } + + memcpy(ctrl->uv_mode, s->prob.p.uv_mode, sizeof(ctrl->uv_mode)); +} + +static void fill_frame(struct v4l2_ctrl_vp9_frame *ctrl, AVCodecContext *avctx) +{ + const VP9Context *s = avctx->priv_data; + AVFrame *ref; + int i; + + *ctrl = (struct v4l2_ctrl_vp9_frame) { + .lf = { + .level = s->s.h.filter.level, + .sharpness = s->s.h.filter.sharpness, + }, + + .quant = { + .base_q_idx = s->s.h.yac_qi, + .delta_q_y_dc = s->s.h.ydc_qdelta, + .delta_q_uv_dc = s->s.h.uvdc_qdelta, + .delta_q_uv_ac = s->s.h.uvac_qdelta, + }, + + .compressed_header_size = s->s.h.compressed_header_size, + .uncompressed_header_size = s->s.h.uncompressed_header_size, + .frame_width_minus_1 = avctx->width - 1, + .frame_height_minus_1 = avctx->height - 1, + .render_width_minus_1 = s->w - 1, + .render_height_minus_1 = s->h - 1, + .reset_frame_context = s->s.h.resetctx > 0 ? s->s.h.resetctx - 1 : 0, + .frame_context_idx = s->s.h.framectxid, + .profile = s->s.h.profile, + .bit_depth = s->s.h.bpp, + .interpolation_filter = s->s.h.filtermode ^ (s->s.h.filtermode <= 1), + .tile_cols_log2 = s->s.h.tiling.log2_tile_cols, + .tile_rows_log2 = s->s.h.tiling.log2_tile_rows, + .reference_mode = s->s.h.comppredmode, + }; + + for (i = 0; i < 4; i++) + ctrl->lf.ref_deltas[i] = s->s.h.lf_delta.ref[i]; + + for (i = 0; i < 2; i++) + ctrl->lf.mode_deltas[i] = s->s.h.lf_delta.mode[i]; + + if (s->s.h.lf_delta.enabled) + ctrl->lf.flags |= V4L2_VP9_LOOP_FILTER_FLAG_DELTA_ENABLED; + + if (s->s.h.lf_delta.updated) + ctrl->lf.flags |= V4L2_VP9_LOOP_FILTER_FLAG_DELTA_UPDATE; + + for (i = 0; i < 8; i++) { + if (s->s.h.segmentation.feat[i].q_enabled) { + ctrl->seg.feature_data[i][V4L2_VP9_SEG_LVL_ALT_Q] = + s->s.h.segmentation.feat[i].q_val; + ctrl->seg.feature_enabled[i] |= 1 << V4L2_VP9_SEG_LVL_ALT_Q; + } + + if (s->s.h.segmentation.feat[i].lf_enabled) { + ctrl->seg.feature_data[i][V4L2_VP9_SEG_LVL_ALT_L] = + s->s.h.segmentation.feat[i].lf_val; + ctrl->seg.feature_enabled[i] |= 1 << V4L2_VP9_SEG_LVL_ALT_L; + } + + if (s->s.h.segmentation.feat[i].ref_enabled) { + ctrl->seg.feature_data[i][V4L2_VP9_SEG_LVL_REF_FRAME] = + s->s.h.segmentation.feat[i].ref_val; + ctrl->seg.feature_enabled[i] |= 1 << V4L2_VP9_SEG_LVL_REF_FRAME; + } + + if (s->s.h.segmentation.feat[i].skip_enabled) + ctrl->seg.feature_enabled[i] |= 1 << V4L2_VP9_SEG_LVL_SKIP; + } + + for (i = 0; i < 7; i++) + ctrl->seg.tree_probs[i] = s->s.h.segmentation.prob[i]; + + if (s->s.h.segmentation.temporal) { + for (i = 0; i < 3; i++) + ctrl->seg.pred_probs[i] = s->s.h.segmentation.pred_prob[i]; + } else { + memset(ctrl->seg.pred_probs, 255, sizeof(ctrl->seg.pred_probs)); + } + + if (s->s.h.segmentation.enabled) + ctrl->seg.flags |= V4L2_VP9_SEGMENTATION_FLAG_ENABLED; + + if (s->s.h.segmentation.update_map) + ctrl->seg.flags |= V4L2_VP9_SEGMENTATION_FLAG_UPDATE_MAP; + + if (s->s.h.segmentation.temporal) + ctrl->seg.flags |= V4L2_VP9_SEGMENTATION_FLAG_TEMPORAL_UPDATE; + + if (s->s.h.segmentation.update_data) + ctrl->seg.flags |= V4L2_VP9_SEGMENTATION_FLAG_UPDATE_DATA; + + if (s->s.h.segmentation.absolute_vals) + ctrl->seg.flags |= V4L2_VP9_SEGMENTATION_FLAG_ABS_OR_DELTA_UPDATE; + + if (s->s.h.keyframe) + ctrl->flags |= V4L2_VP9_FRAME_FLAG_KEY_FRAME; + + if (!s->s.h.invisible) + ctrl->flags |= V4L2_VP9_FRAME_FLAG_SHOW_FRAME; + + if (s->s.h.errorres) + ctrl->flags |= V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT; + + if (s->s.h.intraonly) + ctrl->flags |= V4L2_VP9_FRAME_FLAG_INTRA_ONLY; + + if (!s->s.h.keyframe && s->s.h.highprecisionmvs) + ctrl->flags |= V4L2_VP9_FRAME_FLAG_ALLOW_HIGH_PREC_MV; + + if (s->s.h.refreshctx) + ctrl->flags |= V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX; + + if (s->s.h.parallelmode) + ctrl->flags |= V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE; + + if (s->ss_h) + ctrl->flags |= V4L2_VP9_FRAME_FLAG_X_SUBSAMPLING; + + if (s->ss_v) + ctrl->flags |= V4L2_VP9_FRAME_FLAG_Y_SUBSAMPLING; + + if (avctx->color_range == AVCOL_RANGE_JPEG) + ctrl->flags |= V4L2_VP9_FRAME_FLAG_COLOR_RANGE_FULL_SWING; + + ref = s->s.refs[s->s.h.refidx[0]].f; + if (ref && ref->private_ref) + ctrl->last_frame_ts = ff_v4l2_request_get_capture_timestamp(ref); + + ref = s->s.refs[s->s.h.refidx[1]].f; + if (ref && ref->private_ref) + ctrl->golden_frame_ts = ff_v4l2_request_get_capture_timestamp(ref); + + ref = s->s.refs[s->s.h.refidx[2]].f; + if (ref && ref->private_ref) + ctrl->alt_frame_ts = ff_v4l2_request_get_capture_timestamp(ref); + + if (s->s.h.signbias[0]) + ctrl->ref_frame_sign_bias |= V4L2_VP9_SIGN_BIAS_LAST; + + if (s->s.h.signbias[1]) + ctrl->ref_frame_sign_bias |= V4L2_VP9_SIGN_BIAS_GOLDEN; + + if (s->s.h.signbias[2]) + ctrl->ref_frame_sign_bias |= V4L2_VP9_SIGN_BIAS_ALT; +} + +static int v4l2_request_vp9_start_frame(AVCodecContext *avctx, + const uint8_t *buffer, + uint32_t size) +{ + const VP9SharedContext *h = avctx->priv_data; + const VP9Frame *f = &h->frames[CUR_FRAME]; + V4L2RequestContextVP9 *ctx = avctx->internal->hwaccel_priv_data; + V4L2RequestControlsVP9 *controls = f->hwaccel_picture_private; + int ret; + + ret = ff_v4l2_request_start_frame(avctx, &controls->pic, f->tf.f); + if (ret) + return ret; + + fill_frame(&controls->frame, avctx); + + if (ctx->has_compressed_hdr) + fill_compressed_hdr(&controls->compressed_hdr, buffer, size, avctx); + + return 0; +} + +static int v4l2_request_vp9_decode_slice(AVCodecContext *avctx, + const uint8_t *buffer, uint32_t size) +{ + const VP9SharedContext *h = avctx->priv_data; + V4L2RequestControlsVP9 *controls = h->frames[CUR_FRAME].hwaccel_picture_private; + + return ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); +} + +static int v4l2_request_vp9_end_frame(AVCodecContext *avctx) +{ + const VP9SharedContext *h = avctx->priv_data; + V4L2RequestContextVP9 *ctx = avctx->internal->hwaccel_priv_data; + V4L2RequestControlsVP9 *controls = h->frames[CUR_FRAME].hwaccel_picture_private; + int count = 0; + + struct v4l2_ext_control control[V4L2_VP9_CONTROLS_MAX] = {}; + + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_VP9_FRAME, + .ptr = &controls->frame, + .size = sizeof(controls->frame), + }; + + if (ctx->has_compressed_hdr) { + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_VP9_COMPRESSED_HDR, + .ptr = &controls->compressed_hdr, + .size = sizeof(controls->compressed_hdr), + }; + } + + return ff_v4l2_request_decode_frame(avctx, &controls->pic, control, count); +} + +static int v4l2_request_vp9_post_frames_ctx(AVCodecContext *avctx) +{ + V4L2RequestContextVP9 *ctx = avctx->internal->hwaccel_priv_data; + + struct v4l2_query_ext_ctrl compressed_hdr = { + .id = V4L2_CID_STATELESS_VP9_COMPRESSED_HDR, + }; + + // TODO: check V4L2_CID_MPEG_VIDEO_VP9_PROFILE + // TODO: check V4L2_CID_MPEG_VIDEO_VP9_LEVEL + + if (!ff_v4l2_request_query_control(avctx, &compressed_hdr)) + ctx->has_compressed_hdr = true; + else + ctx->has_compressed_hdr = false; + + return 0; +} + +static int v4l2_request_vp9_init(AVCodecContext *avctx) +{ + struct v4l2_ctrl_vp9_frame frame; + + struct v4l2_ext_control control[] = { + { + .id = V4L2_CID_STATELESS_VP9_FRAME, + .ptr = &frame, + .size = sizeof(frame), + }, + }; + + fill_frame(&frame, avctx); + + return ff_v4l2_request_init(avctx, control, FF_ARRAY_ELEMS(control), + v4l2_request_vp9_post_frames_ctx); +} + +static int v4l2_request_vp9_frame_params(AVCodecContext *avctx, + AVBufferRef *hw_frames_ctx) +{ + const VP9Context *s = avctx->priv_data; + uint8_t bit_depth = s ? s->s.h.bpp : 0; + + return ff_v4l2_request_frame_params(avctx, hw_frames_ctx, + V4L2_PIX_FMT_VP9_FRAME, bit_depth); +} + +const FFHWAccel ff_vp9_v4l2request_hwaccel = { + .p.name = "vp9_v4l2request", + .p.type = AVMEDIA_TYPE_VIDEO, + .p.id = AV_CODEC_ID_VP9, + .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, + .start_frame = v4l2_request_vp9_start_frame, + .decode_slice = v4l2_request_vp9_decode_slice, + .end_frame = v4l2_request_vp9_end_frame, + .flush = ff_v4l2_request_flush, + .frame_priv_data_size = sizeof(V4L2RequestControlsVP9), + .init = v4l2_request_vp9_init, + .uninit = ff_v4l2_request_uninit, + .priv_data_size = sizeof(V4L2RequestContextVP9), + .frame_params = v4l2_request_vp9_frame_params, + .caps_internal = HWACCEL_CAP_ASYNC_SAFE, +}; diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c index 8ede2e2eb3..6608f61fc3 100644 --- a/libavcodec/vp9.c +++ b/libavcodec/vp9.c @@ -163,6 +163,7 @@ static int update_size(AVCodecContext *avctx, int w, int h) CONFIG_VP9_D3D11VA_HWACCEL * 2 + \ CONFIG_VP9_D3D12VA_HWACCEL + \ CONFIG_VP9_NVDEC_HWACCEL + \ + CONFIG_VP9_V4L2REQUEST_HWACCEL + \ CONFIG_VP9_VAAPI_HWACCEL + \ CONFIG_VP9_VDPAU_HWACCEL + \ CONFIG_VP9_VIDEOTOOLBOX_HWACCEL) @@ -202,6 +203,9 @@ static int update_size(AVCodecContext *avctx, int w, int h) #endif #if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL *fmtp++ = AV_PIX_FMT_VIDEOTOOLBOX; +#endif +#if CONFIG_VP9_V4L2REQUEST_HWACCEL + *fmtp++ = AV_PIX_FMT_DRM_PRIME; #endif break; case AV_PIX_FMT_YUV420P12: @@ -697,7 +701,8 @@ static int decode_frame_header(AVCodecContext *avctx, get_bits(&s->gb, 8) : 255; } - if (get_bits1(&s->gb)) { + s->s.h.segmentation.update_data = get_bits1(&s->gb); + if (s->s.h.segmentation.update_data) { s->s.h.segmentation.absolute_vals = get_bits1(&s->gb); for (i = 0; i < 8; i++) { if ((s->s.h.segmentation.feat[i].q_enabled = get_bits1(&s->gb))) @@ -1870,6 +1875,9 @@ const FFCodec ff_vp9_decoder = { #endif #if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL HWACCEL_VIDEOTOOLBOX(vp9), +#endif +#if CONFIG_VP9_V4L2REQUEST_HWACCEL + HWACCEL_V4L2REQUEST(vp9), #endif NULL }, diff --git a/libavcodec/vp9shared.h b/libavcodec/vp9shared.h index 8a450c26a6..37e389e9e1 100644 --- a/libavcodec/vp9shared.h +++ b/libavcodec/vp9shared.h @@ -136,6 +136,7 @@ typedef struct VP9BitstreamHeader { uint8_t temporal; uint8_t absolute_vals; uint8_t update_map; + uint8_t update_data; uint8_t prob[7]; uint8_t pred_prob[3]; struct { -- 2.53.0 From fd7d5ba8d10e94bf73ce621d8b8a370fd3a8b515 Mon Sep 17 00:00:00 2001 From: Jonas Karlman Date: Sat, 6 Jul 2024 08:41:30 +0000 Subject: [PATCH 11/13] avcodec: Add V4L2 Request API av1 hwaccel Add a V4L2 Request API hwaccel for AV1. Support for AV1 is enabled when Linux kernel headers declare the control id V4L2_CID_STATELESS_AV1_FRAME, added in v6.5. Signed-off-by: Jonas Karlman --- configure | 3 + libavcodec/Makefile | 1 + libavcodec/av1dec.c | 10 + libavcodec/hwaccels.h | 1 + libavcodec/v4l2_request_av1.c | 635 ++++++++++++++++++++++++++++++++++ 5 files changed, 650 insertions(+) create mode 100644 libavcodec/v4l2_request_av1.c diff --git a/configure b/configure index 7ce04a1604..be92dc96e9 100755 --- a/configure +++ b/configure @@ -3172,6 +3172,8 @@ av1_dxva2_hwaccel_deps="dxva2 DXVA_PicParams_AV1" av1_dxva2_hwaccel_select="av1_decoder" av1_nvdec_hwaccel_deps="nvdec CUVIDAV1PICPARAMS" av1_nvdec_hwaccel_select="av1_decoder" +av1_v4l2request_hwaccel_deps="v4l2_request av1_v4l2_request" +av1_v4l2request_hwaccel_select="av1_decoder" av1_vaapi_hwaccel_deps="vaapi VADecPictureParameterBufferAV1_bit_depth_idx" av1_vaapi_hwaccel_select="av1_decoder" av1_vdpau_hwaccel_deps="vdpau VdpPictureInfoAV1" @@ -7235,6 +7237,7 @@ if enabled v4l2_m2m; then fi if enabled v4l2_request; then + check_cc av1_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_AV1_SEQUENCE" check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_H264_DECODE_MODE" check_cc hevc_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_HEVC_SPS" check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" diff --git a/libavcodec/Makefile b/libavcodec/Makefile index d3225ecbef..9043aab8ba 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1007,6 +1007,7 @@ OBJS-$(CONFIG_AV1_D3D11VA_HWACCEL) += dxva2_av1.o OBJS-$(CONFIG_AV1_DXVA2_HWACCEL) += dxva2_av1.o OBJS-$(CONFIG_AV1_D3D12VA_HWACCEL) += dxva2_av1.o d3d12va_av1.o OBJS-$(CONFIG_AV1_NVDEC_HWACCEL) += nvdec_av1.o +OBJS-$(CONFIG_AV1_V4L2REQUEST_HWACCEL) += v4l2_request_av1.o OBJS-$(CONFIG_AV1_VAAPI_HWACCEL) += vaapi_av1.o OBJS-$(CONFIG_AV1_VDPAU_HWACCEL) += vdpau_av1.o OBJS-$(CONFIG_AV1_VULKAN_HWACCEL) += vulkan_decode.o vulkan_av1.o diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c index 1d5b9ef4f4..a4ec92f24d 100644 --- a/libavcodec/av1dec.c +++ b/libavcodec/av1dec.c @@ -539,6 +539,7 @@ static int get_pixel_format(AVCodecContext *avctx) CONFIG_AV1_D3D11VA_HWACCEL * 2 + \ CONFIG_AV1_D3D12VA_HWACCEL + \ CONFIG_AV1_NVDEC_HWACCEL + \ + CONFIG_AV1_V4L2REQUEST_HWACCEL + \ CONFIG_AV1_VAAPI_HWACCEL + \ CONFIG_AV1_VDPAU_HWACCEL + \ CONFIG_AV1_VULKAN_HWACCEL) @@ -570,6 +571,9 @@ static int get_pixel_format(AVCodecContext *avctx) #endif #if CONFIG_AV1_VULKAN_HWACCEL *fmtp++ = AV_PIX_FMT_VULKAN; +#endif +#if CONFIG_AV1_V4L2REQUEST_HWACCEL + *fmtp++ = AV_PIX_FMT_DRM_PRIME; #endif break; case AV_PIX_FMT_YUV420P10: @@ -594,6 +598,9 @@ static int get_pixel_format(AVCodecContext *avctx) #endif #if CONFIG_AV1_VULKAN_HWACCEL *fmtp++ = AV_PIX_FMT_VULKAN; +#endif +#if CONFIG_AV1_V4L2REQUEST_HWACCEL + *fmtp++ = AV_PIX_FMT_DRM_PRIME; #endif break; case AV_PIX_FMT_YUV420P12: @@ -1597,6 +1604,9 @@ const FFCodec ff_av1_decoder = { #if CONFIG_AV1_VULKAN_HWACCEL HWACCEL_VULKAN(av1), #endif +#if CONFIG_AV1_V4L2REQUEST_HWACCEL + HWACCEL_V4L2REQUEST(av1), +#endif NULL }, diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h index 8571cc7e8b..55bc3f2509 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h @@ -24,6 +24,7 @@ extern const struct FFHWAccel ff_av1_d3d11va2_hwaccel; extern const struct FFHWAccel ff_av1_d3d12va_hwaccel; extern const struct FFHWAccel ff_av1_dxva2_hwaccel; extern const struct FFHWAccel ff_av1_nvdec_hwaccel; +extern const struct FFHWAccel ff_av1_v4l2request_hwaccel; extern const struct FFHWAccel ff_av1_vaapi_hwaccel; extern const struct FFHWAccel ff_av1_vdpau_hwaccel; extern const struct FFHWAccel ff_av1_vulkan_hwaccel; diff --git a/libavcodec/v4l2_request_av1.c b/libavcodec/v4l2_request_av1.c new file mode 100644 index 0000000000..487ce9ce50 --- /dev/null +++ b/libavcodec/v4l2_request_av1.c @@ -0,0 +1,635 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include "libavutil/mem.h" +#include "hwaccel_internal.h" +#include "hwconfig.h" +#include "av1dec.h" +#include "internal.h" +#include "v4l2_request.h" + +#define V4L2_AV1_CONTROLS_MAX 4 + +typedef struct V4L2RequestContextAV1 { + V4L2RequestContext base; + bool has_film_grain; +} V4L2RequestContextAV1; + +typedef struct V4L2RequestControlsAV1 { + V4L2RequestPictureContext pic; + struct v4l2_ctrl_av1_sequence sequence; + struct v4l2_ctrl_av1_frame frame; + struct v4l2_ctrl_av1_film_grain film_grain; + struct v4l2_ctrl_av1_tile_group_entry tile_group_entry; + struct v4l2_ctrl_av1_tile_group_entry *tile_group_entries; + unsigned int allocated_tile_group_entries; + unsigned int num_tile_group_entries; +} V4L2RequestControlsAV1; + +static int get_bit_depth_from_seq(const AV1RawSequenceHeader *seq) +{ + if (seq->seq_profile == AV_PROFILE_AV1_PROFESSIONAL && + seq->color_config.high_bitdepth) + return seq->color_config.twelve_bit ? 12 : 10; + else + return seq->color_config.high_bitdepth ? 10 : 8; +} + +static void fill_sequence(struct v4l2_ctrl_av1_sequence *ctrl, + const AV1DecContext *s) +{ + const AV1RawSequenceHeader *seq = s->raw_seq; + + *ctrl = (struct v4l2_ctrl_av1_sequence) { + .seq_profile = seq->seq_profile, + .order_hint_bits = seq->enable_order_hint ? + seq->order_hint_bits_minus_1 + 1 : 0, + .bit_depth = get_bit_depth_from_seq(seq), + .max_frame_width_minus_1 = seq->max_frame_width_minus_1, + .max_frame_height_minus_1 = seq->max_frame_height_minus_1, + }; + + if (seq->still_picture) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_STILL_PICTURE; + + if (seq->use_128x128_superblock) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK; + + if (seq->enable_filter_intra) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_FILTER_INTRA; + + if (seq->enable_intra_edge_filter) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_INTRA_EDGE_FILTER; + + if (seq->enable_interintra_compound) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_INTERINTRA_COMPOUND; + + if (seq->enable_masked_compound) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_MASKED_COMPOUND; + + if (seq->enable_warped_motion) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_WARPED_MOTION; + + if (seq->enable_dual_filter) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_DUAL_FILTER; + + if (seq->enable_order_hint) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT; + + if (seq->enable_jnt_comp) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_JNT_COMP; + + if (seq->enable_ref_frame_mvs) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_REF_FRAME_MVS; + + if (seq->enable_superres) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_SUPERRES; + + if (seq->enable_cdef) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_CDEF; + + if (seq->enable_restoration) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_RESTORATION; + + if (seq->color_config.mono_chrome) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_MONO_CHROME; + + if (seq->color_config.color_range) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_COLOR_RANGE; + + if (seq->color_config.subsampling_x) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X; + + if (seq->color_config.subsampling_y) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_Y; + + if (seq->film_grain_params_present) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_FILM_GRAIN_PARAMS_PRESENT; + + if (seq->color_config.separate_uv_delta_q) + ctrl->flags |= V4L2_AV1_SEQUENCE_FLAG_SEPARATE_UV_DELTA_Q; +} + +static void fill_frame(struct v4l2_ctrl_av1_frame *ctrl, const AV1DecContext *s) +{ + const AV1RawFrameHeader *frame_header = s->raw_frame_header; + const CodedBitstreamAV1Context *cbctx = s->cbc->priv_data; + uint8_t remap_lr_type[4] = { + V4L2_AV1_FRAME_RESTORE_NONE, + V4L2_AV1_FRAME_RESTORE_SWITCHABLE, + V4L2_AV1_FRAME_RESTORE_WIENER, + V4L2_AV1_FRAME_RESTORE_SGRPROJ, + }; + int i, j; + + *ctrl = (struct v4l2_ctrl_av1_frame) { + .tile_info = { + .context_update_tile_id = frame_header->context_update_tile_id, + .tile_cols = frame_header->tile_cols, + .tile_rows = frame_header->tile_rows, + .tile_size_bytes = frame_header->tile_cols_log2 > 0 || + frame_header->tile_rows_log2 > 0 ? + frame_header->tile_size_bytes_minus1 + 1 : 0, + }, + + .quantization = { + .base_q_idx = frame_header->base_q_idx, + .delta_q_y_dc = frame_header->delta_q_y_dc, + .delta_q_u_dc = frame_header->delta_q_u_dc, + .delta_q_u_ac = frame_header->delta_q_u_ac, + .delta_q_v_dc = frame_header->delta_q_v_dc, + .delta_q_v_ac = frame_header->delta_q_v_ac, + .qm_y = frame_header->qm_y, + .qm_u = frame_header->qm_u, + .qm_v = frame_header->qm_v, + .delta_q_res = frame_header->delta_q_res, + }, + + .loop_filter = { + .level[0] = frame_header->loop_filter_level[0], + .level[1] = frame_header->loop_filter_level[1], + .level[2] = frame_header->loop_filter_level[2], + .level[3] = frame_header->loop_filter_level[3], + .sharpness = frame_header->loop_filter_sharpness, + .mode_deltas[0] = frame_header->loop_filter_mode_deltas[0], + .mode_deltas[1] = frame_header->loop_filter_mode_deltas[1], + .delta_lf_res = frame_header->delta_lf_res, + }, + + .cdef = { + .damping_minus_3 = frame_header->cdef_damping_minus_3, + .bits = frame_header->cdef_bits, + }, + + .loop_restoration = { + .lr_unit_shift = frame_header->lr_unit_shift, + .lr_uv_shift = frame_header->lr_uv_shift, + }, + + .superres_denom = frame_header->use_superres ? + frame_header->coded_denom + AV1_SUPERRES_DENOM_MIN : + AV1_SUPERRES_NUM, + .skip_mode_frame[0] = frame_header->skip_mode_present ? + s->cur_frame.skip_mode_frame_idx[0] : 0, + .skip_mode_frame[1] = frame_header->skip_mode_present ? + s->cur_frame.skip_mode_frame_idx[1] : 0, + .primary_ref_frame = frame_header->primary_ref_frame, + .frame_type = frame_header->frame_type, + .order_hint = frame_header->order_hint, + .upscaled_width = cbctx->upscaled_width, + .interpolation_filter = frame_header->interpolation_filter, + .tx_mode = frame_header->tx_mode, + .frame_width_minus_1 = cbctx->frame_width - 1, + .frame_height_minus_1 = cbctx->frame_height - 1, + .render_width_minus_1 = cbctx->render_width - 1, + .render_height_minus_1 = cbctx->render_height - 1, + .current_frame_id = frame_header->current_frame_id, + .refresh_frame_flags = frame_header->refresh_frame_flags, + }; + + if (frame_header->segmentation_enabled) + ctrl->segmentation.flags |= V4L2_AV1_SEGMENTATION_FLAG_ENABLED; + + if (frame_header->segmentation_update_map) + ctrl->segmentation.flags |= V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; + + if (frame_header->segmentation_temporal_update) + ctrl->segmentation.flags |= V4L2_AV1_SEGMENTATION_FLAG_TEMPORAL_UPDATE; + + if (frame_header->segmentation_update_data) + ctrl->segmentation.flags |= V4L2_AV1_SEGMENTATION_FLAG_UPDATE_DATA; + + for (i = 0; i < AV1_MAX_SEGMENTS; i++) { + for (j = 0; j < AV1_SEG_LVL_MAX; j++) { + if (frame_header->feature_enabled[i][j]) { + ctrl->segmentation.feature_enabled[i] |= V4L2_AV1_SEGMENT_FEATURE_ENABLED(j); + ctrl->segmentation.last_active_seg_id = i; + if (j >= AV1_SEG_LVL_REF_FRAME) + ctrl->segmentation.flags |= V4L2_AV1_SEGMENTATION_FLAG_SEG_ID_PRE_SKIP; + } + ctrl->segmentation.feature_data[i][j] = frame_header->feature_value[i][j]; + } + } + + if (frame_header->uniform_tile_spacing_flag) + ctrl->tile_info.flags |= V4L2_AV1_TILE_INFO_FLAG_UNIFORM_TILE_SPACING; + + for (i = 0; i < frame_header->tile_cols; i++) { + ctrl->tile_info.mi_col_starts[i] = frame_header->tile_start_col_sb[i]; + ctrl->tile_info.width_in_sbs_minus_1[i] = frame_header->width_in_sbs_minus_1[i]; + } + ctrl->tile_info.mi_col_starts[i] = 2 * ((cbctx->frame_width + 7) >> 3); + + for (i = 0; i < frame_header->tile_rows; i++) { + ctrl->tile_info.mi_row_starts[i] = frame_header->tile_start_row_sb[i]; + ctrl->tile_info.height_in_sbs_minus_1[i] = frame_header->height_in_sbs_minus_1[i]; + } + ctrl->tile_info.mi_row_starts[i] = 2 * ((cbctx->frame_height + 7) >> 3); + + if (frame_header->diff_uv_delta) + ctrl->quantization.flags |= V4L2_AV1_QUANTIZATION_FLAG_DIFF_UV_DELTA; + + if (frame_header->using_qmatrix) + ctrl->quantization.flags |= V4L2_AV1_QUANTIZATION_FLAG_USING_QMATRIX; + + if (frame_header->delta_q_present) + ctrl->quantization.flags |= V4L2_AV1_QUANTIZATION_FLAG_DELTA_Q_PRESENT; + + if (frame_header->loop_filter_delta_enabled) + ctrl->loop_filter.flags |= V4L2_AV1_LOOP_FILTER_FLAG_DELTA_ENABLED; + + if (frame_header->loop_filter_delta_update) + ctrl->loop_filter.flags |= V4L2_AV1_LOOP_FILTER_FLAG_DELTA_UPDATE; + + if (frame_header->delta_lf_present) + ctrl->loop_filter.flags |= V4L2_AV1_LOOP_FILTER_FLAG_DELTA_LF_PRESENT; + + if (frame_header->delta_lf_multi) + ctrl->loop_filter.flags |= V4L2_AV1_LOOP_FILTER_FLAG_DELTA_LF_MULTI; + + for (i = 0; i < AV1_NUM_REF_FRAMES; i++) { + ctrl->loop_filter.ref_deltas[i] = frame_header->loop_filter_ref_deltas[i]; + } + + for (i = 0; i < cbctx->num_planes; i++) { + ctrl->loop_restoration.frame_restoration_type[i] = + remap_lr_type[frame_header->lr_type[i]]; + if (frame_header->lr_type[i] != AV1_RESTORE_NONE) { + ctrl->loop_restoration.flags |= V4L2_AV1_LOOP_RESTORATION_FLAG_USES_LR; + if (i > 0) + ctrl->loop_restoration.flags |= V4L2_AV1_LOOP_RESTORATION_FLAG_USES_CHROMA_LR; + } + } + if (ctrl->loop_restoration.flags & V4L2_AV1_LOOP_RESTORATION_FLAG_USES_LR) { + ctrl->loop_restoration.loop_restoration_size[0] = + 1 << (6 + frame_header->lr_unit_shift); + ctrl->loop_restoration.loop_restoration_size[1] = + 1 << (6 + frame_header->lr_unit_shift - frame_header->lr_uv_shift); + ctrl->loop_restoration.loop_restoration_size[2] = + 1 << (6 + frame_header->lr_unit_shift - frame_header->lr_uv_shift); + } + + for (i = 0; i < AV1_TOTAL_REFS_PER_FRAME; i++) { + ctrl->global_motion.type[i] = s->cur_frame.gm_type[i]; + for (j = 0; j < 6; ++j) { + ctrl->global_motion.params[i][j] = s->cur_frame.gm_params[i][j]; + if (s->cur_frame.gm_invalid[i]) + ctrl->global_motion.invalid |= V4L2_AV1_GLOBAL_MOTION_IS_INVALID(i); + } + + if (frame_header->is_global[i]) + ctrl->global_motion.flags[i] |= V4L2_AV1_GLOBAL_MOTION_FLAG_IS_GLOBAL; + + if (frame_header->is_rot_zoom[i]) + ctrl->global_motion.flags[i] |= V4L2_AV1_GLOBAL_MOTION_FLAG_IS_ROT_ZOOM; + + if (frame_header->is_translation[i]) + ctrl->global_motion.flags[i] |= V4L2_AV1_GLOBAL_MOTION_FLAG_IS_TRANSLATION; + } + + for (i = 0; i < AV1_TOTAL_REFS_PER_FRAME; i++) { + AVFrame *ref = s->ref[i].f; + + ctrl->order_hints[i] = s->cur_frame.order_hints[i]; + if (ref) + ctrl->reference_frame_ts[i] = ff_v4l2_request_get_capture_timestamp(ref); + if (i < AV1_REFS_PER_FRAME) + ctrl->ref_frame_idx[i] = frame_header->ref_frame_idx[i]; + } + + for (i = 0; i < (1 << frame_header->cdef_bits); i++) { + ctrl->cdef.y_pri_strength[i] = frame_header->cdef_y_pri_strength[i]; + ctrl->cdef.y_sec_strength[i] = frame_header->cdef_y_sec_strength[i]; + ctrl->cdef.uv_pri_strength[i] = frame_header->cdef_uv_pri_strength[i]; + ctrl->cdef.uv_sec_strength[i] = frame_header->cdef_uv_sec_strength[i]; + } + + if (frame_header->show_frame) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_SHOW_FRAME; + + if (frame_header->showable_frame) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_SHOWABLE_FRAME; + + if (frame_header->error_resilient_mode) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE; + + if (frame_header->disable_cdf_update) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_DISABLE_CDF_UPDATE; + + if (frame_header->allow_screen_content_tools) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_ALLOW_SCREEN_CONTENT_TOOLS; + + if (s->cur_frame.force_integer_mv) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_FORCE_INTEGER_MV; + + if (frame_header->allow_intrabc) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_ALLOW_INTRABC; + + if (frame_header->use_superres) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_USE_SUPERRES; + + if (frame_header->allow_high_precision_mv) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_ALLOW_HIGH_PRECISION_MV; + + if (frame_header->is_motion_mode_switchable) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_IS_MOTION_MODE_SWITCHABLE; + + if (frame_header->use_ref_frame_mvs) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS; + + if (frame_header->disable_frame_end_update_cdf) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + + if (frame_header->allow_warped_motion) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_ALLOW_WARPED_MOTION; + + if (frame_header->reference_select) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_REFERENCE_SELECT; + + if (frame_header->reduced_tx_set) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_REDUCED_TX_SET; + + if (frame_header->skip_mode_present) { + ctrl->flags |= V4L2_AV1_FRAME_FLAG_SKIP_MODE_ALLOWED; // FIXME + ctrl->flags |= V4L2_AV1_FRAME_FLAG_SKIP_MODE_PRESENT; + } + + if (frame_header->frame_size_override_flag) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_FRAME_SIZE_OVERRIDE; + + if (frame_header->buffer_removal_time_present_flag) { + ctrl->flags |= V4L2_AV1_FRAME_FLAG_BUFFER_REMOVAL_TIME_PRESENT; + for (i = 0; i < AV1_MAX_OPERATING_POINTS; i++) + ctrl->buffer_removal_time[i] = frame_header->buffer_removal_time[i]; + } + + if (frame_header->frame_refs_short_signaling) + ctrl->flags |= V4L2_AV1_FRAME_FLAG_FRAME_REFS_SHORT_SIGNALING; +} + +static void fill_film_grain(struct v4l2_ctrl_av1_film_grain *ctrl, + const AV1DecContext *s) +{ + const AV1RawFilmGrainParams *film_grain = &s->cur_frame.film_grain; + int i; + + *ctrl = (struct v4l2_ctrl_av1_film_grain) { + .cr_mult = film_grain->cr_mult, + .grain_seed = film_grain->grain_seed, + .film_grain_params_ref_idx = film_grain->film_grain_params_ref_idx, + .num_y_points = film_grain->num_y_points, + .num_cb_points = film_grain->num_cb_points, + .num_cr_points = film_grain->num_cr_points, + .grain_scaling_minus_8 = film_grain->grain_scaling_minus_8, + .ar_coeff_lag = film_grain->ar_coeff_lag, + .ar_coeff_shift_minus_6 = film_grain->ar_coeff_shift_minus_6, + .grain_scale_shift = film_grain->grain_scale_shift, + .cb_mult = film_grain->cb_mult, + .cb_luma_mult = film_grain->cb_luma_mult, + .cr_luma_mult = film_grain->cr_luma_mult, + .cb_offset = film_grain->cb_offset, + .cr_offset = film_grain->cr_offset, + }; + + if (film_grain->apply_grain) + ctrl->flags |= V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN; + + if (film_grain->update_grain) + ctrl->flags |= V4L2_AV1_FILM_GRAIN_FLAG_UPDATE_GRAIN; + + if (film_grain->chroma_scaling_from_luma) + ctrl->flags |= V4L2_AV1_FILM_GRAIN_FLAG_CHROMA_SCALING_FROM_LUMA; + + if (film_grain->overlap_flag) + ctrl->flags |= V4L2_AV1_FILM_GRAIN_FLAG_OVERLAP; + + if (film_grain->clip_to_restricted_range) + ctrl->flags |= V4L2_AV1_FILM_GRAIN_FLAG_CLIP_TO_RESTRICTED_RANGE; + + if (!film_grain->apply_grain) + return; + + for (i = 0; i < film_grain->num_y_points; i++) { + ctrl->point_y_value[i] = film_grain->point_y_value[i]; + ctrl->point_y_scaling[i] = film_grain->point_y_scaling[i]; + } + + for (i = 0; i < film_grain->num_cb_points; i++) { + ctrl->point_cb_value[i] = film_grain->point_cb_value[i]; + ctrl->point_cb_scaling[i] = film_grain->point_cb_scaling[i]; + } + + for (i = 0; i < film_grain->num_cr_points; i++) { + ctrl->point_cr_value[i] = film_grain->point_cr_value[i]; + ctrl->point_cr_scaling[i] = film_grain->point_cr_scaling[i]; + } + + for (i = 0; i < 24; i++) { + ctrl->ar_coeffs_y_plus_128[i] = film_grain->ar_coeffs_y_plus_128[i]; + } + + for (i = 0; i < 25; i++) { + ctrl->ar_coeffs_cb_plus_128[i] = film_grain->ar_coeffs_cb_plus_128[i]; + ctrl->ar_coeffs_cr_plus_128[i] = film_grain->ar_coeffs_cr_plus_128[i]; + } +} + +static int v4l2_request_av1_start_frame(AVCodecContext *avctx, + av_unused const uint8_t *buffer, + av_unused uint32_t size) +{ + const AV1DecContext *s = avctx->priv_data; + V4L2RequestContextAV1 *ctx = avctx->internal->hwaccel_priv_data; + V4L2RequestControlsAV1 *controls = s->cur_frame.hwaccel_picture_private; + int ret; + + ret = ff_v4l2_request_start_frame(avctx, &controls->pic, s->cur_frame.f); + if (ret) + return ret; + + fill_sequence(&controls->sequence, s); + fill_frame(&controls->frame, s); + + if (ctx->has_film_grain) + fill_film_grain(&controls->film_grain, s); + + controls->tile_group_entries = &controls->tile_group_entry; + controls->allocated_tile_group_entries = 0; + controls->num_tile_group_entries = 0; + + return 0; +} + +static int v4l2_request_av1_decode_slice(AVCodecContext *avctx, + const uint8_t *buffer, uint32_t size) +{ + const AV1DecContext *s = avctx->priv_data; + const AV1RawFrameHeader *fh = s->raw_frame_header; + V4L2RequestControlsAV1 *controls = s->cur_frame.hwaccel_picture_private; + + controls->num_tile_group_entries = fh->tile_cols * fh->tile_rows; + if (controls->num_tile_group_entries > V4L2_AV1_MAX_TILE_COUNT) + return AVERROR(EINVAL); + + if (controls->num_tile_group_entries > 1 && + controls->num_tile_group_entries > controls->allocated_tile_group_entries) { + struct v4l2_ctrl_av1_tile_group_entry *tile_group_entries; + + tile_group_entries = av_realloc_array(controls->allocated_tile_group_entries ? + controls->tile_group_entries : NULL, + controls->num_tile_group_entries, + sizeof(*controls->tile_group_entries)); + if (!tile_group_entries) + return AVERROR(ENOMEM); + + if (!controls->allocated_tile_group_entries) + memcpy(tile_group_entries, controls->tile_group_entries, + sizeof(*controls->tile_group_entries)); + + controls->tile_group_entries = tile_group_entries; + controls->allocated_tile_group_entries = controls->num_tile_group_entries; + } + + for (int i = 0; i < controls->num_tile_group_entries; i++) { + controls->tile_group_entries[i] = (struct v4l2_ctrl_av1_tile_group_entry) { + .tile_offset = controls->pic.output->bytesused + + s->tile_group_info[i].tile_offset, + .tile_size = s->tile_group_info[i].tile_size, + .tile_row = s->tile_group_info[i].tile_row, + .tile_col = s->tile_group_info[i].tile_column, + }; + } + + return ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); +} + +static int v4l2_request_av1_end_frame(AVCodecContext *avctx) +{ + const AV1DecContext *s = avctx->priv_data; + V4L2RequestContextAV1 *ctx = avctx->internal->hwaccel_priv_data; + V4L2RequestControlsAV1 *controls = s->cur_frame.hwaccel_picture_private; + int count = 0; + + struct v4l2_ext_control control[V4L2_AV1_CONTROLS_MAX] = {}; + + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_AV1_SEQUENCE, + .ptr = &controls->sequence, + .size = sizeof(controls->sequence), + }; + + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_AV1_FRAME, + .ptr = &controls->frame, + .size = sizeof(controls->frame), + }; + + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY, + .ptr = controls->tile_group_entries, + .size = sizeof(*controls->tile_group_entries) * + FFMAX(controls->num_tile_group_entries, 1), + }; + + if (ctx->has_film_grain) { + control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_AV1_FILM_GRAIN, + .ptr = &controls->film_grain, + .size = sizeof(controls->film_grain), + }; + } + + return ff_v4l2_request_decode_frame(avctx, &controls->pic, control, count); +} + +static void v4l2_request_av1_free_frame_priv(FFRefStructOpaque hwctx, void *data) +{ + V4L2RequestControlsAV1 *controls = data; + + if (controls->allocated_tile_group_entries) + av_freep(&controls->tile_group_entries); +} + +static int v4l2_request_av1_post_frames_ctx(AVCodecContext *avctx) +{ + V4L2RequestContextAV1 *ctx = avctx->internal->hwaccel_priv_data; + + struct v4l2_query_ext_ctrl film_grain = { + .id = V4L2_CID_STATELESS_AV1_FILM_GRAIN, + }; + + // TODO: check V4L2_CID_MPEG_VIDEO_AV1_PROFILE + // TODO: check V4L2_CID_MPEG_VIDEO_AV1_LEVEL + + if (!ff_v4l2_request_query_control(avctx, &film_grain)) + ctx->has_film_grain = true; + else + ctx->has_film_grain = false; + + return 0; +} + +static int v4l2_request_av1_init(AVCodecContext *avctx) +{ + const AV1DecContext *s = avctx->priv_data; + struct v4l2_ctrl_av1_sequence sequence; + + struct v4l2_ext_control control[] = { + { + .id = V4L2_CID_STATELESS_AV1_SEQUENCE, + .ptr = &sequence, + .size = sizeof(sequence), + }, + }; + + fill_sequence(&sequence, s); + + return ff_v4l2_request_init(avctx, control, FF_ARRAY_ELEMS(control), + v4l2_request_av1_post_frames_ctx); +} + +static int v4l2_request_av1_frame_params(AVCodecContext *avctx, + AVBufferRef *hw_frames_ctx) +{ + const AV1DecContext *s = avctx->priv_data; + const AV1RawSequenceHeader *seq = s ? s->raw_seq : NULL; + uint8_t bit_depth = seq ? get_bit_depth_from_seq(seq) : 0; + + return ff_v4l2_request_frame_params(avctx, hw_frames_ctx, + V4L2_PIX_FMT_AV1_FRAME, bit_depth); +} + +const FFHWAccel ff_av1_v4l2request_hwaccel = { + .p.name = "av1_v4l2request", + .p.type = AVMEDIA_TYPE_VIDEO, + .p.id = AV_CODEC_ID_AV1, + .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, + .start_frame = v4l2_request_av1_start_frame, + .decode_slice = v4l2_request_av1_decode_slice, + .end_frame = v4l2_request_av1_end_frame, + .flush = ff_v4l2_request_flush, + .free_frame_priv = v4l2_request_av1_free_frame_priv, + .frame_priv_data_size = sizeof(V4L2RequestControlsAV1), + .init = v4l2_request_av1_init, + .uninit = ff_v4l2_request_uninit, + .priv_data_size = sizeof(V4L2RequestContextAV1), + .frame_params = v4l2_request_av1_frame_params, + .caps_internal = HWACCEL_CAP_ASYNC_SAFE, +}; -- 2.53.0 From 00adc757b934374ed36a690ea48276acdece7971 Mon Sep 17 00:00:00 2001 From: Jernej Skrabec Date: Sun, 14 Dec 2025 01:12:06 +0000 Subject: [PATCH 12/13] avutil/hwcontext_v4l2request: Add support for AFBC_16X16_SPLIT pix fmts --- libavutil/hwcontext_v4l2request.c | 40 +++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/libavutil/hwcontext_v4l2request.c b/libavutil/hwcontext_v4l2request.c index f316f57826..9ca42623e7 100644 --- a/libavutil/hwcontext_v4l2request.c +++ b/libavutil/hwcontext_v4l2request.c @@ -79,6 +79,28 @@ static const struct { #if defined(V4L2_PIX_FMT_P010) && defined(DRM_FORMAT_P010) { V4L2_PIX_FMT_P010, AV_PIX_FMT_P010, DRM_FORMAT_P010, DRM_FORMAT_MOD_LINEAR, 10 }, #endif +#if defined(V4L2_PIX_FMT_YUV420_8_AFBC_16X16_SPLIT) + { + .pixelformat = V4L2_PIX_FMT_YUV420_8_AFBC_16X16_SPLIT, + .sw_format = AV_PIX_FMT_YUV420P, + .drm_format = DRM_FORMAT_YUV420_8BIT, + .format_modifier = DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | + AFBC_FORMAT_MOD_SPARSE | + AFBC_FORMAT_MOD_SPLIT), + .bit_depth = 8, + }, +#endif +#if defined(V4L2_PIX_FMT_YUV420_10_AFBC_16X16_SPLIT) + { + .pixelformat = V4L2_PIX_FMT_YUV420_10_AFBC_16X16_SPLIT, + .sw_format = AV_PIX_FMT_YUV420P10, + .drm_format = DRM_FORMAT_YUV420_10BIT, + .format_modifier = DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | + AFBC_FORMAT_MOD_SPARSE | + AFBC_FORMAT_MOD_SPLIT), + .bit_depth = 10, + }, +#endif }; static int v4l2request_set_drm_descriptor(AVDRMFrameDescriptor *desc, @@ -109,7 +131,7 @@ static int v4l2request_set_drm_descriptor(AVDRMFrameDescriptor *desc, } desc->nb_layers = 1; - layer->nb_planes = 2; + layer->nb_planes = 1; layer->planes[0].object_index = 0; layer->planes[0].offset = 0; @@ -117,12 +139,16 @@ static int v4l2request_set_drm_descriptor(AVDRMFrameDescriptor *desc, format->fmt.pix_mp.plane_fmt[0].bytesperline : format->fmt.pix.bytesperline; - layer->planes[1].object_index = 0; - layer->planes[1].offset = layer->planes[0].pitch * - (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? - format->fmt.pix_mp.height : - format->fmt.pix.height); - layer->planes[1].pitch = layer->planes[0].pitch; + // AFBC formats only use 1 plane, remaining use 2 planes + if ((desc->objects[0].format_modifier >> 56) != DRM_FORMAT_MOD_VENDOR_ARM) { + layer->nb_planes = 2; + layer->planes[1].object_index = 0; + layer->planes[1].offset = layer->planes[0].pitch * + (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? + format->fmt.pix_mp.height : + format->fmt.pix.height); + layer->planes[1].pitch = layer->planes[0].pitch; + } return 0; } -- 2.53.0 From 6dc680a6f22d0b1400d7361a5bccd0e5c96ed49a Mon Sep 17 00:00:00 2001 From: Jonas Karlman Date: Sun, 14 Dec 2025 01:14:17 +0000 Subject: [PATCH 13/13] avutil/hwcontext_v4l2request: Add support for BROADCOM_SAND128 pix fmts --- libavutil/hwcontext_v4l2request.c | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/libavutil/hwcontext_v4l2request.c b/libavutil/hwcontext_v4l2request.c index 9ca42623e7..3d3f395e56 100644 --- a/libavutil/hwcontext_v4l2request.c +++ b/libavutil/hwcontext_v4l2request.c @@ -101,6 +101,24 @@ static const struct { .bit_depth = 10, }, #endif +#if defined(V4L2_PIX_FMT_NV12_COL128) && defined(V4L2_PIX_FMT_NV12_10_COL128) + { + .pixelformat = V4L2_PIX_FMT_NV12_COL128, + .sw_format = AV_PIX_FMT_YUV420P, + .drm_format = DRM_FORMAT_NV12, + .format_modifier = DRM_FORMAT_MOD_BROADCOM_SAND128, + .bit_depth = 8, + }, +#if defined(DRM_FORMAT_P030) + { + .pixelformat = V4L2_PIX_FMT_NV12_10_COL128, + .sw_format = AV_PIX_FMT_YUV420P10, + .drm_format = DRM_FORMAT_P030, + .format_modifier = DRM_FORMAT_MOD_BROADCOM_SAND128, + .bit_depth = 10, + }, +#endif +#endif }; static int v4l2request_set_drm_descriptor(AVDRMFrameDescriptor *desc, @@ -150,6 +168,25 @@ static int v4l2request_set_drm_descriptor(AVDRMFrameDescriptor *desc, layer->planes[1].pitch = layer->planes[0].pitch; } +#if defined(V4L2_PIX_FMT_NV12_COL128) && defined(V4L2_PIX_FMT_NV12_10_COL128) + // Raspberry Pi formats need special handling + if (pixelformat == V4L2_PIX_FMT_NV12_COL128 || + pixelformat == V4L2_PIX_FMT_NV12_10_COL128) { + desc->objects[0].format_modifier = + DRM_FORMAT_MOD_BROADCOM_SAND128_COL_HEIGHT(layer->planes[0].pitch); + layer->planes[1].offset = 128 * + (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? + format->fmt.pix_mp.height : + format->fmt.pix.height); + layer->planes[0].pitch = (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? + format->fmt.pix_mp.width : + format->fmt.pix.width); + if (pixelformat == V4L2_PIX_FMT_NV12_10_COL128) + layer->planes[0].pitch *= 2; + layer->planes[1].pitch = layer->planes[0].pitch; + } +#endif + return 0; } -- 2.53.0