libavutil/hwcontext_vulkan.h

   1 /*
   2  * This file is part of FFmpeg.
   3  *
   4  * FFmpeg is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2.1 of the License, or (at your option) any later version.
   8  *
   9  * FFmpeg is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public
  15  * License along with FFmpeg; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  17  */
  18
  19 #ifndef AVUTIL_HWCONTEXT_VULKAN_H
  20 #define AVUTIL_HWCONTEXT_VULKAN_H
  21
  22 #if defined(_WIN32) && !defined(VK_USE_PLATFORM_WIN32_KHR)
  23 #define VK_USE_PLATFORM_WIN32_KHR
  24 #endif
  25 #include <vulkan/vulkan.h>
  26
  27 #include "pixfmt.h"
  28 #include "frame.h"
  29 #include "hwcontext.h"
  30
  31 typedef struct AVVkFrame AVVkFrame;
  32
  33 typedef struct AVVulkanDeviceQueueFamily {
  34     /* Queue family index */
  35     int idx;
  36     /* Number of queues in the queue family in use */
  37     int num;
  38     /* Queue family capabilities. Must be non-zero.
  39      * Flags may be removed to indicate the queue family may not be used
  40      * for a given purpose. */
  41     VkQueueFlagBits flags;
  42     /* Vulkan implementations are allowed to list multiple video queues
  43      * which differ in what they can encode or decode. */
  44     VkVideoCodecOperationFlagBitsKHR video_caps;
  45 } AVVulkanDeviceQueueFamily;
  46
  47 /**
  48  * @file
  49  * API-specific header for AV_HWDEVICE_TYPE_VULKAN.
  50  *
  51  * For user-allocated pools, AVHWFramesContext.pool must return AVBufferRefs
  52  * with the data pointer set to an AVVkFrame.
  53  */
  54
  55 /**
  56  * Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
  57  * All of these can be set before init to change what the context uses
  58  */
  59 typedef struct AVVulkanDeviceContext {
  60     /**
  61      * Custom memory allocator, else NULL
  62      */
  63     const VkAllocationCallbacks *alloc;
  64
  65     /**
  66      * Pointer to a vkGetInstanceProcAddr loading function.
  67      * If unset, will dynamically load and use libvulkan.
  68      */
  69     PFN_vkGetInstanceProcAddr get_proc_addr;
  70
  71     /**
  72      * Vulkan instance. Must be at least version 1.3.
  73      */
  74     VkInstance inst;
  75
  76     /**
  77      * Physical device
  78      */
  79     VkPhysicalDevice phys_dev;
  80
  81     /**
  82      * Active device
  83      */
  84     VkDevice act_dev;
  85
  86     /**
  87      * This structure should be set to the set of features that present and enabled
  88      * during device creation. When a device is created by FFmpeg, it will default to
  89      * enabling all that are present of the shaderImageGatherExtended,
  90      * fragmentStoresAndAtomics, shaderInt64 and vertexPipelineStoresAndAtomics features.
  91      */
  92     VkPhysicalDeviceFeatures2 device_features;
  93
  94     /**
  95      * Enabled instance extensions.
  96      * If supplying your own device context, set this to an array of strings, with
  97      * each entry containing the specified Vulkan extension string to enable.
  98      * Duplicates are possible and accepted.
  99      * If no extensions are enabled, set these fields to NULL, and 0 respectively.
 100      */
 101     const char * const *enabled_inst_extensions;
 102     int nb_enabled_inst_extensions;
 103
 104     /**
 105      * Enabled device extensions. By default, VK_KHR_external_memory_fd,
 106      * VK_EXT_external_memory_dma_buf, VK_EXT_image_drm_format_modifier,
 107      * VK_KHR_external_semaphore_fd and VK_EXT_external_memory_host are enabled if found.
 108      * If supplying your own device context, these fields takes the same format as
 109      * the above fields, with the same conditions that duplicates are possible
 110      * and accepted, and that NULL and 0 respectively means no extensions are enabled.
 111      */
 112     const char * const *enabled_dev_extensions;
 113     int nb_enabled_dev_extensions;
 114
 115 #if FF_API_VULKAN_FIXED_QUEUES
 116     /**
 117      * Queue family index for graphics operations, and the number of queues
 118      * enabled for it. If unavailable, will be set to -1. Not required.
 119      * av_hwdevice_create() will attempt to find a dedicated queue for each
 120      * queue family, or pick the one with the least unrelated flags set.
 121      * Queue indices here may overlap if a queue has to share capabilities.
 122      */
 123     attribute_deprecated
 124     int queue_family_index;
 125     attribute_deprecated
 126     int nb_graphics_queues;
 127
 128     /**
 129      * Queue family index for transfer operations and the number of queues
 130      * enabled. Required.
 131      */
 132     attribute_deprecated
 133     int queue_family_tx_index;
 134     attribute_deprecated
 135     int nb_tx_queues;
 136
 137     /**
 138      * Queue family index for compute operations and the number of queues
 139      * enabled. Required.
 140      */
 141     attribute_deprecated
 142     int queue_family_comp_index;
 143     attribute_deprecated
 144     int nb_comp_queues;
 145
 146     /**
 147      * Queue family index for video encode ops, and the amount of queues enabled.
 148      * If the device doesn't support such, queue_family_encode_index will be -1.
 149      * Not required.
 150      */
 151     attribute_deprecated
 152     int queue_family_encode_index;
 153     attribute_deprecated
 154     int nb_encode_queues;
 155
 156     /**
 157      * Queue family index for video decode ops, and the amount of queues enabled.
 158      * If the device doesn't support such, queue_family_decode_index will be -1.
 159      * Not required.
 160      */
 161     attribute_deprecated
 162     int queue_family_decode_index;
 163     attribute_deprecated
 164     int nb_decode_queues;
 165 #endif
 166
 167     /**
 168      * Locks a queue, preventing other threads from submitting any command
 169      * buffers to this queue.
 170      * If set to NULL, will be set to lavu-internal functions that utilize a
 171      * mutex.
 172      */
 173     void (*lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index);
 174
 175     /**
 176      * Similar to lock_queue(), unlocks a queue. Must only be called after locking.
 177      */
 178     void (*unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index);
 179
 180     /**
 181      * Queue families used. Must be preferentially ordered. List may contain
 182      * duplicates.
 183      *
 184      * For compatibility reasons, all the enabled queue families listed above
 185      * (queue_family_(tx/comp/encode/decode)_index) must also be included in
 186      * this list until they're removed after deprecation.
 187      */
 188     AVVulkanDeviceQueueFamily qf[64];
 189     int nb_qf;
 190 } AVVulkanDeviceContext;
 191
 192 /**
 193  * Defines the behaviour of frame allocation.
 194  */
 195 typedef enum AVVkFrameFlags {
 196     /* Unless this flag is set, autodetected flags will be OR'd based on the
 197      * device and tiling during av_hwframe_ctx_init(). */
 198     AV_VK_FRAME_FLAG_NONE              = (1ULL << 0),
 199
 200     /* Disables multiplane images.
 201      * This is required to export/import images from CUDA. */
 202     AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE = (1ULL << 2),
 203 } AVVkFrameFlags;
 204
 205 /**
 206  * Allocated as AVHWFramesContext.hwctx, used to set pool-specific options
 207  */
 208 typedef struct AVVulkanFramesContext {
 209     /**
 210      * Controls the tiling of allocated frames.
 211      * If left as VK_IMAGE_TILING_OPTIMAL (0), will use optimal tiling.
 212      * Can be set to VK_IMAGE_TILING_LINEAR to force linear images,
 213      * or VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT to force DMABUF-backed
 214      * images.
 215      * @note Imported frames from other APIs ignore this.
 216      */
 217     VkImageTiling tiling;
 218
 219     /**
 220      * Defines extra usage of output frames. If non-zero, all flags MUST be
 221      * supported by the VkFormat. Regardless, frames will always have the
 222      * following usage flags enabled, if supported by the format:
 223      * - VK_IMAGE_USAGE_SAMPLED_BIT
 224      * - VK_IMAGE_USAGE_STORAGE_BIT
 225      * - VK_IMAGE_USAGE_TRANSFER_SRC_BIT
 226      * - VK_IMAGE_USAGE_TRANSFER_DST_BIT
 227      */
 228     VkImageUsageFlagBits usage;
 229
 230     /**
 231      * Extension data for image creation.
 232      * If DRM tiling is used, a VkImageDrmFormatModifierListCreateInfoEXT structure
 233      * can be added to specify the exact modifier to use.
 234      *
 235      * Additional structures may be added at av_hwframe_ctx_init() time,
 236      * which will be freed automatically on uninit(), so users must only free
 237      * any structures they've allocated themselves.
 238      */
 239     void *create_pnext;
 240
 241     /**
 242      * Extension data for memory allocation. Must have as many entries as
 243      * the number of planes of the sw_format.
 244      * This will be chained to VkExportMemoryAllocateInfo, which is used
 245      * to make all pool images exportable to other APIs if the necessary
 246      * extensions are present in enabled_dev_extensions.
 247      */
 248     void *alloc_pnext[AV_NUM_DATA_POINTERS];
 249
 250     /**
 251      * A combination of AVVkFrameFlags. Unless AV_VK_FRAME_FLAG_NONE is set,
 252      * autodetected flags will be OR'd based on the device and tiling during
 253      * av_hwframe_ctx_init().
 254      */
 255     AVVkFrameFlags flags;
 256
 257     /**
 258      * Flags to set during image creation. If unset, defaults to
 259      * VK_IMAGE_CREATE_ALIAS_BIT.
 260      */
 261     VkImageCreateFlags img_flags;
 262
 263     /**
 264      * Vulkan format for each image. MUST be compatible with the pixel format.
 265      * If unset, will be automatically set.
 266      * There are at most two compatible formats for a frame - a multiplane
 267      * format, and a single-plane multi-image format.
 268      */
 269     VkFormat format[AV_NUM_DATA_POINTERS];
 270
 271     /**
 272      * Number of layers each image will have.
 273      */
 274     int nb_layers;
 275
 276     /**
 277      * Locks a frame, preventing other threads from changing frame properties.
 278      * Users SHOULD only ever lock just before command submission in order
 279      * to get accurate frame properties, and unlock immediately after command
 280      * submission without waiting for it to finish.
 281      *
 282      * If unset, will be set to lavu-internal functions that utilize a mutex.
 283      */
 284     void (*lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf);
 285
 286     /**
 287      * Similar to lock_frame(), unlocks a frame. Must only be called after locking.
 288      */
 289     void (*unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf);
 290 } AVVulkanFramesContext;
 291
 292 /*
 293  * Frame structure.
 294  *
 295  * @note the size of this structure is not part of the ABI, to allocate
 296  * you must use @av_vk_frame_alloc().
 297  */
 298 struct AVVkFrame {
 299     /**
 300      * Vulkan images to which the memory is bound to.
 301      * May be one for multiplane formats, or multiple.
 302      */
 303     VkImage img[AV_NUM_DATA_POINTERS];
 304
 305     /**
 306      * Tiling for the frame.
 307      */
 308     VkImageTiling tiling;
 309
 310     /**
 311      * Memory backing the images. Either one, or as many as there are planes
 312      * in the sw_format.
 313      * In case of having multiple VkImages, but one memory, the offset field
 314      * will indicate the bound offset for each image.
 315      */
 316     VkDeviceMemory mem[AV_NUM_DATA_POINTERS];
 317     size_t size[AV_NUM_DATA_POINTERS];
 318
 319     /**
 320      * OR'd flags for all memory allocated
 321      */
 322     VkMemoryPropertyFlagBits flags;
 323
 324     /**
 325      * Updated after every barrier. One per VkImage.
 326      */
 327     VkAccessFlagBits access[AV_NUM_DATA_POINTERS];
 328     VkImageLayout layout[AV_NUM_DATA_POINTERS];
 329
 330     /**
 331      * Synchronization timeline semaphores, one for each VkImage.
 332      * Must not be freed manually. Must be waited on at every submission using
 333      * the value in sem_value, and must be signalled at every submission,
 334      * using an incremented value.
 335      */
 336     VkSemaphore sem[AV_NUM_DATA_POINTERS];
 337
 338     /**
 339      * Up to date semaphore value at which each image becomes accessible.
 340      * One per VkImage.
 341      * Clients must wait on this value when submitting a command queue,
 342      * and increment it when signalling.
 343      */
 344     uint64_t sem_value[AV_NUM_DATA_POINTERS];
 345
 346     /**
 347      * Internal data.
 348      */
 349     struct AVVkFrameInternal *internal;
 350
 351     /**
 352      * Describes the binding offset of each image to the VkDeviceMemory.
 353      * One per VkImage.
 354      */
 355     ptrdiff_t offset[AV_NUM_DATA_POINTERS];
 356
 357     /**
 358      * Queue family of the images. Must be VK_QUEUE_FAMILY_IGNORED if
 359      * the image was allocated with the CONCURRENT concurrency option.
 360      * One per VkImage.
 361      */
 362     uint32_t queue_family[AV_NUM_DATA_POINTERS];
 363 };
 364
 365 /**
 366  * Allocates a single AVVkFrame and initializes everything as 0.
 367  * @note Must be freed via av_free()
 368  */
 369 AVVkFrame *av_vk_frame_alloc(void);
 370
 371 /**
 372  * Returns the optimal per-plane Vulkan format for a given sw_format,
 373  * one for each plane.
 374  * Returns NULL on unsupported formats.
 375  */
 376 const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p);
 377
 378 #endif /* AVUTIL_HWCONTEXT_VULKAN_H */