Package org.lwjgl.vulkan

Class NVOpticalFlow

java.lang.Object
org.lwjgl.vulkan.NVOpticalFlow
public class NVOpticalFlow extends Object
Optical flow are fundamental algorithms in computer vision (CV) area. This extension allows applications to estimate 2D displacement of pixels between two frames.
Note

This extension is designed to be used with upcoming NVIDIA Optical Flow SDK Version 5 which will be available on NVIDIA Developer webpage.

Examples

 // Example querying available input formats
 VkOpticalFlowImageFormatInfoNV ofFormatInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_INFO_NV };
 ofFormatInfo.usage = VK_OPTICAL_FLOW_USAGE_INPUT_BIT_NV;
 
 uint32_t count = 0;
 vkGetPhysicalDeviceOpticalFlowImageFormatsNV(physicalDevice, &ofFormatInfo, &count, NULL);
 VkOpticalFlowImageFormatPropertiesNV* fmt = new VkOpticalFlowImageFormatPropertiesNV[count];
 memset(fmt, 0, count  * sizeof(VkOpticalFlowImageFormatPropertiesNV));
 for (uint32_t i = 0; i < count; i++) {
     fmt[i].sType = VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_PROPERTIES_NV;
 }
 vkGetPhysicalDeviceOpticalFlowImageFormatsNV(physicalDevice, &ofFormatInfo, &count, fmt);
 
 // Pick one of the available formats
 VkFormat inputFormat = fmt[0].format;
 
 // Check feature support for optimal tiling
 VkFormatProperties3 formatProperties3 = { VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3 };
 VkFormatProperties2 formatProperties2 = { VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2, &formatProperties3 };
 vkGetPhysicalDeviceFormatProperties2(physicalDevice, inputFormat, &formatProperties2);
 if (!(formatProperties3.optimalTilingFeatures & VK_FORMAT_FEATURE_2_OPTICAL_FLOW_IMAGE_BIT_NV)) {
     return false;
 }
 
 // Check support for image creation parameters
 VkPhysicalDeviceImageFormatInfo2 imageFormatInfo2 = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, &ofFormatInfo };
 imageFormatInfo2.format = inputFormat;
 imageFormatInfo2.type = VK_IMAGE_TYPE_2D;
 imageFormatInfo2.tiling = VK_IMAGE_TILING_OPTIMAL;
 imageFormatInfo2.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
 
 VkImageFormatProperties2 imageFormatProperties2 = { VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2 };
 if (vkGetPhysicalDeviceImageFormatProperties2(physicalDevice, &imageFormatInfo2, &imageFormatProperties2) != VK_SUCCESS) {
     return false;
 }
 
 VkImageCreateInfo imageCreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, &ofFormatInfo };
 imageCreateInfo.imageType = VK_IMAGE_TYPE_2D;
 imageCreateInfo.format = inputFormat;
 imageCreateInfo.extent = { width, height, (uint32_t)1};
 imageCreateInfo.mipLevels = 1;
 imageCreateInfo.arrayLayers = 1;
 imageCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT;
 imageCreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;;
 imageCreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
 
 vkCreateImage(device, &imageCreateInfo, NULL, &input);
 "allocate memory, bind image, create view"
 
 "do the same for reference and output"
 
 // Create optical flow session
 VkOpticalFlowSessionCreateInfoNV sessionCreateInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_SESSION_CREATE_INFO_NV };
 sessionCreateInfo.width = width;
 sessionCreateInfo.height = height;
 sessionCreateInfo.imageFormat = inputFormat;
 sessionCreateInfo.flowVectorFormat = outputFormat;
 sessionCreateInfo.outputGridSize = VK_OPTICAL_FLOW_GRID_SIZE_4X4_BIT_NV;
 sessionCreateInfo.performanceLevel = VK_OPTICAL_FLOW_PERFORMANCE_LEVEL_SLOW_NV;
 VkOpticalFlowSessionNV session;
 vkCreateOpticalFlowSessionNV(device, &sessionCreateInfo, NULL, &session);
 
 "allocate command buffer"
 
 "transfer images to VK_PIPELINE_STAGE_2_OPTICAL_FLOW_BIT_NV"
 "transfer input images to VK_ACCESS_2_OPTICAL_FLOW_READ_BIT_NV and output image to VK_ACCESS_2_OPTICAL_FLOW_WRITE_BIT_NV"
 
 vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_INPUT_NV, inputView, VK_IMAGE_LAYOUT_GENERAL);
 vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_REFERENCE_NV, refView, VK_IMAGE_LAYOUT_GENERAL);
 vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_FLOW_VECTOR_NV, outputView, VK_IMAGE_LAYOUT_GENERAL);
 
 VkOpticalFlowExecuteInfoNV opticalFlowExecuteInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_EXECUTE_INFO_NV };
 vkCmdOpticalFlowExecuteNV(cmd, session, &opticalFlowExecuteInfo);
 
 "submit command buffer"
Name String
VK_NV_optical_flow
Extension Type
Device extension
Registered Extension Number
465
Revision
1
Extension and Version Dependencies
VK_KHR_get_physical_device_properties2 or Version 1.1 and VK_KHR_format_feature_flags2 and VK_KHR_synchronization2 or Version 1.3
Contact
Other Extension Metadata
Last Modified Date
2022-09-26
Contributors
  • Carsten Rohde, NVIDIA
  • Vipul Parashar, NVIDIA
  • Jeff Bolz, NVIDIA
  • Eric Werness, NVIDIA