From bb3e6f8f20adcacc3c319bc944ec9276415599cc Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Mon, 12 Jun 2023 10:39:42 +0200 Subject: [PATCH 01/53] Add a new sample - subgroups operation --- samples/CMakeLists.txt | 2 + .../subgroups_operations/CMakeLists.txt | 31 ++ .../extensions/subgroups_operations/README.md | 51 ++ .../subgroups_operations.cpp | 503 ++++++++++++++++++ .../subgroups_operations.h | 134 +++++ shaders/subgroups_operations/base.frag | 28 + shaders/subgroups_operations/base.vert | 35 ++ shaders/subgroups_operations/blur.comp | 67 +++ 8 files changed, 851 insertions(+) create mode 100644 samples/extensions/subgroups_operations/CMakeLists.txt create mode 100644 samples/extensions/subgroups_operations/README.md create mode 100644 samples/extensions/subgroups_operations/subgroups_operations.cpp create mode 100644 samples/extensions/subgroups_operations/subgroups_operations.h create mode 100644 shaders/subgroups_operations/base.frag create mode 100644 shaders/subgroups_operations/base.vert create mode 100644 shaders/subgroups_operations/blur.comp diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index e5d79a834..6fa328f1f 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -86,6 +86,8 @@ set(ORDER_LIST "fragment_shader_barycentric" "gshader_to_mshader" "color_write_enable" + "subgroups_operations" + #Performance Samples "swapchain_images" diff --git a/samples/extensions/subgroups_operations/CMakeLists.txt b/samples/extensions/subgroups_operations/CMakeLists.txt new file mode 100644 index 000000000..fe4d63f95 --- /dev/null +++ b/samples/extensions/subgroups_operations/CMakeLists.txt @@ -0,0 +1,31 @@ + # Copyright (c) 2023, Mobica Limited + # + # SPDX-License-Identifier: Apache-2.0 + # + # Licensed under the Apache License, Version 2.0 the "License"; + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + +get_filename_component(FOLDER_NAME ${CMAKE_CURRENT_LIST_DIR} NAME) +get_filename_component(PARENT_DIR ${CMAKE_CURRENT_LIST_DIR} PATH) +get_filename_component(CATEGORY_NAME ${PARENT_DIR} NAME) + +add_sample( + ID ${FOLDER_NAME} + CATEGORY ${CATEGORY_NAME} + AUTHOR "Mobica" + NAME "subgroups_operations" + DESCRIPTION "Demonstrates the use of a subgroups feature" + SHADER_FILES_GLSL + "subgroups_operations/base.vert" + "subgroups_operations/base.frag" + "subgroups_operations/blur.comp") diff --git a/samples/extensions/subgroups_operations/README.md b/samples/extensions/subgroups_operations/README.md new file mode 100644 index 000000000..0c5fabba5 --- /dev/null +++ b/samples/extensions/subgroups_operations/README.md @@ -0,0 +1,51 @@ + + +# Subgroups Operations + + +## Overview + +This sample demonstrates how to use and enable the Subgroups operations feature. + + +## GLSL Shaders +In shader enable `GL_KHR_shader_subgroup_basic` extension. +TODO: add more description + + +## Enabling the Feature + +To enable the subgroups feature in the Vulkan API you must create a Vulkan instance with a minimum version of version 1.1. Enable the extension `VK_EXT_subgroup_size_control` and get the subgroup properties of a physical device +```C++ +VkPhysicalDeviceSubgroupProperties subgroups_properties; +subgroups_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; +subgroups_properties.pNext = VK_NULL_HANDLE; + +VkPhysicalDeviceProperties2 device_properties2 = {}; +device_properties2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; +device_properties2.pNext = &subgroups_properties; +vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); +``` +TODO: add more description + +## Additional information + +More about subgroups you can read [in this article](https://www.khronos.org/blog/vulkan-subgroup-tutorial). +Documentation: https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkPhysicalDeviceSubgroupProperties.html diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp new file mode 100644 index 000000000..55e210ee4 --- /dev/null +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -0,0 +1,503 @@ +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "subgroups_operations.h" + +void SubgroupsOperations::Pipeline::destroy(VkDevice device) +{ + vkDestroyPipeline(device, pipeline, nullptr); + vkDestroyPipelineLayout(device, pipeline_layout, nullptr); +} + +SubgroupsOperations::SubgroupsOperations() +{ + set_api_version(VK_API_VERSION_1_1); + title = "Subgroups operations"; + camera.type = vkb::CameraType::LookAt; + + camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); + camera.set_position({0.0f, 0.0f, -2.0f}); + add_device_extension(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); + add_device_extension(VK_EXT_SHADER_SUBGROUP_BALLOT_EXTENSION_NAME); // is needed??? + add_device_extension(VK_EXT_SHADER_SUBGROUP_VOTE_EXTENSION_NAME); // is needed??? +} + +SubgroupsOperations::~SubgroupsOperations() +{ + if (device) + { + compute.pipelines._default.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); + vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); + vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); + + texture_object.pipeline.destroy(get_device().get_handle()); + vkDestroySemaphore(get_device().get_handle(), texture_object.semaphore, nullptr); + } +} + +bool SubgroupsOperations::prepare(vkb::Platform &platform) +{ + if (!ApiVulkanSample::prepare(platform)) + { + return false; + } + + load_assets(); + setup_descriptor_pool(); + prepare_compute(); + prepare_graphics(); + + prepared = true; + return true; +} + +void SubgroupsOperations::prepare_compute() +{ + create_compute_queue(); + create_compute_command_pool(); + create_compute_command_buffer(); + create_compute_descriptor_set_layout(); + create_compute_descriptor_set(); + preapre_compute_pipeline_layout(); + prepare_compute_pipeline(); + build_compute_command_buffer(); +} + +void SubgroupsOperations::create_compute_queue() +{ + // create compute queue and get family index + compute.queue_family_index = get_device().get_queue_family_index(VK_QUEUE_COMPUTE_BIT); + + vkGetDeviceQueue(get_device().get_handle(), compute.queue_family_index, 0u, &compute.queue); +} + +void SubgroupsOperations::create_compute_command_pool() +{ + VkCommandPoolCreateInfo command_pool_create_info = {}; + command_pool_create_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + command_pool_create_info.queueFamilyIndex = compute.queue_family_index; + command_pool_create_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + VK_CHECK(vkCreateCommandPool(get_device().get_handle(), &command_pool_create_info, nullptr, &compute.command_pool)); +} + +void SubgroupsOperations::create_compute_command_buffer() +{ + // Create a command buffer for compute operations + VkCommandBufferAllocateInfo command_buffer_allocate_info = + vkb::initializers::command_buffer_allocate_info( + compute.command_pool, + VK_COMMAND_BUFFER_LEVEL_PRIMARY, + 1u); + + VK_CHECK(vkAllocateCommandBuffers(get_device().get_handle(), &command_buffer_allocate_info, &compute.command_buffer)); + + // Semaphore for compute & graphics sync + VkSemaphoreCreateInfo semaphore_create_info = vkb::initializers::semaphore_create_info(); + VK_CHECK(vkCreateSemaphore(get_device().get_handle(), &semaphore_create_info, nullptr, &compute.semaphore)); +} + +void SubgroupsOperations::create_compute_descriptor_set_layout() +{ + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0), // input image + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1), // result image + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2) // kernel matrix + }; + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); +} + +void SubgroupsOperations::create_compute_descriptor_set() +{ + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1); + + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); +} + +void SubgroupsOperations::preapre_compute_pipeline_layout() +{ + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &compute.descriptor_set_layout; + + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &compute.pipelines._default.pipeline_layout)); +} + +void SubgroupsOperations::prepare_compute_pipeline() +{ + VkComputePipelineCreateInfo computeInfo = {}; + computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + computeInfo.layout = compute.pipelines._default.pipeline_layout; + computeInfo.stage = load_shader("subgroups_operations/blur.comp", VK_SHADER_STAGE_COMPUTE_BIT); + + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines._default.pipeline)); +} + +void SubgroupsOperations::build_compute_command_buffer() +{ + VkSubmitInfo submit_info = {}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + vkResetCommandBuffer(compute.command_buffer, 0u); + + // record command + VkCommandBufferBeginInfo begin_info = {}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); + + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, texture_object.texture.image->get_extent().width / 32u, texture_object.texture.image->get_extent().height / 32u, 1u); + + VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); +} + +void SubgroupsOperations::prepare_graphics() +{ + prepare_uniform_buffers(); + setup_descriptor_set_layout(); + setup_pipelines(); + create_semaphore(); + setup_descriptor_set(); + build_command_buffers(); +} + +void SubgroupsOperations::create_semaphore() +{ + VkSemaphoreCreateInfo semaphore_create_info = vkb::initializers::semaphore_create_info(); + VK_CHECK(vkCreateSemaphore(get_device().get_handle(), &semaphore_create_info, nullptr, &texture_object.semaphore)); +} + +void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) +{ + if (gpu.get_features().samplerAnisotropy) + { + gpu.get_mutable_requested_features().samplerAnisotropy = VK_TRUE; + } + subgroups_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; + subgroups_properties.pNext = VK_NULL_HANDLE; + + VkPhysicalDeviceProperties2 device_properties2 = {}; + device_properties2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + device_properties2.pNext = &subgroups_properties; + vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); +} + +void SubgroupsOperations::prepare_uniform_buffers() +{ + texture_uniform_buffer = std::make_unique(get_device(), sizeof(texture_ubo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + + update_uniform_buffers(); +} + +void SubgroupsOperations::generate_quad() +{ + std::vector vertices = + { + {{1.0f, 1.0f, 0.0f}, {1.0f, 1.0f}}, + {{-1.0f, 1.0f, 0.0f}, {0.0f, 1.0f}}, + {{-1.0f, -1.0f, 0.0f}, {0.0f, 0.0f}}, + {{1.0f, -1.0f, 0.0f}, {1.0f, 0.0f}}}; + std::vector indices = {0u, 1u, 2u, 2u, 3u, 0u}; + texture_object.buffers.index_count = static_cast(indices.size()); + auto vertex_buffer_size = vkb::to_u32(vertices.size() * sizeof(TextureQuadVertex)); + auto index_buffer_size = vkb::to_u32(indices.size() * sizeof(uint32_t)); + texture_object.buffers.vertex = std::make_unique(get_device(), + vertex_buffer_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); + texture_object.buffers.vertex->update(vertices.data(), vertex_buffer_size); + texture_object.buffers.index = std::make_unique(get_device(), + index_buffer_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); + texture_object.buffers.index->update(indices.data(), index_buffer_size); +} + +void SubgroupsOperations::update_uniform_buffers() +{ + texture_ubo.model = glm::mat4(1.0f); + texture_ubo.model = glm::translate(texture_ubo.model, glm::vec3(0.0f)); + texture_ubo.view = camera.matrices.view; + texture_ubo.projection = camera.matrices.perspective; + + texture_uniform_buffer->convert_and_update(texture_ubo); +} + +void SubgroupsOperations::setup_pipelines() +{ + VkPipelineInputAssemblyStateCreateInfo input_assembly_state = + vkb::initializers::pipeline_input_assembly_state_create_info( + VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, + 0u, + VK_FALSE); + VkPipelineRasterizationStateCreateInfo rasterization_state = + vkb::initializers::pipeline_rasterization_state_create_info( + VK_POLYGON_MODE_FILL, + VK_CULL_MODE_NONE, + VK_FRONT_FACE_COUNTER_CLOCKWISE, + 0u); + VkPipelineColorBlendAttachmentState blend_attachment_state = + vkb::initializers::pipeline_color_blend_attachment_state( + 0xf, + VK_FALSE); + VkPipelineColorBlendStateCreateInfo color_blend_state = + vkb::initializers::pipeline_color_blend_state_create_info( + 1u, + &blend_attachment_state); + VkPipelineDepthStencilStateCreateInfo depth_stencil_state = + vkb::initializers::pipeline_depth_stencil_state_create_info( + VK_TRUE, + VK_TRUE, + VK_COMPARE_OP_GREATER); + VkPipelineViewportStateCreateInfo viewport_state = + vkb::initializers::pipeline_viewport_state_create_info(1u, 1u, 0u); + VkPipelineMultisampleStateCreateInfo multisample_state = + vkb::initializers::pipeline_multisample_state_create_info( + VK_SAMPLE_COUNT_1_BIT, + 0u); + std::vector dynamic_state_enables = { + VK_DYNAMIC_STATE_VIEWPORT, + VK_DYNAMIC_STATE_SCISSOR}; + VkPipelineDynamicStateCreateInfo dynamic_state = + vkb::initializers::pipeline_dynamic_state_create_info( + dynamic_state_enables.data(), + static_cast(dynamic_state_enables.size()), + 0u); + std::array shader_stages; + shader_stages[0] = load_shader("subgroups_operations/base.vert", VK_SHADER_STAGE_VERTEX_BIT); + shader_stages[1] = load_shader("subgroups_operations/base.frag", VK_SHADER_STAGE_FRAGMENT_BIT); + const std::vector vertex_input_bindings = { + vkb::initializers::vertex_input_binding_description(0u, sizeof(TextureQuadVertex), VK_VERTEX_INPUT_RATE_VERTEX), + }; + const std::vector vertex_input_attributes = { + vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(TextureQuadVertex, pos)), + vkb::initializers::vertex_input_attribute_description(0u, 1u, VK_FORMAT_R32G32_SFLOAT, offsetof(TextureQuadVertex, uv)), + }; + VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); + vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); + vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); + vertex_input_state.vertexAttributeDescriptionCount = static_cast(vertex_input_attributes.size()); + vertex_input_state.pVertexAttributeDescriptions = vertex_input_attributes.data(); + VkGraphicsPipelineCreateInfo pipeline_create_info = + vkb::initializers::pipeline_create_info( + texture_object.pipeline.pipeline_layout, + render_pass, + 0u); + pipeline_create_info.pVertexInputState = &vertex_input_state; + pipeline_create_info.pInputAssemblyState = &input_assembly_state; + pipeline_create_info.pRasterizationState = &rasterization_state; + pipeline_create_info.pColorBlendState = &color_blend_state; + pipeline_create_info.pMultisampleState = &multisample_state; + pipeline_create_info.pViewportState = &viewport_state; + pipeline_create_info.pDepthStencilState = &depth_stencil_state; + pipeline_create_info.pDynamicState = &dynamic_state; + pipeline_create_info.stageCount = static_cast(shader_stages.size()); + pipeline_create_info.pStages = shader_stages.data(); + VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &texture_object.pipeline.pipeline)); +} + +void SubgroupsOperations::setup_descriptor_pool() +{ + std::vector pool_sizes = { + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1u)}; + VkDescriptorPoolCreateInfo descriptor_pool_create_info = + vkb::initializers::descriptor_pool_create_info( + static_cast(pool_sizes.size()), + pool_sizes.data(), + 2u); + VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); +} + +void SubgroupsOperations::setup_descriptor_set_layout() +{ + std::vector set_layout_bindings = + { + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + VK_SHADER_STAGE_VERTEX_BIT, + 0u), + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + VK_SHADER_STAGE_FRAGMENT_BIT, + 1u)}; + VkDescriptorSetLayoutCreateInfo descriptor_layout = + vkb::initializers::descriptor_set_layout_create_info( + set_layout_bindings.data(), + static_cast(set_layout_bindings.size())); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &texture_object.descriptor_set_layout)); + VkPipelineLayoutCreateInfo pipeline_layout_create_info = + vkb::initializers::pipeline_layout_create_info( + &texture_object.descriptor_set_layout, + 1u); + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &pipeline_layout_create_info, nullptr, &texture_object.pipeline.pipeline_layout)); +} + +void SubgroupsOperations::setup_descriptor_set() +{ + VkDescriptorSetAllocateInfo alloc_info = + vkb::initializers::descriptor_set_allocate_info( + descriptor_pool, + &texture_object.descriptor_set_layout, + 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &texture_object.descriptor_set)); + VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*texture_uniform_buffer); + VkDescriptorImageInfo image_descriptor = create_descriptor(texture_object.texture); + std::vector write_descriptor_sets = + { + vkb::initializers::write_descriptor_set( + texture_object.descriptor_set, + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + 0u, + &buffer_descriptor), + vkb::initializers::write_descriptor_set( + texture_object.descriptor_set, + VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + 1u, + &image_descriptor)}; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, NULL); +} + +void SubgroupsOperations::load_assets() +{ + texture_object.texture = load_texture("textures/vulkan_logo_full.ktx", vkb::sg::Image::Color); + generate_quad(); +} + +void SubgroupsOperations::build_command_buffers() +{ + VkCommandBufferBeginInfo command_buffer_begin_info = vkb::initializers::command_buffer_begin_info(); + + std::array clear_values; + clear_values[0].color = {{0.0f, 0.0f, 0.0f, 0.0f}}; + clear_values[1].depthStencil = {0.0f, 0u}; + + VkRenderPassBeginInfo render_pass_begin_info = vkb::initializers::render_pass_begin_info(); + render_pass_begin_info.renderPass = render_pass; + render_pass_begin_info.renderArea.extent.width = width; + render_pass_begin_info.renderArea.extent.height = height; + render_pass_begin_info.clearValueCount = static_cast(clear_values.size()); + render_pass_begin_info.pClearValues = clear_values.data(); + + for (uint32_t i = 0u; i < draw_cmd_buffers.size(); ++i) + { + render_pass_begin_info.framebuffer = framebuffers[i]; + + VK_CHECK(vkBeginCommandBuffer(draw_cmd_buffers[i], &command_buffer_begin_info)); + + vkCmdBeginRenderPass(draw_cmd_buffers[i], &render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); + + VkViewport viewport = vkb::initializers::viewport(static_cast(width), static_cast(height), 0.0f, 1.0f); + vkCmdSetViewport(draw_cmd_buffers[i], 0u, 1u, &viewport); + + VkRect2D scissor = vkb::initializers::rect2D(width, height, 0, 0); + vkCmdSetScissor(draw_cmd_buffers[i], 0u, 1u, &scissor); + + // draw texture + { + vkCmdBindDescriptorSets(draw_cmd_buffers[i], VK_PIPELINE_BIND_POINT_GRAPHICS, texture_object.pipeline.pipeline_layout, 0, 1, &texture_object.descriptor_set, 0, nullptr); + vkCmdBindPipeline(draw_cmd_buffers[i], VK_PIPELINE_BIND_POINT_GRAPHICS, texture_object.pipeline.pipeline); + + VkDeviceSize offset[] = {0}; + vkCmdBindVertexBuffers(draw_cmd_buffers[i], 0, 1, texture_object.buffers.vertex->get(), offset); + vkCmdBindIndexBuffer(draw_cmd_buffers[i], texture_object.buffers.index->get_handle(), 0, VK_INDEX_TYPE_UINT32); + + vkCmdDrawIndexed(draw_cmd_buffers[i], texture_object.buffers.index_count, 1u, 0u, 0u, 0u); + } + + draw_ui(draw_cmd_buffers[i]); + + vkCmdEndRenderPass(draw_cmd_buffers[i]); + + VK_CHECK(vkEndCommandBuffer(draw_cmd_buffers[i])); + } +} + +void SubgroupsOperations::draw() +{ + VkPipelineStageFlags graphics_wait_stage_masks[] = {VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT}; + VkSemaphore graphics_wait_semaphores[] = {compute.semaphore, semaphores.acquired_image_ready}; + VkSemaphore graphics_signal_semaphores[] = {texture_object.semaphore, semaphores.render_complete}; + + ApiVulkanSample::prepare_frame(); + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &draw_cmd_buffers[current_buffer]; + submit_info.waitSemaphoreCount = 2; + submit_info.pWaitSemaphores = graphics_wait_semaphores; + submit_info.pWaitDstStageMask = graphics_wait_stage_masks; + submit_info.signalSemaphoreCount = 2; + submit_info.pSignalSemaphores = graphics_signal_semaphores; + VK_CHECK(vkQueueSubmit(queue, 1u, &submit_info, VK_NULL_HANDLE)); + ApiVulkanSample::submit_frame(); + + // Wait for rendering finished + VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; + + // Submit compute commands + VkSubmitInfo compute_submit_info = vkb::initializers::submit_info(); + compute_submit_info.commandBufferCount = 1u; + compute_submit_info.pCommandBuffers = &compute.command_buffer; + compute_submit_info.waitSemaphoreCount = 1u; + compute_submit_info.pWaitSemaphores = &texture_object.semaphore; + compute_submit_info.pWaitDstStageMask = &wait_stage_mask; + compute_submit_info.signalSemaphoreCount = 1u; + compute_submit_info.pSignalSemaphores = &compute.semaphore; + + VK_CHECK(vkQueueSubmit(compute.queue, 1u, &compute_submit_info, VK_NULL_HANDLE)); +} + +void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) +{ + if (drawer.header("Settings")) + { + if (drawer.combo_box("Filters", &gui_settings.selected_filtr, GuiSettings::init_filters_name())) + { + update_uniform_buffers(); + } + } +} + +bool SubgroupsOperations::resize(const uint32_t width, const uint32_t height) +{ + if (!ApiVulkanSample::resize(width, height)) + return false; + build_compute_command_buffer(); + build_command_buffers(); + update_uniform_buffers(); + return true; +} + +void SubgroupsOperations::render(float delta_time) +{ + if (!prepared) + { + return; + } + draw(); + if (camera.updated) + { + update_uniform_buffers(); + } +} + +std::unique_ptr create_subgroups_operations() +{ + return std::make_unique(); +} diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h new file mode 100644 index 000000000..43186a287 --- /dev/null +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -0,0 +1,134 @@ +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "api_vulkan_sample.h" + +struct TextureQuadVertex +{ + glm::vec3 pos; + glm::vec2 uv; +}; + +class SubgroupsOperations : public ApiVulkanSample +{ + public: + SubgroupsOperations(); + ~SubgroupsOperations(); + + bool prepare(vkb::Platform &platform) override; + void request_gpu_features(vkb::PhysicalDevice &gpu) override; + void build_command_buffers() override; + void render(float delta_time) override; + bool resize(const uint32_t width, const uint32_t height) override; + void on_update_ui_overlay(vkb::Drawer &drawer) override; + + void draw(); + void load_assets(); + + void prepare_graphics(); + void generate_quad(); + void setup_descriptor_pool(); + void setup_descriptor_set_layout(); + void setup_descriptor_set(); + void setup_pipelines(); + void create_semaphore(); + void prepare_uniform_buffers(); + void update_uniform_buffers(); + + void prepare_compute(); + void create_compute_queue(); + void create_compute_command_pool(); + void create_compute_command_buffer(); + void create_compute_descriptor_set_layout(); + void create_compute_descriptor_set(); + + void preapre_compute_pipeline_layout(); + void prepare_compute_pipeline(); + + void build_compute_command_buffer(); + + struct GuiSettings + { + int32_t selected_filtr = 0; + static std::vector init_filters_name() + { + std::vector filtrs = { + {"Blur"}, {"Sharpen"}, {"Edge detection"}, {"Custom"}}; + return filtrs; + } + } gui_settings; + + struct Pipeline + { + void destroy(VkDevice device); + + VkPipeline pipeline; + VkPipelineLayout pipeline_layout; + }; + + struct + { + VkSemaphore semaphore; + VkDescriptorSetLayout descriptor_set_layout; + VkDescriptorSet descriptor_set; + Pipeline pipeline; + Texture texture; + + struct + { + std::unique_ptr vertex; + std::unique_ptr index; + uint32_t index_count = {0u}; + } buffers; + } texture_object; + + struct + { + glm::mat3 kernel; + } kernel_ubo; + + std::unique_ptr texture_uniform_buffer; + + struct + { + glm::mat4 projection; + glm::mat4 view; + glm::mat4 model; + } texture_ubo; + + struct + { + VkQueue queue; + VkCommandPool command_pool; + VkCommandBuffer command_buffer; + VkSemaphore semaphore; + VkDescriptorSetLayout descriptor_set_layout; + VkDescriptorSet descriptor_set; + uint32_t queue_family_index; + + struct + { + Pipeline _default; + } pipelines; + } compute; + + VkPhysicalDeviceSubgroupProperties subgroups_properties; +}; + +std::unique_ptr create_subgroups_operations(); diff --git a/shaders/subgroups_operations/base.frag b/shaders/subgroups_operations/base.frag new file mode 100644 index 000000000..25a7084b4 --- /dev/null +++ b/shaders/subgroups_operations/base.frag @@ -0,0 +1,28 @@ +#version 450 +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +layout (binding = 1) uniform sampler2D samplerColor; + +layout (location = 0) in vec2 inUV; + +layout (location = 0) out vec4 outFragColor; + +void main() +{ + outFragColor = texture(samplerColor, inUV); +} \ No newline at end of file diff --git a/shaders/subgroups_operations/base.vert b/shaders/subgroups_operations/base.vert new file mode 100644 index 000000000..f992345da --- /dev/null +++ b/shaders/subgroups_operations/base.vert @@ -0,0 +1,35 @@ +#version 450 +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +layout (location = 0) in vec3 inPos; +layout (location = 1) in vec2 inUV; + +layout(location = 0) out vec2 outUv; + +layout (binding = 0) uniform Ubo +{ + mat4 projection; + mat4 model; + vec4 view; +} ubo; + +void main() +{ + outUv = inUV; + gl_Position = ubo.projection * ubo.model * vec4(inPos.xyz, 1.0f); +} diff --git a/shaders/subgroups_operations/blur.comp b/shaders/subgroups_operations/blur.comp new file mode 100644 index 000000000..1d322a112 --- /dev/null +++ b/shaders/subgroups_operations/blur.comp @@ -0,0 +1,67 @@ +#version 450 +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +#extension GL_KHR_shader_subgroup_basic : enable + +layout (local_size_x = 32, local_size_y = 32, local_size_z = 1) in; + +layout(binding = 0, rgba8) uniform readonly image2D inputTexture; + +layout(binding = 1, rgba8) uniform writeonly image2D outputTexture; + +layout (binding = 2) uniform KernelUbo { + mat3 kernel; +} kernelUbo; + + +struct ImageMatrix +{ + float texel[9]; +} imageMatrix; + +void main() +{ + ivec2 textureCoord = ivec2(gl_GlobalInvocationID.xy); + vec4 resultTexelColor = vec4(0.0); + float denom = 1.0; + float offset = 0.5; + + int currentTexel = 0; + for (int x = -1; x < 2; ++x) + { + for (int y = -1; y < 2; ++y) + { + ivec2 orginalTexelCoord = ivec2(gl_GlobalInvocationID.x + x, gl_GlobalInvocationID.y + y); + vec3 texelRgb = imageLoad(inputTexture, orginalTexelCoord).rgb; + imageMatrix.texel[currentTexel] = (texelRgb.r + texelRgb.g + texelRgb.b) / 3.0; // calculate avg texel color + ++currentTexel; + } + } + + float avgColor = 0.0; + for (int i = 0; i < 9; ++i) + { + avgColor += float(kernelUbo.kernel[i] * imageMatrix.texel[i]); + } + + vec3 rgb = vec3(clamp(avgColor / denom + offset, 0.0, 1.0)); + + resultTexelColor = vec4(rgb, 1.0); + + imageStore(outputTexture, textureCoord, resultTexelColor); +} From e0cb8eaf2f6d2de05df543825893ef19eccf1f34 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Tue, 13 Jun 2023 18:24:35 +0200 Subject: [PATCH 02/53] Add ocean fft --- .../subgroups_operations.cpp | 313 ++++++++---------- .../subgroups_operations.h | 87 +++-- shaders/subgroups_operations/ocean.frag | 25 ++ shaders/subgroups_operations/ocean.vert | 31 ++ 4 files changed, 234 insertions(+), 222 deletions(-) create mode 100644 shaders/subgroups_operations/ocean.frag create mode 100644 shaders/subgroups_operations/ocean.vert diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 55e210ee4..7c1ad43d8 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -40,13 +40,13 @@ SubgroupsOperations::~SubgroupsOperations() { if (device) { - compute.pipelines._default.destroy(get_device().get_handle()); - vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); - vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); - vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); + // compute.pipelines._default.destroy(get_device().get_handle()); + // vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); + // vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); + // vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); - texture_object.pipeline.destroy(get_device().get_handle()); - vkDestroySemaphore(get_device().get_handle(), texture_object.semaphore, nullptr); + ocean.pipelines._default.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), ocean.descriptor_set_layout, nullptr); } } @@ -58,9 +58,17 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) } load_assets(); + // prepare_compute(); + + // graphics pipeline + generate_grid(); + prepare_uniform_buffers(); setup_descriptor_pool(); - prepare_compute(); - prepare_graphics(); + create_descriptor_set_layout(); + create_descriptor_set(); + create_pipelines(); + + build_command_buffers(); prepared = true; return true; @@ -75,7 +83,7 @@ void SubgroupsOperations::prepare_compute() create_compute_descriptor_set(); preapre_compute_pipeline_layout(); prepare_compute_pipeline(); - build_compute_command_buffer(); + // build_compute_command_buffer(); } void SubgroupsOperations::create_compute_queue() @@ -113,21 +121,21 @@ void SubgroupsOperations::create_compute_command_buffer() void SubgroupsOperations::create_compute_descriptor_set_layout() { - std::vector set_layout_bindngs = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0), // input image - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1), // result image - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2) // kernel matrix - }; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); - - VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); + // std::vector set_layout_bindngs = { + // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0), // input image + // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1), // result image + // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2) // kernel matrix + // }; + // VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + + // VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); } void SubgroupsOperations::create_compute_descriptor_set() { VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); + // VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); } void SubgroupsOperations::preapre_compute_pipeline_layout() @@ -163,27 +171,11 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, texture_object.texture.image->get_extent().width / 32u, texture_object.texture.image->get_extent().height / 32u, 1u); + // vkCmdDispatch(compute.command_buffer, texture_object.texture.image->get_extent().width / 32u, texture_object.texture.image->get_extent().height / 32u, 1u); VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); } -void SubgroupsOperations::prepare_graphics() -{ - prepare_uniform_buffers(); - setup_descriptor_set_layout(); - setup_pipelines(); - create_semaphore(); - setup_descriptor_set(); - build_command_buffers(); -} - -void SubgroupsOperations::create_semaphore() -{ - VkSemaphoreCreateInfo semaphore_create_info = vkb::initializers::semaphore_create_info(); - VK_CHECK(vkCreateSemaphore(get_device().get_handle(), &semaphore_create_info, nullptr, &texture_object.semaphore)); -} - void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) { if (gpu.get_features().samplerAnisotropy) @@ -199,48 +191,90 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); } -void SubgroupsOperations::prepare_uniform_buffers() +void SubgroupsOperations::load_assets() { - texture_uniform_buffer = std::make_unique(get_device(), sizeof(texture_ubo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + generate_grid(); +} +void SubgroupsOperations::prepare_uniform_buffers() +{ + camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); update_uniform_buffers(); } -void SubgroupsOperations::generate_quad() +void SubgroupsOperations::generate_grid() { - std::vector vertices = - { - {{1.0f, 1.0f, 0.0f}, {1.0f, 1.0f}}, - {{-1.0f, 1.0f, 0.0f}, {0.0f, 1.0f}}, - {{-1.0f, -1.0f, 0.0f}, {0.0f, 0.0f}}, - {{1.0f, -1.0f, 0.0f}, {1.0f, 0.0f}}}; - std::vector indices = {0u, 1u, 2u, 2u, 3u, 0u}; - texture_object.buffers.index_count = static_cast(indices.size()); - auto vertex_buffer_size = vkb::to_u32(vertices.size() * sizeof(TextureQuadVertex)); - auto index_buffer_size = vkb::to_u32(indices.size() * sizeof(uint32_t)); - texture_object.buffers.vertex = std::make_unique(get_device(), - vertex_buffer_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, - VMA_MEMORY_USAGE_CPU_TO_GPU); - texture_object.buffers.vertex->update(vertices.data(), vertex_buffer_size); - texture_object.buffers.index = std::make_unique(get_device(), - index_buffer_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, - VMA_MEMORY_USAGE_CPU_TO_GPU); - texture_object.buffers.index->update(indices.data(), index_buffer_size); + uint32_t totalIndices = (grid_size * 2) * (grid_size - 1); + std::vector grid_vertices(grid_size * grid_size); + std::vector grid_indices; + + glm::vec3 point = glm::vec3(0.0f); + for (uint32_t x = 0; x < grid_size; ++x) + { + for (uint32_t z = 0; z < grid_size; ++z) + { + uint32_t idx = x + z * grid_size; + grid_vertices[idx].pos = point; + point.z += 1.0f / grid_size; + grid_indices.push_back(idx); + } + point.x += 1.0f / grid_size; + } + + init_grid.index_count = static_cast(grid_indices.size()); + auto vertex_buffer_size = vkb::to_u32(grid_vertices.size() * sizeof(Vertex)); + auto index_buffer_size = vkb::to_u32(grid_indices.size() * sizeof(uint32_t)); + init_grid.vertex = std::make_unique(get_device(), + vertex_buffer_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); + init_grid.vertex->update(grid_vertices.data(), vertex_buffer_size); + + init_grid.index = std::make_unique(get_device(), + index_buffer_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); + init_grid.index->update(grid_indices.data(), index_buffer_size); } -void SubgroupsOperations::update_uniform_buffers() +void SubgroupsOperations::setup_descriptor_pool() { - texture_ubo.model = glm::mat4(1.0f); - texture_ubo.model = glm::translate(texture_ubo.model, glm::vec3(0.0f)); - texture_ubo.view = camera.matrices.view; - texture_ubo.projection = camera.matrices.perspective; + std::vector pool_sizes = { + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1u)}; + VkDescriptorPoolCreateInfo descriptor_pool_create_info = + vkb::initializers::descriptor_pool_create_info( + static_cast(pool_sizes.size()), + pool_sizes.data(), + 2u); + VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); +} - texture_uniform_buffer->convert_and_update(texture_ubo); +void SubgroupsOperations::create_descriptor_set_layout() +{ + std::vector set_layout_bindings = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT, 0u) // ubo + }; + + VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); + + VkPipelineLayoutCreateInfo pipeline_layout_create_info = vkb::initializers::pipeline_layout_create_info(&ocean.descriptor_set_layout); + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &pipeline_layout_create_info, nullptr, &ocean.pipelines._default.pipeline_layout)); } -void SubgroupsOperations::setup_pipelines() +void SubgroupsOperations::create_descriptor_set() +{ + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &ocean.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &ocean.descriptor_set)); + + VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*camera_ubo); + std::vector write_descriptor_sets = { + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor)}; + + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); +} + +void SubgroupsOperations::create_pipelines() { VkPipelineInputAssemblyStateCreateInfo input_assembly_state = vkb::initializers::pipeline_input_assembly_state_create_info( @@ -281,14 +315,13 @@ void SubgroupsOperations::setup_pipelines() static_cast(dynamic_state_enables.size()), 0u); std::array shader_stages; - shader_stages[0] = load_shader("subgroups_operations/base.vert", VK_SHADER_STAGE_VERTEX_BIT); - shader_stages[1] = load_shader("subgroups_operations/base.frag", VK_SHADER_STAGE_FRAGMENT_BIT); + shader_stages[0] = load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT); + shader_stages[1] = load_shader("subgroups_operations/ocean.frag", VK_SHADER_STAGE_FRAGMENT_BIT); const std::vector vertex_input_bindings = { - vkb::initializers::vertex_input_binding_description(0u, sizeof(TextureQuadVertex), VK_VERTEX_INPUT_RATE_VERTEX), + vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX), }; const std::vector vertex_input_attributes = { - vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(TextureQuadVertex, pos)), - vkb::initializers::vertex_input_attribute_description(0u, 1u, VK_FORMAT_R32G32_SFLOAT, offsetof(TextureQuadVertex, uv)), + vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), }; VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); @@ -297,7 +330,7 @@ void SubgroupsOperations::setup_pipelines() vertex_input_state.pVertexAttributeDescriptions = vertex_input_attributes.data(); VkGraphicsPipelineCreateInfo pipeline_create_info = vkb::initializers::pipeline_create_info( - texture_object.pipeline.pipeline_layout, + ocean.pipelines._default.pipeline_layout, render_pass, 0u); pipeline_create_info.pVertexInputState = &vertex_input_state; @@ -310,75 +343,18 @@ void SubgroupsOperations::setup_pipelines() pipeline_create_info.pDynamicState = &dynamic_state; pipeline_create_info.stageCount = static_cast(shader_stages.size()); pipeline_create_info.pStages = shader_stages.data(); - VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &texture_object.pipeline.pipeline)); -} - -void SubgroupsOperations::setup_descriptor_pool() -{ - std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1u), - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1u)}; - VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info( - static_cast(pool_sizes.size()), - pool_sizes.data(), - 2u); - VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); + VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines._default.pipeline)); } -void SubgroupsOperations::setup_descriptor_set_layout() +void SubgroupsOperations::update_uniform_buffers() { - std::vector set_layout_bindings = - { - vkb::initializers::descriptor_set_layout_binding( - VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, - VK_SHADER_STAGE_VERTEX_BIT, - 0u), - vkb::initializers::descriptor_set_layout_binding( - VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - VK_SHADER_STAGE_FRAGMENT_BIT, - 1u)}; - VkDescriptorSetLayoutCreateInfo descriptor_layout = - vkb::initializers::descriptor_set_layout_create_info( - set_layout_bindings.data(), - static_cast(set_layout_bindings.size())); - VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &texture_object.descriptor_set_layout)); - VkPipelineLayoutCreateInfo pipeline_layout_create_info = - vkb::initializers::pipeline_layout_create_info( - &texture_object.descriptor_set_layout, - 1u); - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &pipeline_layout_create_info, nullptr, &texture_object.pipeline.pipeline_layout)); -} + CameraUbo ubo; + ubo.model = glm::mat4(1.0f); + ubo.model = glm::translate(ubo.model, glm::vec3(0.0f)); + ubo.view = camera.matrices.view; + ubo.projection = camera.matrices.perspective; -void SubgroupsOperations::setup_descriptor_set() -{ - VkDescriptorSetAllocateInfo alloc_info = - vkb::initializers::descriptor_set_allocate_info( - descriptor_pool, - &texture_object.descriptor_set_layout, - 1u); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &texture_object.descriptor_set)); - VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*texture_uniform_buffer); - VkDescriptorImageInfo image_descriptor = create_descriptor(texture_object.texture); - std::vector write_descriptor_sets = - { - vkb::initializers::write_descriptor_set( - texture_object.descriptor_set, - VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, - 0u, - &buffer_descriptor), - vkb::initializers::write_descriptor_set( - texture_object.descriptor_set, - VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - 1u, - &image_descriptor)}; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, NULL); -} - -void SubgroupsOperations::load_assets() -{ - texture_object.texture = load_texture("textures/vulkan_logo_full.ktx", vkb::sg::Image::Color); - generate_quad(); + camera_ubo->convert_and_update(ubo); } void SubgroupsOperations::build_command_buffers() @@ -399,34 +375,35 @@ void SubgroupsOperations::build_command_buffers() for (uint32_t i = 0u; i < draw_cmd_buffers.size(); ++i) { render_pass_begin_info.framebuffer = framebuffers[i]; + auto &cmd_buff = draw_cmd_buffers[i]; - VK_CHECK(vkBeginCommandBuffer(draw_cmd_buffers[i], &command_buffer_begin_info)); + VK_CHECK(vkBeginCommandBuffer(cmd_buff, &command_buffer_begin_info)); - vkCmdBeginRenderPass(draw_cmd_buffers[i], &render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); + vkCmdBeginRenderPass(cmd_buff, &render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); VkViewport viewport = vkb::initializers::viewport(static_cast(width), static_cast(height), 0.0f, 1.0f); - vkCmdSetViewport(draw_cmd_buffers[i], 0u, 1u, &viewport); + vkCmdSetViewport(cmd_buff, 0u, 1u, &viewport); VkRect2D scissor = vkb::initializers::rect2D(width, height, 0, 0); - vkCmdSetScissor(draw_cmd_buffers[i], 0u, 1u, &scissor); + vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); - // draw texture + // draw ocean { - vkCmdBindDescriptorSets(draw_cmd_buffers[i], VK_PIPELINE_BIND_POINT_GRAPHICS, texture_object.pipeline.pipeline_layout, 0, 1, &texture_object.descriptor_set, 0, nullptr); - vkCmdBindPipeline(draw_cmd_buffers[i], VK_PIPELINE_BIND_POINT_GRAPHICS, texture_object.pipeline.pipeline); + vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); + vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline); VkDeviceSize offset[] = {0}; - vkCmdBindVertexBuffers(draw_cmd_buffers[i], 0, 1, texture_object.buffers.vertex->get(), offset); - vkCmdBindIndexBuffer(draw_cmd_buffers[i], texture_object.buffers.index->get_handle(), 0, VK_INDEX_TYPE_UINT32); + vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, init_grid.vertex->get(), offset); + vkCmdBindIndexBuffer(cmd_buff, init_grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); - vkCmdDrawIndexed(draw_cmd_buffers[i], texture_object.buffers.index_count, 1u, 0u, 0u, 0u); + vkCmdDrawIndexed(cmd_buff, init_grid.index_count, 1u, 0u, 0u, 0u); } - draw_ui(draw_cmd_buffers[i]); + draw_ui(cmd_buff); - vkCmdEndRenderPass(draw_cmd_buffers[i]); + vkCmdEndRenderPass(cmd_buff); - VK_CHECK(vkEndCommandBuffer(draw_cmd_buffers[i])); + VK_CHECK(vkEndCommandBuffer(cmd_buff)); } } @@ -434,16 +411,16 @@ void SubgroupsOperations::draw() { VkPipelineStageFlags graphics_wait_stage_masks[] = {VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT}; VkSemaphore graphics_wait_semaphores[] = {compute.semaphore, semaphores.acquired_image_ready}; - VkSemaphore graphics_signal_semaphores[] = {texture_object.semaphore, semaphores.render_complete}; + VkSemaphore graphics_signal_semaphores[] = {VK_NULL_HANDLE, semaphores.render_complete}; ApiVulkanSample::prepare_frame(); - submit_info.commandBufferCount = 1; - submit_info.pCommandBuffers = &draw_cmd_buffers[current_buffer]; - submit_info.waitSemaphoreCount = 2; - submit_info.pWaitSemaphores = graphics_wait_semaphores; - submit_info.pWaitDstStageMask = graphics_wait_stage_masks; - submit_info.signalSemaphoreCount = 2; - submit_info.pSignalSemaphores = graphics_signal_semaphores; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &draw_cmd_buffers[current_buffer]; + // submit_info.waitSemaphoreCount = 2; + // submit_info.pWaitSemaphores = graphics_wait_semaphores; + // submit_info.pWaitDstStageMask = graphics_wait_stage_masks; + // submit_info.signalSemaphoreCount = 2; + // submit_info.pSignalSemaphores = graphics_signal_semaphores; VK_CHECK(vkQueueSubmit(queue, 1u, &submit_info, VK_NULL_HANDLE)); ApiVulkanSample::submit_frame(); @@ -451,36 +428,28 @@ void SubgroupsOperations::draw() VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; // Submit compute commands - VkSubmitInfo compute_submit_info = vkb::initializers::submit_info(); - compute_submit_info.commandBufferCount = 1u; - compute_submit_info.pCommandBuffers = &compute.command_buffer; - compute_submit_info.waitSemaphoreCount = 1u; - compute_submit_info.pWaitSemaphores = &texture_object.semaphore; + VkSubmitInfo compute_submit_info = vkb::initializers::submit_info(); + compute_submit_info.commandBufferCount = 1u; + compute_submit_info.pCommandBuffers = &compute.command_buffer; + compute_submit_info.waitSemaphoreCount = 1u; + compute_submit_info.pWaitDstStageMask = &wait_stage_mask; compute_submit_info.signalSemaphoreCount = 1u; compute_submit_info.pSignalSemaphores = &compute.semaphore; - VK_CHECK(vkQueueSubmit(compute.queue, 1u, &compute_submit_info, VK_NULL_HANDLE)); + // VK_CHECK(vkQueueSubmit(compute.queue, 1u, &compute_submit_info, VK_NULL_HANDLE)); } void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) { - if (drawer.header("Settings")) - { - if (drawer.combo_box("Filters", &gui_settings.selected_filtr, GuiSettings::init_filters_name())) - { - update_uniform_buffers(); - } - } } bool SubgroupsOperations::resize(const uint32_t width, const uint32_t height) { if (!ApiVulkanSample::resize(width, height)) return false; - build_compute_command_buffer(); + // build_compute_command_buffer(); build_command_buffers(); - update_uniform_buffers(); return true; } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 43186a287..4c0b9fcea 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -19,12 +19,6 @@ #include "api_vulkan_sample.h" -struct TextureQuadVertex -{ - glm::vec3 pos; - glm::vec2 uv; -}; - class SubgroupsOperations : public ApiVulkanSample { public: @@ -41,16 +35,6 @@ class SubgroupsOperations : public ApiVulkanSample void draw(); void load_assets(); - void prepare_graphics(); - void generate_quad(); - void setup_descriptor_pool(); - void setup_descriptor_set_layout(); - void setup_descriptor_set(); - void setup_pipelines(); - void create_semaphore(); - void prepare_uniform_buffers(); - void update_uniform_buffers(); - void prepare_compute(); void create_compute_queue(); void create_compute_command_pool(); @@ -63,16 +47,17 @@ class SubgroupsOperations : public ApiVulkanSample void build_compute_command_buffer(); - struct GuiSettings - { - int32_t selected_filtr = 0; - static std::vector init_filters_name() - { - std::vector filtrs = { - {"Blur"}, {"Sharpen"}, {"Edge detection"}, {"Custom"}}; - return filtrs; - } - } gui_settings; + /////////////////////////////////////////////// + void generate_grid(); + void prepare_uniform_buffers(); + void setup_descriptor_pool(); + void create_descriptor_set_layout(); + void create_descriptor_set(); + void create_pipelines(); + + void update_uniform_buffers(); + + struct Pipeline { @@ -82,35 +67,25 @@ class SubgroupsOperations : public ApiVulkanSample VkPipelineLayout pipeline_layout; }; - struct + struct GridBuffers { - VkSemaphore semaphore; - VkDescriptorSetLayout descriptor_set_layout; - VkDescriptorSet descriptor_set; - Pipeline pipeline; - Texture texture; - - struct - { - std::unique_ptr vertex; - std::unique_ptr index; - uint32_t index_count = {0u}; - } buffers; - } texture_object; + std::unique_ptr vertex; + std::unique_ptr index; + uint32_t index_count = {0u}; + }; - struct + struct CameraUbo { - glm::mat3 kernel; - } kernel_ubo; + alignas(16) glm::mat4 projection; + alignas(16) glm::mat4 view; + alignas(16) glm::mat4 model; + }; - std::unique_ptr texture_uniform_buffer; + GridBuffers init_grid; // input buffer for compute shader - struct - { - glm::mat4 projection; - glm::mat4 view; - glm::mat4 model; - } texture_ubo; + uint32_t grid_size = {128u}; + + std::unique_ptr camera_ubo; struct { @@ -128,6 +103,18 @@ class SubgroupsOperations : public ApiVulkanSample } pipelines; } compute; + struct + { + GridBuffers grid; // output (result) buffer for compute shader + + VkDescriptorSetLayout descriptor_set_layout; + VkDescriptorSet descriptor_set; + struct + { + Pipeline _default; + } pipelines; + } ocean; + VkPhysicalDeviceSubgroupProperties subgroups_properties; }; diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag new file mode 100644 index 000000000..94e2445a5 --- /dev/null +++ b/shaders/subgroups_operations/ocean.frag @@ -0,0 +1,25 @@ +#version 450 +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +layout (location = 0) out vec4 outFragColor; + +void main() +{ + outFragColor = vec4(0.0f, 0.5f, 1.0f, 1.0f); // should be blue +} \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.vert b/shaders/subgroups_operations/ocean.vert new file mode 100644 index 000000000..909b01b76 --- /dev/null +++ b/shaders/subgroups_operations/ocean.vert @@ -0,0 +1,31 @@ +#version 450 +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +layout (location = 0) in vec3 inPos; + +layout (binding = 0) uniform Ubo +{ + mat4 projection; + mat4 view; + mat4 model; +} ubo; + +void main() +{ + gl_Position = ubo.projection * ubo.view * ubo.model * vec4(inPos.xyz, 1.0f); +} From d563f62105573dc6c8d09a24b202e97c07a55c59 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 14 Jun 2023 08:34:52 +0200 Subject: [PATCH 03/53] Fix grid generation algorithm --- .../subgroups_operations.cpp | 663 +++++++++--------- .../subgroups_operations.h | 194 ++--- 2 files changed, 450 insertions(+), 407 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 7c1ad43d8..6bfcbaa5e 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -19,454 +19,491 @@ void SubgroupsOperations::Pipeline::destroy(VkDevice device) { - vkDestroyPipeline(device, pipeline, nullptr); - vkDestroyPipelineLayout(device, pipeline_layout, nullptr); + if (pipeline != VK_NULL_HANDLE) + { + vkDestroyPipeline(device, pipeline, nullptr); + } + if (pipeline_layout != VK_NULL_HANDLE) + { + vkDestroyPipelineLayout(device, pipeline_layout, nullptr); + } } SubgroupsOperations::SubgroupsOperations() { - set_api_version(VK_API_VERSION_1_1); - title = "Subgroups operations"; - camera.type = vkb::CameraType::LookAt; - - camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); - camera.set_position({0.0f, 0.0f, -2.0f}); - add_device_extension(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); - add_device_extension(VK_EXT_SHADER_SUBGROUP_BALLOT_EXTENSION_NAME); // is needed??? - add_device_extension(VK_EXT_SHADER_SUBGROUP_VOTE_EXTENSION_NAME); // is needed??? + set_api_version(VK_API_VERSION_1_1); + title = "Subgroups operations"; + camera.type = vkb::CameraType::LookAt; + + camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); + camera.set_position({0.0f, 0.0f, -2.0f}); + add_device_extension(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); + add_device_extension(VK_EXT_SHADER_SUBGROUP_BALLOT_EXTENSION_NAME); // is needed??? + add_device_extension(VK_EXT_SHADER_SUBGROUP_VOTE_EXTENSION_NAME); // is needed??? } SubgroupsOperations::~SubgroupsOperations() { - if (device) - { - // compute.pipelines._default.destroy(get_device().get_handle()); - // vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); - // vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); - // vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); - - ocean.pipelines._default.destroy(get_device().get_handle()); - vkDestroyDescriptorSetLayout(get_device().get_handle(), ocean.descriptor_set_layout, nullptr); - } + if (device) + { + // compute.pipelines._default.destroy(get_device().get_handle()); + // vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); + // vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); + // vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); + + ocean.pipelines._default.destroy(get_device().get_handle()); + ocean.pipelines.wireframe.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), ocean.descriptor_set_layout, nullptr); + } } bool SubgroupsOperations::prepare(vkb::Platform &platform) { - if (!ApiVulkanSample::prepare(platform)) - { - return false; - } - - load_assets(); - // prepare_compute(); - - // graphics pipeline - generate_grid(); - prepare_uniform_buffers(); - setup_descriptor_pool(); - create_descriptor_set_layout(); - create_descriptor_set(); - create_pipelines(); - - build_command_buffers(); - - prepared = true; - return true; + if (!ApiVulkanSample::prepare(platform)) + { + return false; + } + + load_assets(); + // prepare_compute(); + + // graphics pipeline + generate_grid(); + prepare_uniform_buffers(); + setup_descriptor_pool(); + create_descriptor_set_layout(); + create_descriptor_set(); + create_pipelines(); + + build_command_buffers(); + + prepared = true; + return true; } void SubgroupsOperations::prepare_compute() { - create_compute_queue(); - create_compute_command_pool(); - create_compute_command_buffer(); - create_compute_descriptor_set_layout(); - create_compute_descriptor_set(); - preapre_compute_pipeline_layout(); - prepare_compute_pipeline(); - // build_compute_command_buffer(); + create_compute_queue(); + create_compute_command_pool(); + create_compute_command_buffer(); + create_compute_descriptor_set_layout(); + create_compute_descriptor_set(); + preapre_compute_pipeline_layout(); + prepare_compute_pipeline(); + // build_compute_command_buffer(); } void SubgroupsOperations::create_compute_queue() { - // create compute queue and get family index - compute.queue_family_index = get_device().get_queue_family_index(VK_QUEUE_COMPUTE_BIT); + // create compute queue and get family index + compute.queue_family_index = get_device().get_queue_family_index(VK_QUEUE_COMPUTE_BIT); - vkGetDeviceQueue(get_device().get_handle(), compute.queue_family_index, 0u, &compute.queue); + vkGetDeviceQueue(get_device().get_handle(), compute.queue_family_index, 0u, &compute.queue); } void SubgroupsOperations::create_compute_command_pool() { - VkCommandPoolCreateInfo command_pool_create_info = {}; - command_pool_create_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; - command_pool_create_info.queueFamilyIndex = compute.queue_family_index; - command_pool_create_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; - VK_CHECK(vkCreateCommandPool(get_device().get_handle(), &command_pool_create_info, nullptr, &compute.command_pool)); + VkCommandPoolCreateInfo command_pool_create_info = {}; + command_pool_create_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + command_pool_create_info.queueFamilyIndex = compute.queue_family_index; + command_pool_create_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + VK_CHECK(vkCreateCommandPool(get_device().get_handle(), &command_pool_create_info, nullptr, &compute.command_pool)); } void SubgroupsOperations::create_compute_command_buffer() { - // Create a command buffer for compute operations - VkCommandBufferAllocateInfo command_buffer_allocate_info = - vkb::initializers::command_buffer_allocate_info( - compute.command_pool, - VK_COMMAND_BUFFER_LEVEL_PRIMARY, - 1u); - - VK_CHECK(vkAllocateCommandBuffers(get_device().get_handle(), &command_buffer_allocate_info, &compute.command_buffer)); - - // Semaphore for compute & graphics sync - VkSemaphoreCreateInfo semaphore_create_info = vkb::initializers::semaphore_create_info(); - VK_CHECK(vkCreateSemaphore(get_device().get_handle(), &semaphore_create_info, nullptr, &compute.semaphore)); + // Create a command buffer for compute operations + VkCommandBufferAllocateInfo command_buffer_allocate_info = + vkb::initializers::command_buffer_allocate_info( + compute.command_pool, + VK_COMMAND_BUFFER_LEVEL_PRIMARY, + 1u); + + VK_CHECK(vkAllocateCommandBuffers(get_device().get_handle(), &command_buffer_allocate_info, &compute.command_buffer)); + + // Semaphore for compute & graphics sync + VkSemaphoreCreateInfo semaphore_create_info = vkb::initializers::semaphore_create_info(); + VK_CHECK(vkCreateSemaphore(get_device().get_handle(), &semaphore_create_info, nullptr, &compute.semaphore)); } void SubgroupsOperations::create_compute_descriptor_set_layout() { - // std::vector set_layout_bindngs = { - // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0), // input image - // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1), // result image - // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2) // kernel matrix - // }; - // VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); - - // VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); + // std::vector set_layout_bindngs = { + // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0), // input image + // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1), // result image + // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2) // kernel matrix + // }; + // VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + + // VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); } void SubgroupsOperations::create_compute_descriptor_set() { - VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1); + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1); - // VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); + // VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); } void SubgroupsOperations::preapre_compute_pipeline_layout() { - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; - compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - compute_pipeline_layout_info.setLayoutCount = 1u; - compute_pipeline_layout_info.pSetLayouts = &compute.descriptor_set_layout; + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &compute.descriptor_set_layout; - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &compute.pipelines._default.pipeline_layout)); + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &compute.pipelines._default.pipeline_layout)); } void SubgroupsOperations::prepare_compute_pipeline() { - VkComputePipelineCreateInfo computeInfo = {}; - computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; - computeInfo.layout = compute.pipelines._default.pipeline_layout; - computeInfo.stage = load_shader("subgroups_operations/blur.comp", VK_SHADER_STAGE_COMPUTE_BIT); + VkComputePipelineCreateInfo computeInfo = {}; + computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + computeInfo.layout = compute.pipelines._default.pipeline_layout; + computeInfo.stage = load_shader("subgroups_operations/blur.comp", VK_SHADER_STAGE_COMPUTE_BIT); - VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines._default.pipeline)); + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines._default.pipeline)); } void SubgroupsOperations::build_compute_command_buffer() { - VkSubmitInfo submit_info = {}; - submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; - vkResetCommandBuffer(compute.command_buffer, 0u); + VkSubmitInfo submit_info = {}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + vkResetCommandBuffer(compute.command_buffer, 0u); - // record command - VkCommandBufferBeginInfo begin_info = {}; - begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; - VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); + // record command + VkCommandBufferBeginInfo begin_info = {}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); - // vkCmdDispatch(compute.command_buffer, texture_object.texture.image->get_extent().width / 32u, texture_object.texture.image->get_extent().height / 32u, 1u); + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); + // vkCmdDispatch(compute.command_buffer, texture_object.texture.image->get_extent().width / 32u, texture_object.texture.image->get_extent().height / 32u, 1u); - VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); + VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); } void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) { - if (gpu.get_features().samplerAnisotropy) - { - gpu.get_mutable_requested_features().samplerAnisotropy = VK_TRUE; - } - subgroups_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; - subgroups_properties.pNext = VK_NULL_HANDLE; - - VkPhysicalDeviceProperties2 device_properties2 = {}; - device_properties2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; - device_properties2.pNext = &subgroups_properties; - vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); + if (gpu.get_features().samplerAnisotropy) + { + gpu.get_mutable_requested_features().samplerAnisotropy = VK_TRUE; + } + subgroups_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; + subgroups_properties.pNext = VK_NULL_HANDLE; + + VkPhysicalDeviceProperties2 device_properties2 = {}; + device_properties2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + device_properties2.pNext = &subgroups_properties; + vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); } void SubgroupsOperations::load_assets() { - generate_grid(); + generate_grid(); } void SubgroupsOperations::prepare_uniform_buffers() { - camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - update_uniform_buffers(); + camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + update_uniform_buffers(); } void SubgroupsOperations::generate_grid() { - uint32_t totalIndices = (grid_size * 2) * (grid_size - 1); - std::vector grid_vertices(grid_size * grid_size); - std::vector grid_indices; - - glm::vec3 point = glm::vec3(0.0f); - for (uint32_t x = 0; x < grid_size; ++x) - { - for (uint32_t z = 0; z < grid_size; ++z) - { - uint32_t idx = x + z * grid_size; - grid_vertices[idx].pos = point; - point.z += 1.0f / grid_size; - grid_indices.push_back(idx); - } - point.x += 1.0f / grid_size; - } - - init_grid.index_count = static_cast(grid_indices.size()); - auto vertex_buffer_size = vkb::to_u32(grid_vertices.size() * sizeof(Vertex)); - auto index_buffer_size = vkb::to_u32(grid_indices.size() * sizeof(uint32_t)); - init_grid.vertex = std::make_unique(get_device(), + std::vector grid_vertices; + std::vector grid_indices; + + for (uint32_t z = 0; z <= grid_size; ++z) + { + for (uint32_t x = 0; x <= grid_size; ++x) + { + Vertex point; + point.pos.x = static_cast(x) / static_cast(grid_size); + point.pos.z = static_cast(z) / static_cast(grid_size); + grid_vertices.push_back(point); + } + } + + for (uint32_t z = 0; z < grid_size; ++z) + { + for (uint32_t x = 0; x < grid_size; ++x) + { + uint32_t row1 = z * (grid_size + 1); + uint32_t row2 = (z + 1) * (grid_size + 1); + grid_indices.push_back(row1 + x + 0); + grid_indices.push_back(row1 + x + 1); + grid_indices.push_back(row2 + x + 1); + + grid_indices.push_back(row1 + x + 0); + grid_indices.push_back(row2 + x + 1); + grid_indices.push_back(row2 + x); + } + } + + init_grid.index_count = static_cast(grid_indices.size()); + auto vertex_buffer_size = vkb::to_u32(grid_vertices.size() * sizeof(Vertex)); + auto index_buffer_size = vkb::to_u32(grid_indices.size() * sizeof(uint32_t)); + init_grid.vertex = std::make_unique(get_device(), vertex_buffer_size, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - init_grid.vertex->update(grid_vertices.data(), vertex_buffer_size); + init_grid.vertex->update(grid_vertices.data(), vertex_buffer_size); - init_grid.index = std::make_unique(get_device(), - index_buffer_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, - VMA_MEMORY_USAGE_CPU_TO_GPU); - init_grid.index->update(grid_indices.data(), index_buffer_size); + init_grid.index = std::make_unique(get_device(), + index_buffer_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); + init_grid.index->update(grid_indices.data(), index_buffer_size); } void SubgroupsOperations::setup_descriptor_pool() { - std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1u)}; - VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info( - static_cast(pool_sizes.size()), - pool_sizes.data(), - 2u); - VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); + std::vector pool_sizes = { + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1u)}; + VkDescriptorPoolCreateInfo descriptor_pool_create_info = + vkb::initializers::descriptor_pool_create_info( + static_cast(pool_sizes.size()), + pool_sizes.data(), + 2u); + VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } void SubgroupsOperations::create_descriptor_set_layout() { - std::vector set_layout_bindings = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT, 0u) // ubo - }; + std::vector set_layout_bindings = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT, 0u) // ubo + }; - VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); - VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); + VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); - VkPipelineLayoutCreateInfo pipeline_layout_create_info = vkb::initializers::pipeline_layout_create_info(&ocean.descriptor_set_layout); - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &pipeline_layout_create_info, nullptr, &ocean.pipelines._default.pipeline_layout)); + VkPipelineLayoutCreateInfo pipeline_layout_create_info = vkb::initializers::pipeline_layout_create_info(&ocean.descriptor_set_layout); + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &pipeline_layout_create_info, nullptr, &ocean.pipelines._default.pipeline_layout)); } void SubgroupsOperations::create_descriptor_set() { - VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &ocean.descriptor_set_layout, 1u); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &ocean.descriptor_set)); + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &ocean.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &ocean.descriptor_set)); - VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*camera_ubo); - std::vector write_descriptor_sets = { - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor)}; + VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*camera_ubo); + std::vector write_descriptor_sets = { + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor)}; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } void SubgroupsOperations::create_pipelines() { - VkPipelineInputAssemblyStateCreateInfo input_assembly_state = - vkb::initializers::pipeline_input_assembly_state_create_info( - VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, - 0u, - VK_FALSE); - VkPipelineRasterizationStateCreateInfo rasterization_state = - vkb::initializers::pipeline_rasterization_state_create_info( - VK_POLYGON_MODE_FILL, - VK_CULL_MODE_NONE, - VK_FRONT_FACE_COUNTER_CLOCKWISE, - 0u); - VkPipelineColorBlendAttachmentState blend_attachment_state = - vkb::initializers::pipeline_color_blend_attachment_state( - 0xf, - VK_FALSE); - VkPipelineColorBlendStateCreateInfo color_blend_state = - vkb::initializers::pipeline_color_blend_state_create_info( - 1u, - &blend_attachment_state); - VkPipelineDepthStencilStateCreateInfo depth_stencil_state = - vkb::initializers::pipeline_depth_stencil_state_create_info( - VK_TRUE, - VK_TRUE, - VK_COMPARE_OP_GREATER); - VkPipelineViewportStateCreateInfo viewport_state = - vkb::initializers::pipeline_viewport_state_create_info(1u, 1u, 0u); - VkPipelineMultisampleStateCreateInfo multisample_state = - vkb::initializers::pipeline_multisample_state_create_info( - VK_SAMPLE_COUNT_1_BIT, - 0u); - std::vector dynamic_state_enables = { - VK_DYNAMIC_STATE_VIEWPORT, - VK_DYNAMIC_STATE_SCISSOR}; - VkPipelineDynamicStateCreateInfo dynamic_state = - vkb::initializers::pipeline_dynamic_state_create_info( - dynamic_state_enables.data(), - static_cast(dynamic_state_enables.size()), - 0u); - std::array shader_stages; - shader_stages[0] = load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT); - shader_stages[1] = load_shader("subgroups_operations/ocean.frag", VK_SHADER_STAGE_FRAGMENT_BIT); - const std::vector vertex_input_bindings = { - vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX), - }; - const std::vector vertex_input_attributes = { - vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), - }; - VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); - vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); - vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); - vertex_input_state.vertexAttributeDescriptionCount = static_cast(vertex_input_attributes.size()); - vertex_input_state.pVertexAttributeDescriptions = vertex_input_attributes.data(); - VkGraphicsPipelineCreateInfo pipeline_create_info = - vkb::initializers::pipeline_create_info( - ocean.pipelines._default.pipeline_layout, - render_pass, - 0u); - pipeline_create_info.pVertexInputState = &vertex_input_state; - pipeline_create_info.pInputAssemblyState = &input_assembly_state; - pipeline_create_info.pRasterizationState = &rasterization_state; - pipeline_create_info.pColorBlendState = &color_blend_state; - pipeline_create_info.pMultisampleState = &multisample_state; - pipeline_create_info.pViewportState = &viewport_state; - pipeline_create_info.pDepthStencilState = &depth_stencil_state; - pipeline_create_info.pDynamicState = &dynamic_state; - pipeline_create_info.stageCount = static_cast(shader_stages.size()); - pipeline_create_info.pStages = shader_stages.data(); - VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines._default.pipeline)); + VkPipelineInputAssemblyStateCreateInfo input_assembly_state = + vkb::initializers::pipeline_input_assembly_state_create_info( + VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, + 0u, + VK_FALSE); + VkPipelineRasterizationStateCreateInfo rasterization_state = + vkb::initializers::pipeline_rasterization_state_create_info( + VK_POLYGON_MODE_FILL, + VK_CULL_MODE_NONE, + VK_FRONT_FACE_COUNTER_CLOCKWISE, + 0u); + VkPipelineColorBlendAttachmentState blend_attachment_state = + vkb::initializers::pipeline_color_blend_attachment_state( + 0xf, + VK_FALSE); + VkPipelineColorBlendStateCreateInfo color_blend_state = + vkb::initializers::pipeline_color_blend_state_create_info( + 1u, + &blend_attachment_state); + VkPipelineDepthStencilStateCreateInfo depth_stencil_state = + vkb::initializers::pipeline_depth_stencil_state_create_info( + VK_TRUE, + VK_TRUE, + VK_COMPARE_OP_GREATER); + VkPipelineViewportStateCreateInfo viewport_state = + vkb::initializers::pipeline_viewport_state_create_info(1u, 1u, 0u); + VkPipelineMultisampleStateCreateInfo multisample_state = + vkb::initializers::pipeline_multisample_state_create_info( + VK_SAMPLE_COUNT_1_BIT, + 0u); + std::vector dynamic_state_enables = { + VK_DYNAMIC_STATE_VIEWPORT, + VK_DYNAMIC_STATE_SCISSOR}; + VkPipelineDynamicStateCreateInfo dynamic_state = + vkb::initializers::pipeline_dynamic_state_create_info( + dynamic_state_enables.data(), + static_cast(dynamic_state_enables.size()), + 0u); + std::array shader_stages; + shader_stages[0] = load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT); + shader_stages[1] = load_shader("subgroups_operations/ocean.frag", VK_SHADER_STAGE_FRAGMENT_BIT); + const std::vector vertex_input_bindings = { + vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX), + }; + const std::vector vertex_input_attributes = { + vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), + }; + VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); + vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); + vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); + vertex_input_state.vertexAttributeDescriptionCount = static_cast(vertex_input_attributes.size()); + vertex_input_state.pVertexAttributeDescriptions = vertex_input_attributes.data(); + VkGraphicsPipelineCreateInfo pipeline_create_info = + vkb::initializers::pipeline_create_info( + ocean.pipelines._default.pipeline_layout, + render_pass, + 0u); + pipeline_create_info.pVertexInputState = &vertex_input_state; + pipeline_create_info.pInputAssemblyState = &input_assembly_state; + pipeline_create_info.pRasterizationState = &rasterization_state; + pipeline_create_info.pColorBlendState = &color_blend_state; + pipeline_create_info.pMultisampleState = &multisample_state; + pipeline_create_info.pViewportState = &viewport_state; + pipeline_create_info.pDepthStencilState = &depth_stencil_state; + pipeline_create_info.pDynamicState = &dynamic_state; + pipeline_create_info.stageCount = static_cast(shader_stages.size()); + pipeline_create_info.pStages = shader_stages.data(); + VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines._default.pipeline)); + + if (get_device().get_gpu().get_features().fillModeNonSolid) + { + rasterization_state.polygonMode = VK_POLYGON_MODE_LINE; + VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines.wireframe.pipeline)); + + } } void SubgroupsOperations::update_uniform_buffers() { - CameraUbo ubo; - ubo.model = glm::mat4(1.0f); - ubo.model = glm::translate(ubo.model, glm::vec3(0.0f)); - ubo.view = camera.matrices.view; - ubo.projection = camera.matrices.perspective; + CameraUbo ubo; + ubo.model = glm::mat4(1.0f); + ubo.model = glm::translate(ubo.model, glm::vec3(0.0f)); + ubo.view = camera.matrices.view; + ubo.projection = camera.matrices.perspective; - camera_ubo->convert_and_update(ubo); + camera_ubo->convert_and_update(ubo); } void SubgroupsOperations::build_command_buffers() { - VkCommandBufferBeginInfo command_buffer_begin_info = vkb::initializers::command_buffer_begin_info(); + VkCommandBufferBeginInfo command_buffer_begin_info = vkb::initializers::command_buffer_begin_info(); - std::array clear_values; - clear_values[0].color = {{0.0f, 0.0f, 0.0f, 0.0f}}; - clear_values[1].depthStencil = {0.0f, 0u}; + std::array clear_values; + clear_values[0].color = {{0.0f, 0.0f, 0.0f, 0.0f}}; + clear_values[1].depthStencil = {0.0f, 0u}; - VkRenderPassBeginInfo render_pass_begin_info = vkb::initializers::render_pass_begin_info(); - render_pass_begin_info.renderPass = render_pass; - render_pass_begin_info.renderArea.extent.width = width; - render_pass_begin_info.renderArea.extent.height = height; - render_pass_begin_info.clearValueCount = static_cast(clear_values.size()); - render_pass_begin_info.pClearValues = clear_values.data(); + VkRenderPassBeginInfo render_pass_begin_info = vkb::initializers::render_pass_begin_info(); + render_pass_begin_info.renderPass = render_pass; + render_pass_begin_info.renderArea.extent.width = width; + render_pass_begin_info.renderArea.extent.height = height; + render_pass_begin_info.clearValueCount = static_cast(clear_values.size()); + render_pass_begin_info.pClearValues = clear_values.data(); - for (uint32_t i = 0u; i < draw_cmd_buffers.size(); ++i) - { - render_pass_begin_info.framebuffer = framebuffers[i]; - auto &cmd_buff = draw_cmd_buffers[i]; + for (uint32_t i = 0u; i < draw_cmd_buffers.size(); ++i) + { + render_pass_begin_info.framebuffer = framebuffers[i]; + auto &cmd_buff = draw_cmd_buffers[i]; - VK_CHECK(vkBeginCommandBuffer(cmd_buff, &command_buffer_begin_info)); + VK_CHECK(vkBeginCommandBuffer(cmd_buff, &command_buffer_begin_info)); - vkCmdBeginRenderPass(cmd_buff, &render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); + vkCmdBeginRenderPass(cmd_buff, &render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); - VkViewport viewport = vkb::initializers::viewport(static_cast(width), static_cast(height), 0.0f, 1.0f); - vkCmdSetViewport(cmd_buff, 0u, 1u, &viewport); + VkViewport viewport = vkb::initializers::viewport(static_cast(width), static_cast(height), 0.0f, 1.0f); + vkCmdSetViewport(cmd_buff, 0u, 1u, &viewport); - VkRect2D scissor = vkb::initializers::rect2D(width, height, 0, 0); - vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); + VkRect2D scissor = vkb::initializers::rect2D(width, height, 0, 0); + vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); - // draw ocean - { - vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); - vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline); + // draw ocean + { + vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); + vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ui.wireframe ? ocean.pipelines.wireframe.pipeline : ocean.pipelines._default.pipeline); - VkDeviceSize offset[] = {0}; - vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, init_grid.vertex->get(), offset); - vkCmdBindIndexBuffer(cmd_buff, init_grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); + VkDeviceSize offset[] = {0}; + vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, init_grid.vertex->get(), offset); + vkCmdBindIndexBuffer(cmd_buff, init_grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); - vkCmdDrawIndexed(cmd_buff, init_grid.index_count, 1u, 0u, 0u, 0u); - } + vkCmdDrawIndexed(cmd_buff, init_grid.index_count, 1u, 0u, 0u, 0u); + } - draw_ui(cmd_buff); + draw_ui(cmd_buff); - vkCmdEndRenderPass(cmd_buff); + vkCmdEndRenderPass(cmd_buff); - VK_CHECK(vkEndCommandBuffer(cmd_buff)); - } + VK_CHECK(vkEndCommandBuffer(cmd_buff)); + } } void SubgroupsOperations::draw() { - VkPipelineStageFlags graphics_wait_stage_masks[] = {VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT}; - VkSemaphore graphics_wait_semaphores[] = {compute.semaphore, semaphores.acquired_image_ready}; - VkSemaphore graphics_signal_semaphores[] = {VK_NULL_HANDLE, semaphores.render_complete}; - - ApiVulkanSample::prepare_frame(); - submit_info.commandBufferCount = 1; - submit_info.pCommandBuffers = &draw_cmd_buffers[current_buffer]; - // submit_info.waitSemaphoreCount = 2; - // submit_info.pWaitSemaphores = graphics_wait_semaphores; - // submit_info.pWaitDstStageMask = graphics_wait_stage_masks; - // submit_info.signalSemaphoreCount = 2; - // submit_info.pSignalSemaphores = graphics_signal_semaphores; - VK_CHECK(vkQueueSubmit(queue, 1u, &submit_info, VK_NULL_HANDLE)); - ApiVulkanSample::submit_frame(); - - // Wait for rendering finished - VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; - - // Submit compute commands - VkSubmitInfo compute_submit_info = vkb::initializers::submit_info(); - compute_submit_info.commandBufferCount = 1u; - compute_submit_info.pCommandBuffers = &compute.command_buffer; - compute_submit_info.waitSemaphoreCount = 1u; - - compute_submit_info.pWaitDstStageMask = &wait_stage_mask; - compute_submit_info.signalSemaphoreCount = 1u; - compute_submit_info.pSignalSemaphores = &compute.semaphore; - - // VK_CHECK(vkQueueSubmit(compute.queue, 1u, &compute_submit_info, VK_NULL_HANDLE)); + VkPipelineStageFlags graphics_wait_stage_masks[] = {VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT}; + VkSemaphore graphics_wait_semaphores[] = {compute.semaphore, semaphores.acquired_image_ready}; + VkSemaphore graphics_signal_semaphores[] = {VK_NULL_HANDLE, semaphores.render_complete}; + + ApiVulkanSample::prepare_frame(); + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &draw_cmd_buffers[current_buffer]; + // submit_info.waitSemaphoreCount = 2; + // submit_info.pWaitSemaphores = graphics_wait_semaphores; + // submit_info.pWaitDstStageMask = graphics_wait_stage_masks; + // submit_info.signalSemaphoreCount = 2; + // submit_info.pSignalSemaphores = graphics_signal_semaphores; + VK_CHECK(vkQueueSubmit(queue, 1u, &submit_info, VK_NULL_HANDLE)); + ApiVulkanSample::submit_frame(); + + // Wait for rendering finished + VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; + + // Submit compute commands + VkSubmitInfo compute_submit_info = vkb::initializers::submit_info(); + compute_submit_info.commandBufferCount = 1u; + compute_submit_info.pCommandBuffers = &compute.command_buffer; + compute_submit_info.waitSemaphoreCount = 1u; + + compute_submit_info.pWaitDstStageMask = &wait_stage_mask; + compute_submit_info.signalSemaphoreCount = 1u; + compute_submit_info.pSignalSemaphores = &compute.semaphore; + + // VK_CHECK(vkQueueSubmit(compute.queue, 1u, &compute_submit_info, VK_NULL_HANDLE)); } void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) { + if (drawer.header("Settings")) + { + if (get_device().get_gpu().get_features().fillModeNonSolid) + { + if (drawer.checkbox("Wireframe", &ui.wireframe)) + { + build_command_buffers(); + } + } + } } bool SubgroupsOperations::resize(const uint32_t width, const uint32_t height) { - if (!ApiVulkanSample::resize(width, height)) - return false; - // build_compute_command_buffer(); - build_command_buffers(); - return true; + if (!ApiVulkanSample::resize(width, height)) + return false; + // build_compute_command_buffer(); + build_command_buffers(); + return true; } void SubgroupsOperations::render(float delta_time) { - if (!prepared) - { - return; - } - draw(); - if (camera.updated) - { - update_uniform_buffers(); - } + if (!prepared) + { + return; + } + draw(); + if (camera.updated) + { + update_uniform_buffers(); + } } std::unique_ptr create_subgroups_operations() { - return std::make_unique(); + return std::make_unique(); } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 4c0b9fcea..fdb9bd6a9 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -22,100 +22,106 @@ class SubgroupsOperations : public ApiVulkanSample { public: - SubgroupsOperations(); - ~SubgroupsOperations(); - - bool prepare(vkb::Platform &platform) override; - void request_gpu_features(vkb::PhysicalDevice &gpu) override; - void build_command_buffers() override; - void render(float delta_time) override; - bool resize(const uint32_t width, const uint32_t height) override; - void on_update_ui_overlay(vkb::Drawer &drawer) override; - - void draw(); - void load_assets(); - - void prepare_compute(); - void create_compute_queue(); - void create_compute_command_pool(); - void create_compute_command_buffer(); - void create_compute_descriptor_set_layout(); - void create_compute_descriptor_set(); - - void preapre_compute_pipeline_layout(); - void prepare_compute_pipeline(); - - void build_compute_command_buffer(); - - /////////////////////////////////////////////// - void generate_grid(); - void prepare_uniform_buffers(); - void setup_descriptor_pool(); - void create_descriptor_set_layout(); - void create_descriptor_set(); - void create_pipelines(); - - void update_uniform_buffers(); - - - - struct Pipeline - { - void destroy(VkDevice device); - - VkPipeline pipeline; - VkPipelineLayout pipeline_layout; - }; - - struct GridBuffers - { - std::unique_ptr vertex; - std::unique_ptr index; - uint32_t index_count = {0u}; - }; - - struct CameraUbo - { - alignas(16) glm::mat4 projection; - alignas(16) glm::mat4 view; - alignas(16) glm::mat4 model; - }; - - GridBuffers init_grid; // input buffer for compute shader - - uint32_t grid_size = {128u}; - - std::unique_ptr camera_ubo; - - struct - { - VkQueue queue; - VkCommandPool command_pool; - VkCommandBuffer command_buffer; - VkSemaphore semaphore; - VkDescriptorSetLayout descriptor_set_layout; - VkDescriptorSet descriptor_set; - uint32_t queue_family_index; - - struct - { - Pipeline _default; - } pipelines; - } compute; - - struct - { - GridBuffers grid; // output (result) buffer for compute shader - - VkDescriptorSetLayout descriptor_set_layout; - VkDescriptorSet descriptor_set; - struct - { - Pipeline _default; - } pipelines; - } ocean; - - VkPhysicalDeviceSubgroupProperties subgroups_properties; + SubgroupsOperations(); + ~SubgroupsOperations(); + + bool prepare(vkb::Platform &platform) override; + void request_gpu_features(vkb::PhysicalDevice &gpu) override; + void build_command_buffers() override; + void render(float delta_time) override; + bool resize(const uint32_t width, const uint32_t height) override; + void on_update_ui_overlay(vkb::Drawer &drawer) override; + + void draw(); + void load_assets(); + + void prepare_compute(); + void create_compute_queue(); + void create_compute_command_pool(); + void create_compute_command_buffer(); + void create_compute_descriptor_set_layout(); + void create_compute_descriptor_set(); + + void preapre_compute_pipeline_layout(); + void prepare_compute_pipeline(); + + void build_compute_command_buffer(); + + /////////////////////////////////////////////// + void generate_grid(); + void prepare_uniform_buffers(); + void setup_descriptor_pool(); + void create_descriptor_set_layout(); + void create_descriptor_set(); + void create_pipelines(); + + void update_uniform_buffers(); + + + + struct Pipeline + { + void destroy(VkDevice device); + + VkPipeline pipeline; + VkPipelineLayout pipeline_layout; + }; + + struct GridBuffers + { + std::unique_ptr vertex; + std::unique_ptr index; + uint32_t index_count = {0u}; + }; + + struct CameraUbo + { + alignas(16) glm::mat4 projection; + alignas(16) glm::mat4 view; + alignas(16) glm::mat4 model; + }; + + struct GuiConfig + { + bool wireframe = {false}; + } ui; + + GridBuffers init_grid; // input buffer for compute shader + + uint32_t grid_size = {128u}; + + std::unique_ptr camera_ubo; + + struct + { + VkQueue queue; + VkCommandPool command_pool; + VkCommandBuffer command_buffer; + VkSemaphore semaphore; + VkDescriptorSetLayout descriptor_set_layout; + VkDescriptorSet descriptor_set; + uint32_t queue_family_index; + + struct + { + Pipeline _default; + } pipelines; + } compute; + + struct + { + GridBuffers grid; // output (result) buffer for compute shader + + VkDescriptorSetLayout descriptor_set_layout; + VkDescriptorSet descriptor_set; + struct + { + Pipeline _default; + Pipeline wireframe; + } pipelines; + } ocean; + + VkPhysicalDeviceSubgroupProperties subgroups_properties; }; std::unique_ptr create_subgroups_operations(); From 2b57067477b45b33aaa647a21ee7d0c18ee6e80c Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 14 Jun 2023 13:52:02 +0200 Subject: [PATCH 04/53] Add compute pipeline and compute queue --- .../subgroups_operations.cpp | 107 +++++++++++------- .../subgroups_operations.h | 6 + .../ocean_horizontal.comp | 50 ++++++++ 3 files changed, 122 insertions(+), 41 deletions(-) create mode 100644 shaders/subgroups_operations/ocean_horizontal.comp diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 6bfcbaa5e..1de76469e 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -16,6 +16,7 @@ */ #include "subgroups_operations.h" +#include void SubgroupsOperations::Pipeline::destroy(VkDevice device) { @@ -37,6 +38,7 @@ SubgroupsOperations::SubgroupsOperations() camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); camera.set_position({0.0f, 0.0f, -2.0f}); + add_device_extension(VK_KHR_SPIRV_1_4_EXTENSION_NAME); add_device_extension(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); add_device_extension(VK_EXT_SHADER_SUBGROUP_BALLOT_EXTENSION_NAME); // is needed??? add_device_extension(VK_EXT_SHADER_SUBGROUP_VOTE_EXTENSION_NAME); // is needed??? @@ -46,10 +48,10 @@ SubgroupsOperations::~SubgroupsOperations() { if (device) { - // compute.pipelines._default.destroy(get_device().get_handle()); - // vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); - // vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); - // vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); + compute.pipelines._default.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); + vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); + vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); ocean.pipelines._default.destroy(get_device().get_handle()); ocean.pipelines.wireframe.destroy(get_device().get_handle()); @@ -65,12 +67,11 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) } load_assets(); - // prepare_compute(); - - // graphics pipeline - generate_grid(); - prepare_uniform_buffers(); setup_descriptor_pool(); + prepare_uniform_buffers(); + prepare_compute(); + + // grpahics pipeline create_descriptor_set_layout(); create_descriptor_set(); create_pipelines(); @@ -90,7 +91,7 @@ void SubgroupsOperations::prepare_compute() create_compute_descriptor_set(); preapre_compute_pipeline_layout(); prepare_compute_pipeline(); - // build_compute_command_buffer(); + build_compute_command_buffer(); } void SubgroupsOperations::create_compute_queue() @@ -128,21 +129,30 @@ void SubgroupsOperations::create_compute_command_buffer() void SubgroupsOperations::create_compute_descriptor_set_layout() { - // std::vector set_layout_bindngs = { - // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0), // input image - // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1), // result image - // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2) // kernel matrix - // }; - // VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); - - // VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), // time ubo + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), // input vertex + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) // output vertex + }; + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); } void SubgroupsOperations::create_compute_descriptor_set() { - VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1); - - // VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); + + VkDescriptorBufferInfo time_ubo_buffer = create_descriptor(*time_ubo); + VkDescriptorBufferInfo input_vertices_buffer = create_descriptor(*init_grid.vertex); + VkDescriptorBufferInfo output_vertices_buffer = create_descriptor(*init_grid.vertex); // TODO: change on output buffer (temporary solution) + std::vector wirte_descriptor_sets = { + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &time_ubo_buffer), + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &input_vertices_buffer), + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &output_vertices_buffer) + }; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(wirte_descriptor_sets.size()), wirte_descriptor_sets.data(), 0u, nullptr); } void SubgroupsOperations::preapre_compute_pipeline_layout() @@ -160,7 +170,7 @@ void SubgroupsOperations::prepare_compute_pipeline() VkComputePipelineCreateInfo computeInfo = {}; computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; computeInfo.layout = compute.pipelines._default.pipeline_layout; - computeInfo.stage = load_shader("subgroups_operations/blur.comp", VK_SHADER_STAGE_COMPUTE_BIT); + computeInfo.stage = load_shader("subgroups_operations/ocean_horizontal.comp", VK_SHADER_STAGE_COMPUTE_BIT); VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines._default.pipeline)); } @@ -178,7 +188,7 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); - // vkCmdDispatch(compute.command_buffer, texture_object.texture.image->get_extent().width / 32u, texture_object.texture.image->get_extent().height / 32u, 1u); + vkCmdDispatch(compute.command_buffer, grid_size / 16u, grid_size / 16u, 1u); VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); } @@ -189,6 +199,12 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) { gpu.get_mutable_requested_features().samplerAnisotropy = VK_TRUE; } + + if (gpu.get_features().fillModeNonSolid) + { + gpu.get_mutable_requested_features().fillModeNonSolid = VK_TRUE; + } + subgroups_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; subgroups_properties.pNext = VK_NULL_HANDLE; @@ -206,6 +222,7 @@ void SubgroupsOperations::load_assets() void SubgroupsOperations::prepare_uniform_buffers() { camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); update_uniform_buffers(); } @@ -241,7 +258,7 @@ void SubgroupsOperations::generate_grid() } } - init_grid.index_count = static_cast(grid_indices.size()); + ocean.grid.index_count = static_cast(grid_indices.size()); auto vertex_buffer_size = vkb::to_u32(grid_vertices.size() * sizeof(Vertex)); auto index_buffer_size = vkb::to_u32(grid_indices.size() * sizeof(uint32_t)); init_grid.vertex = std::make_unique(get_device(), @@ -250,17 +267,18 @@ void SubgroupsOperations::generate_grid() VMA_MEMORY_USAGE_CPU_TO_GPU); init_grid.vertex->update(grid_vertices.data(), vertex_buffer_size); - init_grid.index = std::make_unique(get_device(), + ocean.grid.index = std::make_unique(get_device(), index_buffer_size, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - init_grid.index->update(grid_indices.data(), index_buffer_size); + ocean.grid.index->update(grid_indices.data(), index_buffer_size); } void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1u)}; + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u)}; VkDescriptorPoolCreateInfo descriptor_pool_create_info = vkb::initializers::descriptor_pool_create_info( static_cast(pool_sizes.size()), @@ -369,12 +387,12 @@ void SubgroupsOperations::create_pipelines() { rasterization_state.polygonMode = VK_POLYGON_MODE_LINE; VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines.wireframe.pipeline)); - } } void SubgroupsOperations::update_uniform_buffers() { + auto t1 = std::chrono::high_resolution_clock::now(); CameraUbo ubo; ubo.model = glm::mat4(1.0f); ubo.model = glm::translate(ubo.model, glm::vec3(0.0f)); @@ -382,6 +400,13 @@ void SubgroupsOperations::update_uniform_buffers() ubo.projection = camera.matrices.perspective; camera_ubo->convert_and_update(ubo); + + TimeUbo t; + auto t2 = std::chrono::high_resolution_clock::now(); + + std::chrono::duration float_time = t1 - t2; // dummy time + t.time = float_time.count(); + time_ubo->convert_and_update(t); } void SubgroupsOperations::build_command_buffers() @@ -421,9 +446,9 @@ void SubgroupsOperations::build_command_buffers() VkDeviceSize offset[] = {0}; vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, init_grid.vertex->get(), offset); - vkCmdBindIndexBuffer(cmd_buff, init_grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); + vkCmdBindIndexBuffer(cmd_buff, ocean.grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); - vkCmdDrawIndexed(cmd_buff, init_grid.index_count, 1u, 0u, 0u, 0u); + vkCmdDrawIndexed(cmd_buff, ocean.grid.index_count, 1u, 0u, 0u, 0u); } draw_ui(cmd_buff); @@ -443,11 +468,11 @@ void SubgroupsOperations::draw() ApiVulkanSample::prepare_frame(); submit_info.commandBufferCount = 1; submit_info.pCommandBuffers = &draw_cmd_buffers[current_buffer]; - // submit_info.waitSemaphoreCount = 2; - // submit_info.pWaitSemaphores = graphics_wait_semaphores; - // submit_info.pWaitDstStageMask = graphics_wait_stage_masks; - // submit_info.signalSemaphoreCount = 2; - // submit_info.pSignalSemaphores = graphics_signal_semaphores; +// submit_info.waitSemaphoreCount = 2; +// submit_info.pWaitSemaphores = graphics_wait_semaphores; +// submit_info.pWaitDstStageMask = graphics_wait_stage_masks; +// submit_info.signalSemaphoreCount = 2; +// submit_info.pSignalSemaphores = graphics_signal_semaphores; VK_CHECK(vkQueueSubmit(queue, 1u, &submit_info, VK_NULL_HANDLE)); ApiVulkanSample::submit_frame(); @@ -458,13 +483,13 @@ void SubgroupsOperations::draw() VkSubmitInfo compute_submit_info = vkb::initializers::submit_info(); compute_submit_info.commandBufferCount = 1u; compute_submit_info.pCommandBuffers = &compute.command_buffer; - compute_submit_info.waitSemaphoreCount = 1u; +// compute_submit_info.waitSemaphoreCount = 1u; - compute_submit_info.pWaitDstStageMask = &wait_stage_mask; - compute_submit_info.signalSemaphoreCount = 1u; - compute_submit_info.pSignalSemaphores = &compute.semaphore; +// compute_submit_info.pWaitDstStageMask = &wait_stage_mask; +// compute_submit_info.signalSemaphoreCount = 1u; +// compute_submit_info.pSignalSemaphores = &compute.semaphore; - // VK_CHECK(vkQueueSubmit(compute.queue, 1u, &compute_submit_info, VK_NULL_HANDLE)); + VK_CHECK(vkQueueSubmit(compute.queue, 1u, &compute_submit_info, VK_NULL_HANDLE)); } void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) @@ -485,7 +510,7 @@ bool SubgroupsOperations::resize(const uint32_t width, const uint32_t height) { if (!ApiVulkanSample::resize(width, height)) return false; - // build_compute_command_buffer(); + build_compute_command_buffer(); build_command_buffers(); return true; } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index fdb9bd6a9..e3195c6b6 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -81,6 +81,11 @@ class SubgroupsOperations : public ApiVulkanSample alignas(16) glm::mat4 model; }; + struct TimeUbo + { + alignas(4) float time; + }; + struct GuiConfig { bool wireframe = {false}; @@ -91,6 +96,7 @@ class SubgroupsOperations : public ApiVulkanSample uint32_t grid_size = {128u}; std::unique_ptr camera_ubo; + std::unique_ptr time_ubo; struct { diff --git a/shaders/subgroups_operations/ocean_horizontal.comp b/shaders/subgroups_operations/ocean_horizontal.comp new file mode 100644 index 000000000..b30005f9f --- /dev/null +++ b/shaders/subgroups_operations/ocean_horizontal.comp @@ -0,0 +1,50 @@ +#version 450 +#extension GL_KHR_shader_subgroup_basic : enable + +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +layout (local_size_x = 16, local_size_y = 16, local_size_z = 1) in; + +struct Vertex { + vec3 position; +}; + +layout (binding = 0) uniform TimeUbo +{ + float time; +} uboTime; + +layout (std140, binding = 1) readonly buffer VertexInBuffer +{ + Vertex verticesIn[]; +}; + +layout (std140, binding = 2) buffer VertexOutBuffer +{ + Vertex verticesOut[]; +}; + +void main() +{ + uint idx = gl_GlobalInvocationID.x + gl_GlobalInvocationID.y; + Vertex inVertex = verticesIn[idx]; + inVertex.position.x = inVertex.position.x * sin(uboTime.time); + // subgroupBarrier(); + verticesOut[idx].position = inVertex.position; + +} From ea486ea265a707e80a01651220f529628dcca11e Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Fri, 16 Jun 2023 14:33:04 +0200 Subject: [PATCH 05/53] Add ability to resize the grid in the UI and a few small changes --- .../subgroups_operations.cpp | 761 +++++++++--------- .../subgroups_operations.h | 216 ++--- shaders/subgroups_operations/base.frag | 28 - shaders/subgroups_operations/base.vert | 35 - shaders/subgroups_operations/blur.comp | 67 -- .../{ocean_horizontal.comp => ocean_fft.comp} | 11 +- 6 files changed, 503 insertions(+), 615 deletions(-) delete mode 100644 shaders/subgroups_operations/base.frag delete mode 100644 shaders/subgroups_operations/base.vert delete mode 100644 shaders/subgroups_operations/blur.comp rename shaders/subgroups_operations/{ocean_horizontal.comp => ocean_fft.comp} (79%) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 1de76469e..d62d6b8c4 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -20,515 +20,528 @@ void SubgroupsOperations::Pipeline::destroy(VkDevice device) { - if (pipeline != VK_NULL_HANDLE) - { - vkDestroyPipeline(device, pipeline, nullptr); - } - if (pipeline_layout != VK_NULL_HANDLE) - { - vkDestroyPipelineLayout(device, pipeline_layout, nullptr); - } + if (pipeline != VK_NULL_HANDLE) + { + vkDestroyPipeline(device, pipeline, nullptr); + } + if (pipeline_layout != VK_NULL_HANDLE) + { + vkDestroyPipelineLayout(device, pipeline_layout, nullptr); + } } SubgroupsOperations::SubgroupsOperations() { - set_api_version(VK_API_VERSION_1_1); - title = "Subgroups operations"; - camera.type = vkb::CameraType::LookAt; - - camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); - camera.set_position({0.0f, 0.0f, -2.0f}); - add_device_extension(VK_KHR_SPIRV_1_4_EXTENSION_NAME); - add_device_extension(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); - add_device_extension(VK_EXT_SHADER_SUBGROUP_BALLOT_EXTENSION_NAME); // is needed??? - add_device_extension(VK_EXT_SHADER_SUBGROUP_VOTE_EXTENSION_NAME); // is needed??? -} + // SPIRV 1.4 requires Vulkan 1.1 + set_api_version(VK_API_VERSION_1_1); + + // Subgroup size control extensions required by this sample + add_device_extension(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); + + // Required for VK_EXT_subgroup_size_control + add_device_extension(VK_KHR_SPIRV_1_4_EXTENSION_NAME); + + // Required by VK_KHR_spirv_1_4 + add_device_extension(VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME); + title = "Subgroups operations"; + camera.type = vkb::CameraType::LookAt; + + camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); + camera.set_position({0.0f, 0.0f, -2.0f}); +} SubgroupsOperations::~SubgroupsOperations() { - if (device) - { - compute.pipelines._default.destroy(get_device().get_handle()); - vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); - vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); - vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); - - ocean.pipelines._default.destroy(get_device().get_handle()); - ocean.pipelines.wireframe.destroy(get_device().get_handle()); - vkDestroyDescriptorSetLayout(get_device().get_handle(), ocean.descriptor_set_layout, nullptr); - } + if (device) + { + compute.pipelines._default.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); + vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); + vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); + + ocean.pipelines._default.destroy(get_device().get_handle()); + ocean.pipelines.wireframe.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), ocean.descriptor_set_layout, nullptr); + } } bool SubgroupsOperations::prepare(vkb::Platform &platform) { - if (!ApiVulkanSample::prepare(platform)) - { - return false; - } + if (!ApiVulkanSample::prepare(platform)) + { + return false; + } - load_assets(); - setup_descriptor_pool(); - prepare_uniform_buffers(); - prepare_compute(); + load_assets(); + setup_descriptor_pool(); + prepare_uniform_buffers(); + prepare_compute(); - // grpahics pipeline - create_descriptor_set_layout(); - create_descriptor_set(); - create_pipelines(); + // prepare grpahics pipeline + create_descriptor_set_layout(); + create_descriptor_set(); + create_pipelines(); - build_command_buffers(); + build_command_buffers(); - prepared = true; - return true; + prepared = true; + return true; } void SubgroupsOperations::prepare_compute() { - create_compute_queue(); - create_compute_command_pool(); - create_compute_command_buffer(); - create_compute_descriptor_set_layout(); - create_compute_descriptor_set(); - preapre_compute_pipeline_layout(); - prepare_compute_pipeline(); - build_compute_command_buffer(); + create_compute_queue(); + create_compute_command_pool(); + create_compute_command_buffer(); + create_compute_descriptor_set_layout(); + create_compute_descriptor_set(); + preapre_compute_pipeline_layout(); + prepare_compute_pipeline(); + build_compute_command_buffer(); } void SubgroupsOperations::create_compute_queue() { - // create compute queue and get family index - compute.queue_family_index = get_device().get_queue_family_index(VK_QUEUE_COMPUTE_BIT); + // create compute queue and get family index + compute.queue_family_index = get_device().get_queue_family_index(VK_QUEUE_COMPUTE_BIT); - vkGetDeviceQueue(get_device().get_handle(), compute.queue_family_index, 0u, &compute.queue); + vkGetDeviceQueue(get_device().get_handle(), compute.queue_family_index, 0u, &compute.queue); } void SubgroupsOperations::create_compute_command_pool() { - VkCommandPoolCreateInfo command_pool_create_info = {}; - command_pool_create_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; - command_pool_create_info.queueFamilyIndex = compute.queue_family_index; - command_pool_create_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; - VK_CHECK(vkCreateCommandPool(get_device().get_handle(), &command_pool_create_info, nullptr, &compute.command_pool)); + VkCommandPoolCreateInfo command_pool_create_info = {}; + command_pool_create_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + command_pool_create_info.queueFamilyIndex = compute.queue_family_index; + command_pool_create_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + VK_CHECK(vkCreateCommandPool(get_device().get_handle(), &command_pool_create_info, nullptr, &compute.command_pool)); } void SubgroupsOperations::create_compute_command_buffer() { - // Create a command buffer for compute operations - VkCommandBufferAllocateInfo command_buffer_allocate_info = - vkb::initializers::command_buffer_allocate_info( - compute.command_pool, - VK_COMMAND_BUFFER_LEVEL_PRIMARY, - 1u); - - VK_CHECK(vkAllocateCommandBuffers(get_device().get_handle(), &command_buffer_allocate_info, &compute.command_buffer)); - - // Semaphore for compute & graphics sync - VkSemaphoreCreateInfo semaphore_create_info = vkb::initializers::semaphore_create_info(); - VK_CHECK(vkCreateSemaphore(get_device().get_handle(), &semaphore_create_info, nullptr, &compute.semaphore)); + // Create a command buffer for compute operations + VkCommandBufferAllocateInfo command_buffer_allocate_info = + vkb::initializers::command_buffer_allocate_info( + compute.command_pool, + VK_COMMAND_BUFFER_LEVEL_PRIMARY, + 1u); + + VK_CHECK(vkAllocateCommandBuffers(get_device().get_handle(), &command_buffer_allocate_info, &compute.command_buffer)); + + // Semaphore for compute & graphics sync + VkSemaphoreCreateInfo semaphore_create_info = vkb::initializers::semaphore_create_info(); + VK_CHECK(vkCreateSemaphore(get_device().get_handle(), &semaphore_create_info, nullptr, &compute.semaphore)); } void SubgroupsOperations::create_compute_descriptor_set_layout() { - std::vector set_layout_bindngs = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), // time ubo - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), // input vertex - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) // output vertex - }; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); - - VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), // ubo + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), // input vertex + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) // output vertex + }; + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); } void SubgroupsOperations::create_compute_descriptor_set() { - VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1u); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); - VkDescriptorBufferInfo time_ubo_buffer = create_descriptor(*time_ubo); - VkDescriptorBufferInfo input_vertices_buffer = create_descriptor(*init_grid.vertex); - VkDescriptorBufferInfo output_vertices_buffer = create_descriptor(*init_grid.vertex); // TODO: change on output buffer (temporary solution) - std::vector wirte_descriptor_sets = { + VkDescriptorBufferInfo time_ubo_buffer = create_descriptor(*compute_ubo); + VkDescriptorBufferInfo input_vertices_buffer = create_descriptor(*input_grid.vertex); + VkDescriptorBufferInfo output_vertices_buffer = create_descriptor(*ocean.grid.vertex); + std::vector wirte_descriptor_sets = { vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &time_ubo_buffer), vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &input_vertices_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &output_vertices_buffer) - }; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(wirte_descriptor_sets.size()), wirte_descriptor_sets.data(), 0u, nullptr); + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &output_vertices_buffer)}; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(wirte_descriptor_sets.size()), wirte_descriptor_sets.data(), 0u, nullptr); } void SubgroupsOperations::preapre_compute_pipeline_layout() { - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; - compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - compute_pipeline_layout_info.setLayoutCount = 1u; - compute_pipeline_layout_info.pSetLayouts = &compute.descriptor_set_layout; + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &compute.descriptor_set_layout; - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &compute.pipelines._default.pipeline_layout)); + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &compute.pipelines._default.pipeline_layout)); } void SubgroupsOperations::prepare_compute_pipeline() { - VkComputePipelineCreateInfo computeInfo = {}; - computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; - computeInfo.layout = compute.pipelines._default.pipeline_layout; - computeInfo.stage = load_shader("subgroups_operations/ocean_horizontal.comp", VK_SHADER_STAGE_COMPUTE_BIT); + VkComputePipelineCreateInfo computeInfo = {}; + computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + computeInfo.layout = compute.pipelines._default.pipeline_layout; + computeInfo.stage = load_shader("subgroups_operations/ocean_fft.comp", VK_SHADER_STAGE_COMPUTE_BIT); - VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines._default.pipeline)); + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines._default.pipeline)); } void SubgroupsOperations::build_compute_command_buffer() { - VkSubmitInfo submit_info = {}; - submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; - vkResetCommandBuffer(compute.command_buffer, 0u); + VkSubmitInfo submit_info = {}; + submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + vkResetCommandBuffer(compute.command_buffer, 0u); - // record command - VkCommandBufferBeginInfo begin_info = {}; - begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; - VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); + // record command + VkCommandBufferBeginInfo begin_info = {}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, grid_size / 16u, grid_size / 16u, 1u); + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, grid_size / 16u, grid_size / 16u, 1u); - VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); + VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); } void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) { - if (gpu.get_features().samplerAnisotropy) - { - gpu.get_mutable_requested_features().samplerAnisotropy = VK_TRUE; - } - - if (gpu.get_features().fillModeNonSolid) - { - gpu.get_mutable_requested_features().fillModeNonSolid = VK_TRUE; - } - - subgroups_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; - subgroups_properties.pNext = VK_NULL_HANDLE; - - VkPhysicalDeviceProperties2 device_properties2 = {}; - device_properties2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; - device_properties2.pNext = &subgroups_properties; - vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); + if (gpu.get_features().fillModeNonSolid) + { + gpu.get_mutable_requested_features().fillModeNonSolid = VK_TRUE; + } + + subgroups_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; + subgroups_properties.pNext = VK_NULL_HANDLE; + + VkPhysicalDeviceProperties2 device_properties2 = {}; + device_properties2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + device_properties2.pNext = &subgroups_properties; + vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); } void SubgroupsOperations::load_assets() { - generate_grid(); + generate_grid(); } void SubgroupsOperations::prepare_uniform_buffers() { - camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - update_uniform_buffers(); + camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + compute_ubo = std::make_unique(get_device(), sizeof(ComputeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + update_uniform_buffers(); } void SubgroupsOperations::generate_grid() { - std::vector grid_vertices; - std::vector grid_indices; - - for (uint32_t z = 0; z <= grid_size; ++z) - { - for (uint32_t x = 0; x <= grid_size; ++x) - { - Vertex point; - point.pos.x = static_cast(x) / static_cast(grid_size); - point.pos.z = static_cast(z) / static_cast(grid_size); - grid_vertices.push_back(point); - } - } - - for (uint32_t z = 0; z < grid_size; ++z) - { - for (uint32_t x = 0; x < grid_size; ++x) - { - uint32_t row1 = z * (grid_size + 1); - uint32_t row2 = (z + 1) * (grid_size + 1); - grid_indices.push_back(row1 + x + 0); - grid_indices.push_back(row1 + x + 1); - grid_indices.push_back(row2 + x + 1); - - grid_indices.push_back(row1 + x + 0); - grid_indices.push_back(row2 + x + 1); - grid_indices.push_back(row2 + x); - } - } - - ocean.grid.index_count = static_cast(grid_indices.size()); - auto vertex_buffer_size = vkb::to_u32(grid_vertices.size() * sizeof(Vertex)); - auto index_buffer_size = vkb::to_u32(grid_indices.size() * sizeof(uint32_t)); - init_grid.vertex = std::make_unique(get_device(), - vertex_buffer_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, - VMA_MEMORY_USAGE_CPU_TO_GPU); - init_grid.vertex->update(grid_vertices.data(), vertex_buffer_size); - - ocean.grid.index = std::make_unique(get_device(), - index_buffer_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, - VMA_MEMORY_USAGE_CPU_TO_GPU); - ocean.grid.index->update(grid_indices.data(), index_buffer_size); + grid_size = static_cast(stoi(ui.grid_sizes[ui.grid_size_idx])); + std::vector grid_vertices; + std::vector grid_indices; + + for (uint32_t z = 0; z <= grid_size; ++z) + { + for (uint32_t x = 0; x <= grid_size; ++x) + { + Vertex point = {}; + point.pos.x = static_cast(x) / static_cast(grid_size); + point.pos.z = static_cast(z) / static_cast(grid_size); + point.pos.y = 0.0f; + grid_vertices.push_back(point); + } + } + + for (uint32_t z = 0; z < grid_size; ++z) + { + for (uint32_t x = 0; x < grid_size; ++x) + { + uint32_t i0 = z * (grid_size + 1) + x; + uint32_t i1 = i0 + 1; + uint32_t i2 = i0 + (grid_size + 1); + uint32_t i3 = i2 + 1; + + grid_indices.push_back(i0); + grid_indices.push_back(i2); + grid_indices.push_back(i1); + + grid_indices.push_back(i1); + grid_indices.push_back(i2); + grid_indices.push_back(i3); + } + } + + ocean.grid.index_count = vkb::to_u32(grid_indices.size()); + auto vertex_buffer_size = vkb::to_u32(grid_vertices.size() * sizeof(Vertex)); + auto index_buffer_size = vkb::to_u32(grid_indices.size() * sizeof(uint32_t)); + + if (input_grid.vertex) + { + input_grid.vertex.reset(nullptr); + } + if (ocean.grid.vertex && ocean.grid.index) + { + ocean.grid.vertex.reset(nullptr); + ocean.grid.index.reset(nullptr); + } + + input_grid.vertex = std::make_unique(get_device(), + vertex_buffer_size, + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); + input_grid.vertex->update(grid_vertices.data(), vertex_buffer_size); + + ocean.grid.index = std::make_unique(get_device(), + index_buffer_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); + ocean.grid.index->update(grid_indices.data(), index_buffer_size); + + ocean.grid.vertex = std::make_unique(get_device(), + vertex_buffer_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VMA_MEMORY_USAGE_GPU_ONLY); } void SubgroupsOperations::setup_descriptor_pool() { - std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u), - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u)}; - VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info( - static_cast(pool_sizes.size()), - pool_sizes.data(), - 2u); - VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); + std::vector pool_sizes = { + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u)}; + VkDescriptorPoolCreateInfo descriptor_pool_create_info = + vkb::initializers::descriptor_pool_create_info( + static_cast(pool_sizes.size()), + pool_sizes.data(), + 2u); + VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } void SubgroupsOperations::create_descriptor_set_layout() { - std::vector set_layout_bindings = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT, 0u) // ubo - }; + std::vector set_layout_bindings = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT, 0u)}; - VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); - VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); + VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); - VkPipelineLayoutCreateInfo pipeline_layout_create_info = vkb::initializers::pipeline_layout_create_info(&ocean.descriptor_set_layout); - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &pipeline_layout_create_info, nullptr, &ocean.pipelines._default.pipeline_layout)); + VkPipelineLayoutCreateInfo pipeline_layout_create_info = vkb::initializers::pipeline_layout_create_info(&ocean.descriptor_set_layout); + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &pipeline_layout_create_info, nullptr, &ocean.pipelines._default.pipeline_layout)); } void SubgroupsOperations::create_descriptor_set() { - VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &ocean.descriptor_set_layout, 1u); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &ocean.descriptor_set)); + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &ocean.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &ocean.descriptor_set)); - VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*camera_ubo); - std::vector write_descriptor_sets = { - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor)}; + VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*camera_ubo); + std::vector write_descriptor_sets = { + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor)}; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } void SubgroupsOperations::create_pipelines() { - VkPipelineInputAssemblyStateCreateInfo input_assembly_state = - vkb::initializers::pipeline_input_assembly_state_create_info( - VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, - 0u, - VK_FALSE); - VkPipelineRasterizationStateCreateInfo rasterization_state = - vkb::initializers::pipeline_rasterization_state_create_info( - VK_POLYGON_MODE_FILL, - VK_CULL_MODE_NONE, - VK_FRONT_FACE_COUNTER_CLOCKWISE, - 0u); - VkPipelineColorBlendAttachmentState blend_attachment_state = - vkb::initializers::pipeline_color_blend_attachment_state( - 0xf, - VK_FALSE); - VkPipelineColorBlendStateCreateInfo color_blend_state = - vkb::initializers::pipeline_color_blend_state_create_info( - 1u, - &blend_attachment_state); - VkPipelineDepthStencilStateCreateInfo depth_stencil_state = - vkb::initializers::pipeline_depth_stencil_state_create_info( - VK_TRUE, - VK_TRUE, - VK_COMPARE_OP_GREATER); - VkPipelineViewportStateCreateInfo viewport_state = - vkb::initializers::pipeline_viewport_state_create_info(1u, 1u, 0u); - VkPipelineMultisampleStateCreateInfo multisample_state = - vkb::initializers::pipeline_multisample_state_create_info( - VK_SAMPLE_COUNT_1_BIT, - 0u); - std::vector dynamic_state_enables = { - VK_DYNAMIC_STATE_VIEWPORT, - VK_DYNAMIC_STATE_SCISSOR}; - VkPipelineDynamicStateCreateInfo dynamic_state = - vkb::initializers::pipeline_dynamic_state_create_info( - dynamic_state_enables.data(), - static_cast(dynamic_state_enables.size()), - 0u); - std::array shader_stages; - shader_stages[0] = load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT); - shader_stages[1] = load_shader("subgroups_operations/ocean.frag", VK_SHADER_STAGE_FRAGMENT_BIT); - const std::vector vertex_input_bindings = { - vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX), - }; - const std::vector vertex_input_attributes = { - vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), - }; - VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); - vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); - vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); - vertex_input_state.vertexAttributeDescriptionCount = static_cast(vertex_input_attributes.size()); - vertex_input_state.pVertexAttributeDescriptions = vertex_input_attributes.data(); - VkGraphicsPipelineCreateInfo pipeline_create_info = - vkb::initializers::pipeline_create_info( - ocean.pipelines._default.pipeline_layout, - render_pass, - 0u); - pipeline_create_info.pVertexInputState = &vertex_input_state; - pipeline_create_info.pInputAssemblyState = &input_assembly_state; - pipeline_create_info.pRasterizationState = &rasterization_state; - pipeline_create_info.pColorBlendState = &color_blend_state; - pipeline_create_info.pMultisampleState = &multisample_state; - pipeline_create_info.pViewportState = &viewport_state; - pipeline_create_info.pDepthStencilState = &depth_stencil_state; - pipeline_create_info.pDynamicState = &dynamic_state; - pipeline_create_info.stageCount = static_cast(shader_stages.size()); - pipeline_create_info.pStages = shader_stages.data(); - VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines._default.pipeline)); - - if (get_device().get_gpu().get_features().fillModeNonSolid) - { - rasterization_state.polygonMode = VK_POLYGON_MODE_LINE; - VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines.wireframe.pipeline)); - } + VkPipelineInputAssemblyStateCreateInfo input_assembly_state = + vkb::initializers::pipeline_input_assembly_state_create_info( + VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, + 0u, + VK_FALSE); + VkPipelineRasterizationStateCreateInfo rasterization_state = + vkb::initializers::pipeline_rasterization_state_create_info( + VK_POLYGON_MODE_FILL, + VK_CULL_MODE_NONE, + VK_FRONT_FACE_COUNTER_CLOCKWISE, + 0u); + VkPipelineColorBlendAttachmentState blend_attachment_state = + vkb::initializers::pipeline_color_blend_attachment_state( + 0xf, + VK_FALSE); + VkPipelineColorBlendStateCreateInfo color_blend_state = + vkb::initializers::pipeline_color_blend_state_create_info( + 1u, + &blend_attachment_state); + VkPipelineDepthStencilStateCreateInfo depth_stencil_state = + vkb::initializers::pipeline_depth_stencil_state_create_info( + VK_TRUE, + VK_TRUE, + VK_COMPARE_OP_GREATER); + VkPipelineViewportStateCreateInfo viewport_state = + vkb::initializers::pipeline_viewport_state_create_info(1u, 1u, 0u); + VkPipelineMultisampleStateCreateInfo multisample_state = + vkb::initializers::pipeline_multisample_state_create_info( + VK_SAMPLE_COUNT_1_BIT, + 0u); + std::vector dynamic_state_enables = { + VK_DYNAMIC_STATE_VIEWPORT, + VK_DYNAMIC_STATE_SCISSOR}; + VkPipelineDynamicStateCreateInfo dynamic_state = + vkb::initializers::pipeline_dynamic_state_create_info( + dynamic_state_enables.data(), + static_cast(dynamic_state_enables.size()), + 0u); + std::array shader_stages; + shader_stages[0] = load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT); + shader_stages[1] = load_shader("subgroups_operations/ocean.frag", VK_SHADER_STAGE_FRAGMENT_BIT); + const std::vector vertex_input_bindings = { + vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX), + }; + const std::vector vertex_input_attributes = { + vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), + }; + VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); + vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); + vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); + vertex_input_state.vertexAttributeDescriptionCount = static_cast(vertex_input_attributes.size()); + vertex_input_state.pVertexAttributeDescriptions = vertex_input_attributes.data(); + VkGraphicsPipelineCreateInfo pipeline_create_info = + vkb::initializers::pipeline_create_info( + ocean.pipelines._default.pipeline_layout, + render_pass, + 0u); + pipeline_create_info.pVertexInputState = &vertex_input_state; + pipeline_create_info.pInputAssemblyState = &input_assembly_state; + pipeline_create_info.pRasterizationState = &rasterization_state; + pipeline_create_info.pColorBlendState = &color_blend_state; + pipeline_create_info.pMultisampleState = &multisample_state; + pipeline_create_info.pViewportState = &viewport_state; + pipeline_create_info.pDepthStencilState = &depth_stencil_state; + pipeline_create_info.pDynamicState = &dynamic_state; + pipeline_create_info.stageCount = static_cast(shader_stages.size()); + pipeline_create_info.pStages = shader_stages.data(); + VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines._default.pipeline)); + + if (get_device().get_gpu().get_features().fillModeNonSolid) + { + rasterization_state.polygonMode = VK_POLYGON_MODE_LINE; + VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines.wireframe.pipeline)); + } } void SubgroupsOperations::update_uniform_buffers() { - auto t1 = std::chrono::high_resolution_clock::now(); - CameraUbo ubo; - ubo.model = glm::mat4(1.0f); - ubo.model = glm::translate(ubo.model, glm::vec3(0.0f)); - ubo.view = camera.matrices.view; - ubo.projection = camera.matrices.perspective; - - camera_ubo->convert_and_update(ubo); - - TimeUbo t; - auto t2 = std::chrono::high_resolution_clock::now(); - - std::chrono::duration float_time = t1 - t2; // dummy time - t.time = float_time.count(); - time_ubo->convert_and_update(t); + CameraUbo ubo; + ubo.model = glm::mat4(1.0f); + ubo.model = glm::translate(ubo.model, glm::vec3(0.0f)); + ubo.view = camera.matrices.view; + ubo.projection = camera.matrices.perspective; + + camera_ubo->convert_and_update(ubo); + + ComputeUbo comp_ubo; + comp_ubo.grid_size = grid_size; + comp_ubo.time = 0.0f; + compute_ubo->convert_and_update(comp_ubo); } void SubgroupsOperations::build_command_buffers() { - VkCommandBufferBeginInfo command_buffer_begin_info = vkb::initializers::command_buffer_begin_info(); + VkCommandBufferBeginInfo command_buffer_begin_info = vkb::initializers::command_buffer_begin_info(); - std::array clear_values; - clear_values[0].color = {{0.0f, 0.0f, 0.0f, 0.0f}}; - clear_values[1].depthStencil = {0.0f, 0u}; + std::array clear_values; + clear_values[0].color = {{0.0f, 0.0f, 0.0f, 0.0f}}; + clear_values[1].depthStencil = {0.0f, 0u}; - VkRenderPassBeginInfo render_pass_begin_info = vkb::initializers::render_pass_begin_info(); - render_pass_begin_info.renderPass = render_pass; - render_pass_begin_info.renderArea.extent.width = width; - render_pass_begin_info.renderArea.extent.height = height; - render_pass_begin_info.clearValueCount = static_cast(clear_values.size()); - render_pass_begin_info.pClearValues = clear_values.data(); + VkRenderPassBeginInfo render_pass_begin_info = vkb::initializers::render_pass_begin_info(); + render_pass_begin_info.renderPass = render_pass; + render_pass_begin_info.renderArea.extent.width = width; + render_pass_begin_info.renderArea.extent.height = height; + render_pass_begin_info.clearValueCount = static_cast(clear_values.size()); + render_pass_begin_info.pClearValues = clear_values.data(); - for (uint32_t i = 0u; i < draw_cmd_buffers.size(); ++i) - { - render_pass_begin_info.framebuffer = framebuffers[i]; - auto &cmd_buff = draw_cmd_buffers[i]; + for (uint32_t i = 0u; i < draw_cmd_buffers.size(); ++i) + { + render_pass_begin_info.framebuffer = framebuffers[i]; + auto &cmd_buff = draw_cmd_buffers[i]; - VK_CHECK(vkBeginCommandBuffer(cmd_buff, &command_buffer_begin_info)); + VK_CHECK(vkBeginCommandBuffer(cmd_buff, &command_buffer_begin_info)); - vkCmdBeginRenderPass(cmd_buff, &render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); + vkCmdBeginRenderPass(cmd_buff, &render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); - VkViewport viewport = vkb::initializers::viewport(static_cast(width), static_cast(height), 0.0f, 1.0f); - vkCmdSetViewport(cmd_buff, 0u, 1u, &viewport); + VkViewport viewport = vkb::initializers::viewport(static_cast(width), static_cast(height), 0.0f, 1.0f); + vkCmdSetViewport(cmd_buff, 0u, 1u, &viewport); - VkRect2D scissor = vkb::initializers::rect2D(width, height, 0, 0); - vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); + VkRect2D scissor = vkb::initializers::rect2D(width, height, 0, 0); + vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); - // draw ocean - { - vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); - vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ui.wireframe ? ocean.pipelines.wireframe.pipeline : ocean.pipelines._default.pipeline); + // draw ocean + { + vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); + vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ui.wireframe ? ocean.pipelines.wireframe.pipeline : ocean.pipelines._default.pipeline); - VkDeviceSize offset[] = {0}; - vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, init_grid.vertex->get(), offset); - vkCmdBindIndexBuffer(cmd_buff, ocean.grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); + VkDeviceSize offset[] = {0}; + vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, ocean.grid.vertex->get(), offset); + vkCmdBindIndexBuffer(cmd_buff, ocean.grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); - vkCmdDrawIndexed(cmd_buff, ocean.grid.index_count, 1u, 0u, 0u, 0u); - } + vkCmdDrawIndexed(cmd_buff, ocean.grid.index_count, 1u, 0u, 0u, 0u); + } - draw_ui(cmd_buff); + draw_ui(cmd_buff); - vkCmdEndRenderPass(cmd_buff); + vkCmdEndRenderPass(cmd_buff); - VK_CHECK(vkEndCommandBuffer(cmd_buff)); - } + VK_CHECK(vkEndCommandBuffer(cmd_buff)); + } } void SubgroupsOperations::draw() { - VkPipelineStageFlags graphics_wait_stage_masks[] = {VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT}; - VkSemaphore graphics_wait_semaphores[] = {compute.semaphore, semaphores.acquired_image_ready}; - VkSemaphore graphics_signal_semaphores[] = {VK_NULL_HANDLE, semaphores.render_complete}; - - ApiVulkanSample::prepare_frame(); - submit_info.commandBufferCount = 1; - submit_info.pCommandBuffers = &draw_cmd_buffers[current_buffer]; -// submit_info.waitSemaphoreCount = 2; -// submit_info.pWaitSemaphores = graphics_wait_semaphores; -// submit_info.pWaitDstStageMask = graphics_wait_stage_masks; -// submit_info.signalSemaphoreCount = 2; -// submit_info.pSignalSemaphores = graphics_signal_semaphores; - VK_CHECK(vkQueueSubmit(queue, 1u, &submit_info, VK_NULL_HANDLE)); - ApiVulkanSample::submit_frame(); - - // Wait for rendering finished - VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; - - // Submit compute commands - VkSubmitInfo compute_submit_info = vkb::initializers::submit_info(); - compute_submit_info.commandBufferCount = 1u; - compute_submit_info.pCommandBuffers = &compute.command_buffer; -// compute_submit_info.waitSemaphoreCount = 1u; - -// compute_submit_info.pWaitDstStageMask = &wait_stage_mask; -// compute_submit_info.signalSemaphoreCount = 1u; -// compute_submit_info.pSignalSemaphores = &compute.semaphore; - - VK_CHECK(vkQueueSubmit(compute.queue, 1u, &compute_submit_info, VK_NULL_HANDLE)); + ApiVulkanSample::prepare_frame(); + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &draw_cmd_buffers[current_buffer]; + + VK_CHECK(vkQueueSubmit(queue, 1u, &submit_info, VK_NULL_HANDLE)); + ApiVulkanSample::submit_frame(); + + // Wait for rendering finished + VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; + + // Submit compute commands + VkSubmitInfo compute_submit_info = vkb::initializers::submit_info(); + compute_submit_info.commandBufferCount = 1u; + compute_submit_info.pCommandBuffers = &compute.command_buffer; + + VK_CHECK(vkQueueSubmit(compute.queue, 1u, &compute_submit_info, VK_NULL_HANDLE)); } void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) { - if (drawer.header("Settings")) - { - if (get_device().get_gpu().get_features().fillModeNonSolid) - { - if (drawer.checkbox("Wireframe", &ui.wireframe)) - { - build_command_buffers(); - } - } - } + if (drawer.header("Settings")) + { + if (get_device().get_gpu().get_features().fillModeNonSolid) + { + if (drawer.checkbox("Wireframe", &ui.wireframe)) + { + build_command_buffers(); + } + } + if (drawer.combo_box("Grid Size", &ui.grid_size_idx, ui.grid_sizes)) + { + // recreate grid mesh + generate_grid(); + update_uniform_buffers(); + build_compute_command_buffer(); + build_command_buffers(); + } + } } bool SubgroupsOperations::resize(const uint32_t width, const uint32_t height) { - if (!ApiVulkanSample::resize(width, height)) - return false; - build_compute_command_buffer(); - build_command_buffers(); - return true; + if (!ApiVulkanSample::resize(width, height)) + return false; + update_uniform_buffers(); + build_compute_command_buffer(); + build_command_buffers(); + return true; } void SubgroupsOperations::render(float delta_time) { - if (!prepared) - { - return; - } - draw(); - if (camera.updated) - { - update_uniform_buffers(); - } + if (!prepared) + { + return; + } + draw(); + if (camera.updated) + { + update_uniform_buffers(); + } } std::unique_ptr create_subgroups_operations() { - return std::make_unique(); + return std::make_unique(); } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index e3195c6b6..2e786d339 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -22,112 +22,116 @@ class SubgroupsOperations : public ApiVulkanSample { public: - SubgroupsOperations(); - ~SubgroupsOperations(); - - bool prepare(vkb::Platform &platform) override; - void request_gpu_features(vkb::PhysicalDevice &gpu) override; - void build_command_buffers() override; - void render(float delta_time) override; - bool resize(const uint32_t width, const uint32_t height) override; - void on_update_ui_overlay(vkb::Drawer &drawer) override; - - void draw(); - void load_assets(); - - void prepare_compute(); - void create_compute_queue(); - void create_compute_command_pool(); - void create_compute_command_buffer(); - void create_compute_descriptor_set_layout(); - void create_compute_descriptor_set(); - - void preapre_compute_pipeline_layout(); - void prepare_compute_pipeline(); - - void build_compute_command_buffer(); - - /////////////////////////////////////////////// - void generate_grid(); - void prepare_uniform_buffers(); - void setup_descriptor_pool(); - void create_descriptor_set_layout(); - void create_descriptor_set(); - void create_pipelines(); - - void update_uniform_buffers(); - - - - struct Pipeline - { - void destroy(VkDevice device); - - VkPipeline pipeline; - VkPipelineLayout pipeline_layout; - }; - - struct GridBuffers - { - std::unique_ptr vertex; - std::unique_ptr index; - uint32_t index_count = {0u}; - }; - - struct CameraUbo - { - alignas(16) glm::mat4 projection; - alignas(16) glm::mat4 view; - alignas(16) glm::mat4 model; - }; - - struct TimeUbo - { - alignas(4) float time; - }; - - struct GuiConfig - { - bool wireframe = {false}; - } ui; - - GridBuffers init_grid; // input buffer for compute shader - - uint32_t grid_size = {128u}; - - std::unique_ptr camera_ubo; - std::unique_ptr time_ubo; - - struct - { - VkQueue queue; - VkCommandPool command_pool; - VkCommandBuffer command_buffer; - VkSemaphore semaphore; - VkDescriptorSetLayout descriptor_set_layout; - VkDescriptorSet descriptor_set; - uint32_t queue_family_index; - - struct - { - Pipeline _default; - } pipelines; - } compute; - - struct - { - GridBuffers grid; // output (result) buffer for compute shader - - VkDescriptorSetLayout descriptor_set_layout; - VkDescriptorSet descriptor_set; - struct - { - Pipeline _default; - Pipeline wireframe; - } pipelines; - } ocean; - - VkPhysicalDeviceSubgroupProperties subgroups_properties; + SubgroupsOperations(); + ~SubgroupsOperations(); + + bool prepare(vkb::Platform &platform) override; + void request_gpu_features(vkb::PhysicalDevice &gpu) override; + void build_command_buffers() override; + void render(float delta_time) override; + bool resize(const uint32_t width, const uint32_t height) override; + void on_update_ui_overlay(vkb::Drawer &drawer) override; + + void draw(); + void load_assets(); + + void prepare_compute(); + void create_compute_queue(); + void create_compute_command_pool(); + void create_compute_command_buffer(); + void create_compute_descriptor_set_layout(); + void create_compute_descriptor_set(); + + void preapre_compute_pipeline_layout(); + void prepare_compute_pipeline(); + + void build_compute_command_buffer(); + + void generate_grid(); + void prepare_uniform_buffers(); + void setup_descriptor_pool(); + void create_descriptor_set_layout(); + void create_descriptor_set(); + void create_pipelines(); + + void update_uniform_buffers(); + + struct Pipeline + { + void destroy(VkDevice device); + + VkPipeline pipeline = {VK_NULL_HANDLE}; + VkPipelineLayout pipeline_layout = {VK_NULL_HANDLE}; + }; + + struct GridBuffers + { + std::unique_ptr vertex; + std::unique_ptr index; + uint32_t index_count = {0u}; + }; + + struct CameraUbo + { + alignas(16) glm::mat4 projection; + alignas(16) glm::mat4 view; + alignas(16) glm::mat4 model; + }; + + struct ComputeUbo + { + alignas(4) uint32_t grid_size; + alignas(4) float time; + }; + + struct GuiConfig + { + GuiConfig() + { + grid_sizes = { + "16", "32", "64", "128", "256"}; + } + bool wireframe = {false}; + int32_t grid_size_idx = {0}; + + std::vector grid_sizes; + } ui; + + GridBuffers input_grid; // input buffer for compute shader + uint32_t grid_size; + std::unique_ptr camera_ubo; + std::unique_ptr compute_ubo; + + struct + { + VkQueue queue; + VkCommandPool command_pool; + VkCommandBuffer command_buffer; + VkSemaphore semaphore; + VkDescriptorSetLayout descriptor_set_layout; + VkDescriptorSet descriptor_set; + uint32_t queue_family_index; + + struct + { + Pipeline _default; + } pipelines; + } compute; + + struct + { + GridBuffers grid; // output (result) buffer for compute shader + + VkDescriptorSetLayout descriptor_set_layout; + VkDescriptorSet descriptor_set; + struct + { + Pipeline _default; + Pipeline wireframe; + } pipelines; + } ocean; + + VkPhysicalDeviceSubgroupProperties subgroups_properties; }; std::unique_ptr create_subgroups_operations(); diff --git a/shaders/subgroups_operations/base.frag b/shaders/subgroups_operations/base.frag deleted file mode 100644 index 25a7084b4..000000000 --- a/shaders/subgroups_operations/base.frag +++ /dev/null @@ -1,28 +0,0 @@ -#version 450 -/* Copyright (c) 2023, Mobica Limited - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -layout (binding = 1) uniform sampler2D samplerColor; - -layout (location = 0) in vec2 inUV; - -layout (location = 0) out vec4 outFragColor; - -void main() -{ - outFragColor = texture(samplerColor, inUV); -} \ No newline at end of file diff --git a/shaders/subgroups_operations/base.vert b/shaders/subgroups_operations/base.vert deleted file mode 100644 index f992345da..000000000 --- a/shaders/subgroups_operations/base.vert +++ /dev/null @@ -1,35 +0,0 @@ -#version 450 -/* Copyright (c) 2023, Mobica Limited - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -layout (location = 0) in vec3 inPos; -layout (location = 1) in vec2 inUV; - -layout(location = 0) out vec2 outUv; - -layout (binding = 0) uniform Ubo -{ - mat4 projection; - mat4 model; - vec4 view; -} ubo; - -void main() -{ - outUv = inUV; - gl_Position = ubo.projection * ubo.model * vec4(inPos.xyz, 1.0f); -} diff --git a/shaders/subgroups_operations/blur.comp b/shaders/subgroups_operations/blur.comp deleted file mode 100644 index 1d322a112..000000000 --- a/shaders/subgroups_operations/blur.comp +++ /dev/null @@ -1,67 +0,0 @@ -#version 450 -/* Copyright (c) 2023, Mobica Limited - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -#extension GL_KHR_shader_subgroup_basic : enable - -layout (local_size_x = 32, local_size_y = 32, local_size_z = 1) in; - -layout(binding = 0, rgba8) uniform readonly image2D inputTexture; - -layout(binding = 1, rgba8) uniform writeonly image2D outputTexture; - -layout (binding = 2) uniform KernelUbo { - mat3 kernel; -} kernelUbo; - - -struct ImageMatrix -{ - float texel[9]; -} imageMatrix; - -void main() -{ - ivec2 textureCoord = ivec2(gl_GlobalInvocationID.xy); - vec4 resultTexelColor = vec4(0.0); - float denom = 1.0; - float offset = 0.5; - - int currentTexel = 0; - for (int x = -1; x < 2; ++x) - { - for (int y = -1; y < 2; ++y) - { - ivec2 orginalTexelCoord = ivec2(gl_GlobalInvocationID.x + x, gl_GlobalInvocationID.y + y); - vec3 texelRgb = imageLoad(inputTexture, orginalTexelCoord).rgb; - imageMatrix.texel[currentTexel] = (texelRgb.r + texelRgb.g + texelRgb.b) / 3.0; // calculate avg texel color - ++currentTexel; - } - } - - float avgColor = 0.0; - for (int i = 0; i < 9; ++i) - { - avgColor += float(kernelUbo.kernel[i] * imageMatrix.texel[i]); - } - - vec3 rgb = vec3(clamp(avgColor / denom + offset, 0.0, 1.0)); - - resultTexelColor = vec4(rgb, 1.0); - - imageStore(outputTexture, textureCoord, resultTexelColor); -} diff --git a/shaders/subgroups_operations/ocean_horizontal.comp b/shaders/subgroups_operations/ocean_fft.comp similarity index 79% rename from shaders/subgroups_operations/ocean_horizontal.comp rename to shaders/subgroups_operations/ocean_fft.comp index b30005f9f..3344e423d 100644 --- a/shaders/subgroups_operations/ocean_horizontal.comp +++ b/shaders/subgroups_operations/ocean_fft.comp @@ -24,10 +24,11 @@ struct Vertex { vec3 position; }; -layout (binding = 0) uniform TimeUbo +layout (binding = 0) uniform Ubo { + uint grid_size; float time; -} uboTime; +} ubo; layout (std140, binding = 1) readonly buffer VertexInBuffer { @@ -41,10 +42,10 @@ layout (std140, binding = 2) buffer VertexOutBuffer void main() { - uint idx = gl_GlobalInvocationID.x + gl_GlobalInvocationID.y; + uint idx = gl_LocalInvocationIndex; // gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * ubo.grid_size; Vertex inVertex = verticesIn[idx]; - inVertex.position.x = inVertex.position.x * sin(uboTime.time); - // subgroupBarrier(); + // inVertex.position.y = sin(inVertex.position.x * inVertex.position.y); // * sin(uboTime.time); + //subgroupBarrier(); verticesOut[idx].position = inVertex.position; } From 014fe37eca86536ef4f734d5e8b41f9a7d8b11de Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Fri, 23 Jun 2023 13:46:12 +0200 Subject: [PATCH 06/53] Animated grid using the sine and cosine functions --- .../subgroups_operations/CMakeLists.txt | 6 +- .../subgroups_operations.cpp | 238 +++++++++++------- .../subgroups_operations.h | 20 +- shaders/subgroups_operations/ocean_fft.comp | 30 ++- 4 files changed, 182 insertions(+), 112 deletions(-) diff --git a/samples/extensions/subgroups_operations/CMakeLists.txt b/samples/extensions/subgroups_operations/CMakeLists.txt index fe4d63f95..d7b0a128b 100644 --- a/samples/extensions/subgroups_operations/CMakeLists.txt +++ b/samples/extensions/subgroups_operations/CMakeLists.txt @@ -26,6 +26,6 @@ add_sample( NAME "subgroups_operations" DESCRIPTION "Demonstrates the use of a subgroups feature" SHADER_FILES_GLSL - "subgroups_operations/base.vert" - "subgroups_operations/base.frag" - "subgroups_operations/blur.comp") + "subgroups_operations/ocean.vert" + "subgroups_operations/ocean.frag" + "subgroups_operations/ocean_fft.comp") diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index d62d6b8c4..f0e504668 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -16,7 +16,7 @@ */ #include "subgroups_operations.h" -#include +#include void SubgroupsOperations::Pipeline::destroy(VkDevice device) { @@ -44,12 +44,17 @@ SubgroupsOperations::SubgroupsOperations() // Required by VK_KHR_spirv_1_4 add_device_extension(VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME); + // Targeting SPIR-V version + vkb::GLSLCompiler::set_target_environment(glslang::EShTargetSpv, glslang::EShTargetSpv_1_4); + title = "Subgroups operations"; camera.type = vkb::CameraType::LookAt; camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); - camera.set_position({0.0f, 0.0f, -2.0f}); + camera.set_position({-1.98f, -2.24f, -28.55f}); + camera.set_rotation({-24.0f, -7.0f, 0.0f}); } + SubgroupsOperations::~SubgroupsOperations() { if (device) @@ -62,6 +67,7 @@ SubgroupsOperations::~SubgroupsOperations() ocean.pipelines._default.destroy(get_device().get_handle()); ocean.pipelines.wireframe.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), ocean.descriptor_set_layout, nullptr); + vkDestroySemaphore(get_device().get_handle(), ocean.semaphore, nullptr); } } @@ -72,18 +78,28 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) return false; } + ocean.graphics_queue_family_index = get_device().get_queue_family_index(VK_QUEUE_GRAPHICS_BIT); + load_assets(); setup_descriptor_pool(); prepare_uniform_buffers(); prepare_compute(); // prepare grpahics pipeline + create_semaphore(); create_descriptor_set_layout(); create_descriptor_set(); create_pipelines(); build_command_buffers(); + // signal semaphore + VkSubmitInfo submit_info = vkb::initializers::submit_info(); + submit_info.signalSemaphoreCount = 1u; + submit_info.pSignalSemaphores = &ocean.semaphore; + VK_CHECK(vkQueueSubmit(queue, 1u, &submit_info, VK_NULL_HANDLE)); + get_device().wait_idle(); + prepared = true; return true; } @@ -121,10 +137,7 @@ void SubgroupsOperations::create_compute_command_buffer() { // Create a command buffer for compute operations VkCommandBufferAllocateInfo command_buffer_allocate_info = - vkb::initializers::command_buffer_allocate_info( - compute.command_pool, - VK_COMMAND_BUFFER_LEVEL_PRIMARY, - 1u); + vkb::initializers::command_buffer_allocate_info(compute.command_pool, VK_COMMAND_BUFFER_LEVEL_PRIMARY, 1u); VK_CHECK(vkAllocateCommandBuffers(get_device().get_handle(), &command_buffer_allocate_info, &compute.command_buffer)); @@ -150,6 +163,11 @@ void SubgroupsOperations::create_compute_descriptor_set() VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1u); VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); + update_compute_descriptor(); +} + +void SubgroupsOperations::update_compute_descriptor() +{ VkDescriptorBufferInfo time_ubo_buffer = create_descriptor(*compute_ubo); VkDescriptorBufferInfo input_vertices_buffer = create_descriptor(*input_grid.vertex); VkDescriptorBufferInfo output_vertices_buffer = create_descriptor(*ocean.grid.vertex); @@ -182,18 +200,42 @@ void SubgroupsOperations::prepare_compute_pipeline() void SubgroupsOperations::build_compute_command_buffer() { - VkSubmitInfo submit_info = {}; - submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; - vkResetCommandBuffer(compute.command_buffer, 0u); - // record command - VkCommandBufferBeginInfo begin_info = {}; - begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); + if (compute.queue_family_index != ocean.graphics_queue_family_index) + { + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.srcAccessMask = 0; + memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; + memory_barrier.buffer = ocean.grid.vertex->get_handle(); + memory_barrier.offset = 0; + memory_barrier.size = ocean.grid.vertex->get_size(); + + vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 1, &memory_barrier, 0, nullptr); + } + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, grid_size / 16u, grid_size / 16u, 1u); + + vkCmdDispatch(compute.command_buffer, grid_size, grid_size, 1u); + + if (compute.queue_family_index != ocean.graphics_queue_family_index) + { + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.srcAccessMask = 0; + memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; + memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.buffer = ocean.grid.vertex->get_handle(); + memory_barrier.offset = 0; + memory_barrier.size = ocean.grid.vertex->get_size(); + + vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, 0, nullptr, 1, &memory_barrier, 0, nullptr); + } VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); } @@ -228,30 +270,29 @@ void SubgroupsOperations::prepare_uniform_buffers() void SubgroupsOperations::generate_grid() { - grid_size = static_cast(stoi(ui.grid_sizes[ui.grid_size_idx])); std::vector grid_vertices; std::vector grid_indices; - for (uint32_t z = 0; z <= grid_size; ++z) + for (uint32_t z = 0u; z <= grid_size; ++z) { - for (uint32_t x = 0; x <= grid_size; ++x) + for (uint32_t x = 0u; x <= grid_size; ++x) { Vertex point = {}; - point.pos.x = static_cast(x) / static_cast(grid_size); - point.pos.z = static_cast(z) / static_cast(grid_size); + point.pos.x = static_cast((static_cast(x) / static_cast(grid_size - 1) * 2 - 1) * (grid_size / 2.0f)); + point.pos.z = static_cast((static_cast(z) / static_cast(grid_size - 1) * 2 - 1) * (grid_size / 2.0f)); point.pos.y = 0.0f; grid_vertices.push_back(point); } } - for (uint32_t z = 0; z < grid_size; ++z) + for (uint32_t z = 0u; z < grid_size; ++z) { - for (uint32_t x = 0; x < grid_size; ++x) + for (uint32_t x = 0u; x < grid_size; ++x) { - uint32_t i0 = z * (grid_size + 1) + x; - uint32_t i1 = i0 + 1; - uint32_t i2 = i0 + (grid_size + 1); - uint32_t i3 = i2 + 1; + uint32_t i0 = z * (grid_size + 1u) + x; + uint32_t i1 = i0 + 1u; + uint32_t i2 = i0 + (grid_size + 1u); + uint32_t i3 = i2 + 1u; grid_indices.push_back(i0); grid_indices.push_back(i2); @@ -263,23 +304,13 @@ void SubgroupsOperations::generate_grid() } } - ocean.grid.index_count = vkb::to_u32(grid_indices.size()); auto vertex_buffer_size = vkb::to_u32(grid_vertices.size() * sizeof(Vertex)); auto index_buffer_size = vkb::to_u32(grid_indices.size() * sizeof(uint32_t)); - - if (input_grid.vertex) - { - input_grid.vertex.reset(nullptr); - } - if (ocean.grid.vertex && ocean.grid.index) - { - ocean.grid.vertex.reset(nullptr); - ocean.grid.index.reset(nullptr); - } + ocean.grid.index_count = vkb::to_u32(grid_indices.size()); input_grid.vertex = std::make_unique(get_device(), vertex_buffer_size, - VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); input_grid.vertex->update(grid_vertices.data(), vertex_buffer_size); @@ -291,20 +322,24 @@ void SubgroupsOperations::generate_grid() ocean.grid.vertex = std::make_unique(get_device(), vertex_buffer_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, VMA_MEMORY_USAGE_GPU_ONLY); } +void SubgroupsOperations::create_semaphore() +{ + // Semaphore for graphics queue + VkSemaphoreCreateInfo semaphore_create_info = vkb::initializers::semaphore_create_info(); + VK_CHECK(vkCreateSemaphore(get_device().get_handle(), &semaphore_create_info, nullptr, &ocean.semaphore)); +} + void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u), vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u)}; VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info( - static_cast(pool_sizes.size()), - pool_sizes.data(), - 2u); + vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 2u); VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } @@ -376,31 +411,26 @@ void SubgroupsOperations::create_pipelines() shader_stages[0] = load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT); shader_stages[1] = load_shader("subgroups_operations/ocean.frag", VK_SHADER_STAGE_FRAGMENT_BIT); const std::vector vertex_input_bindings = { - vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX), - }; + vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX)}; const std::vector vertex_input_attributes = { - vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), - }; + vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos))}; VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); vertex_input_state.vertexAttributeDescriptionCount = static_cast(vertex_input_attributes.size()); vertex_input_state.pVertexAttributeDescriptions = vertex_input_attributes.data(); - VkGraphicsPipelineCreateInfo pipeline_create_info = - vkb::initializers::pipeline_create_info( - ocean.pipelines._default.pipeline_layout, - render_pass, - 0u); - pipeline_create_info.pVertexInputState = &vertex_input_state; - pipeline_create_info.pInputAssemblyState = &input_assembly_state; - pipeline_create_info.pRasterizationState = &rasterization_state; - pipeline_create_info.pColorBlendState = &color_blend_state; - pipeline_create_info.pMultisampleState = &multisample_state; - pipeline_create_info.pViewportState = &viewport_state; - pipeline_create_info.pDepthStencilState = &depth_stencil_state; - pipeline_create_info.pDynamicState = &dynamic_state; - pipeline_create_info.stageCount = static_cast(shader_stages.size()); - pipeline_create_info.pStages = shader_stages.data(); + + VkGraphicsPipelineCreateInfo pipeline_create_info = vkb::initializers::pipeline_create_info(ocean.pipelines._default.pipeline_layout, render_pass, 0u); + pipeline_create_info.pVertexInputState = &vertex_input_state; + pipeline_create_info.pInputAssemblyState = &input_assembly_state; + pipeline_create_info.pRasterizationState = &rasterization_state; + pipeline_create_info.pColorBlendState = &color_blend_state; + pipeline_create_info.pMultisampleState = &multisample_state; + pipeline_create_info.pViewportState = &viewport_state; + pipeline_create_info.pDepthStencilState = &depth_stencil_state; + pipeline_create_info.pDynamicState = &dynamic_state; + pipeline_create_info.stageCount = static_cast(shader_stages.size()); + pipeline_create_info.pStages = shader_stages.data(); VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines._default.pipeline)); if (get_device().get_gpu().get_features().fillModeNonSolid) @@ -422,7 +452,7 @@ void SubgroupsOperations::update_uniform_buffers() ComputeUbo comp_ubo; comp_ubo.grid_size = grid_size; - comp_ubo.time = 0.0f; + comp_ubo.time = float(timer.elapsed()); compute_ubo->convert_and_update(comp_ubo); } @@ -448,12 +478,25 @@ void SubgroupsOperations::build_command_buffers() VK_CHECK(vkBeginCommandBuffer(cmd_buff, &command_buffer_begin_info)); + if (compute.queue_family_index != ocean.graphics_queue_family_index) + { + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT; + memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; + memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.buffer = ocean.grid.vertex->get_handle(); + memory_barrier.offset = 0u; + memory_barrier.size = ocean.grid.vertex->get_size(); + + vkCmdPipelineBarrier(cmd_buff, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 0, 0, nullptr, 1, &memory_barrier, 0, nullptr); + } + vkCmdBeginRenderPass(cmd_buff, &render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); VkViewport viewport = vkb::initializers::viewport(static_cast(width), static_cast(height), 0.0f, 1.0f); vkCmdSetViewport(cmd_buff, 0u, 1u, &viewport); - VkRect2D scissor = vkb::initializers::rect2D(width, height, 0, 0); + VkRect2D scissor = vkb::initializers::rect2D(static_cast(width), static_cast(height), 0, 0); vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); // draw ocean @@ -461,7 +504,7 @@ void SubgroupsOperations::build_command_buffers() vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ui.wireframe ? ocean.pipelines.wireframe.pipeline : ocean.pipelines._default.pipeline); - VkDeviceSize offset[] = {0}; + VkDeviceSize offset[] = {0u}; vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, ocean.grid.vertex->get(), offset); vkCmdBindIndexBuffer(cmd_buff, ocean.grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); @@ -472,28 +515,52 @@ void SubgroupsOperations::build_command_buffers() vkCmdEndRenderPass(cmd_buff); + if (compute.queue_family_index != ocean.graphics_queue_family_index) + { + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT; + memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; + memory_barrier.buffer = ocean.grid.vertex->get_handle(); + memory_barrier.offset = 0u; + memory_barrier.size = ocean.grid.vertex->get_size(); + vkCmdPipelineBarrier(cmd_buff, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 1, &memory_barrier, 0, nullptr); + } + VK_CHECK(vkEndCommandBuffer(cmd_buff)); } } void SubgroupsOperations::draw() { - ApiVulkanSample::prepare_frame(); - submit_info.commandBufferCount = 1; - submit_info.pCommandBuffers = &draw_cmd_buffers[current_buffer]; - - VK_CHECK(vkQueueSubmit(queue, 1u, &submit_info, VK_NULL_HANDLE)); - ApiVulkanSample::submit_frame(); - - // Wait for rendering finished VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; - // Submit compute commands - VkSubmitInfo compute_submit_info = vkb::initializers::submit_info(); - compute_submit_info.commandBufferCount = 1u; - compute_submit_info.pCommandBuffers = &compute.command_buffer; + VkSubmitInfo compute_submit_info = vkb::initializers::submit_info(); + compute_submit_info.commandBufferCount = 1u; + compute_submit_info.pCommandBuffers = &compute.command_buffer; + compute_submit_info.waitSemaphoreCount = 1u; + compute_submit_info.pWaitSemaphores = &ocean.semaphore; + compute_submit_info.pWaitDstStageMask = &wait_stage_mask; + compute_submit_info.signalSemaphoreCount = 1u; + compute_submit_info.pSignalSemaphores = &compute.semaphore; VK_CHECK(vkQueueSubmit(compute.queue, 1u, &compute_submit_info, VK_NULL_HANDLE)); + + VkPipelineStageFlags graphics_wait_stage_masks[] = {VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT}; + VkSemaphore graphics_wait_semaphores[] = {compute.semaphore, semaphores.acquired_image_ready}; + VkSemaphore graphics_signal_semaphores[] = {ocean.semaphore, semaphores.render_complete}; + + ApiVulkanSample::prepare_frame(); + submit_info.commandBufferCount = 1u; + submit_info.pCommandBuffers = &draw_cmd_buffers[current_buffer]; + submit_info.waitSemaphoreCount = 2u; + submit_info.pWaitSemaphores = graphics_wait_semaphores; + submit_info.pWaitDstStageMask = graphics_wait_stage_masks; + submit_info.signalSemaphoreCount = 2u; + submit_info.pSignalSemaphores = graphics_signal_semaphores; + + VK_CHECK(vkQueueSubmit(queue, 1u, &submit_info, VK_NULL_HANDLE)); + ApiVulkanSample::submit_frame(); } void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) @@ -507,14 +574,6 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) build_command_buffers(); } } - if (drawer.combo_box("Grid Size", &ui.grid_size_idx, ui.grid_sizes)) - { - // recreate grid mesh - generate_grid(); - update_uniform_buffers(); - build_compute_command_buffer(); - build_command_buffers(); - } } } @@ -534,11 +593,16 @@ void SubgroupsOperations::render(float delta_time) { return; } + if (!timer.is_running()) + timer.start(); + + update_uniform_buffers(); draw(); - if (camera.updated) - { - update_uniform_buffers(); - } + + auto elapsed_time = static_cast(timer.elapsed()); + + if (elapsed_time >= 10.0f) + timer.lap(); } std::unique_ptr create_subgroups_operations() diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 2e786d339..36a07508c 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -50,11 +50,13 @@ class SubgroupsOperations : public ApiVulkanSample void generate_grid(); void prepare_uniform_buffers(); void setup_descriptor_pool(); + void create_semaphore(); void create_descriptor_set_layout(); void create_descriptor_set(); void create_pipelines(); void update_uniform_buffers(); + void update_compute_descriptor(); struct Pipeline { @@ -86,19 +88,11 @@ class SubgroupsOperations : public ApiVulkanSample struct GuiConfig { - GuiConfig() - { - grid_sizes = { - "16", "32", "64", "128", "256"}; - } - bool wireframe = {false}; - int32_t grid_size_idx = {0}; - - std::vector grid_sizes; + bool wireframe = {false}; } ui; GridBuffers input_grid; // input buffer for compute shader - uint32_t grid_size; + uint32_t grid_size = {32u}; std::unique_ptr camera_ubo; std::unique_ptr compute_ubo; @@ -120,10 +114,11 @@ class SubgroupsOperations : public ApiVulkanSample struct { - GridBuffers grid; // output (result) buffer for compute shader - + GridBuffers grid; // output (result) buffer for compute shader + uint32_t graphics_queue_family_index; VkDescriptorSetLayout descriptor_set_layout; VkDescriptorSet descriptor_set; + VkSemaphore semaphore; struct { Pipeline _default; @@ -132,6 +127,7 @@ class SubgroupsOperations : public ApiVulkanSample } ocean; VkPhysicalDeviceSubgroupProperties subgroups_properties; + vkb::Timer timer; }; std::unique_ptr create_subgroups_operations(); diff --git a/shaders/subgroups_operations/ocean_fft.comp b/shaders/subgroups_operations/ocean_fft.comp index 3344e423d..7c221c462 100644 --- a/shaders/subgroups_operations/ocean_fft.comp +++ b/shaders/subgroups_operations/ocean_fft.comp @@ -18,34 +18,44 @@ * limitations under the License. */ -layout (local_size_x = 16, local_size_y = 16, local_size_z = 1) in; +layout (local_size_x = 32, local_size_y = 32, local_size_z = 1) in; -struct Vertex { +#define PI 3.141592f + +struct Vertex +{ vec3 position; }; -layout (binding = 0) uniform Ubo +layout (set = 0, binding = 0) uniform Ubo { uint grid_size; float time; } ubo; -layout (std140, binding = 1) readonly buffer VertexInBuffer +layout (std140, set = 0, binding = 1) readonly buffer VertexInBuffer { Vertex verticesIn[]; }; -layout (std140, binding = 2) buffer VertexOutBuffer +layout (std140, set = 0, binding = 2) buffer VertexOutBuffer { Vertex verticesOut[]; }; void main() { - uint idx = gl_LocalInvocationIndex; // gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * ubo.grid_size; + float amplitude = 1.5f; + float wave_length = 25.0f; + float speed = 15.0f; + float k = 2.0f * PI / wave_length; + uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; + + subgroupMemoryBarrierBuffer(); + Vertex inVertex = verticesIn[idx]; - // inVertex.position.y = sin(inVertex.position.x * inVertex.position.y); // * sin(uboTime.time); - //subgroupBarrier(); - verticesOut[idx].position = inVertex.position; - + + inVertex.position.y = amplitude * sin(k * (inVertex.position.x - (speed * ubo.time))) * cos(k * (inVertex.position.z - (speed * ubo.time))); + + verticesOut[idx] = inVertex; } From ebf2e8c2987a8fc9320630f824c5d5fcd3c3f495 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 5 Jul 2023 20:02:11 +0200 Subject: [PATCH 07/53] Add a new compute pipeline --- .../subgroups_operations.cpp | 92 ++++++++++--- .../subgroups_operations.h | 46 +++++-- shaders/subgroups_operations/ocean.frag | 8 +- shaders/subgroups_operations/ocean.vert | 4 + shaders/subgroups_operations/ocean_fft.comp | 31 +++-- .../subgroups_operations/ocean_fft_input.comp | 127 ++++++++++++++++++ 6 files changed, 261 insertions(+), 47 deletions(-) create mode 100644 shaders/subgroups_operations/ocean_fft_input.comp diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index f0e504668..5189de214 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -60,6 +60,7 @@ SubgroupsOperations::~SubgroupsOperations() if (device) { compute.pipelines._default.destroy(get_device().get_handle()); + compute.pipelines.fft_input.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); @@ -150,8 +151,8 @@ void SubgroupsOperations::create_compute_descriptor_set_layout() { std::vector set_layout_bindngs = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), // ubo - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), // input vertex - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) // output vertex + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), // parameters fft data + // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) // input fft data }; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); @@ -168,13 +169,13 @@ void SubgroupsOperations::create_compute_descriptor_set() void SubgroupsOperations::update_compute_descriptor() { - VkDescriptorBufferInfo time_ubo_buffer = create_descriptor(*compute_ubo); - VkDescriptorBufferInfo input_vertices_buffer = create_descriptor(*input_grid.vertex); + VkDescriptorBufferInfo time_ubo_buffer = create_descriptor(*fft_params_ubo); + VkDescriptorBufferInfo input_vertices_buffer = create_descriptor(*fft_input_buffer); VkDescriptorBufferInfo output_vertices_buffer = create_descriptor(*ocean.grid.vertex); std::vector wirte_descriptor_sets = { vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &time_ubo_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &input_vertices_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &output_vertices_buffer)}; + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &output_vertices_buffer)}; + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &input_vertices_buffer), vkUpdateDescriptorSets(get_device().get_handle(), static_cast(wirte_descriptor_sets.size()), wirte_descriptor_sets.data(), 0u, nullptr); } @@ -186,6 +187,7 @@ void SubgroupsOperations::preapre_compute_pipeline_layout() compute_pipeline_layout_info.pSetLayouts = &compute.descriptor_set_layout; VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &compute.pipelines._default.pipeline_layout)); + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &compute.pipelines.fft_input.pipeline_layout)); } void SubgroupsOperations::prepare_compute_pipeline() @@ -196,6 +198,10 @@ void SubgroupsOperations::prepare_compute_pipeline() computeInfo.stage = load_shader("subgroups_operations/ocean_fft.comp", VK_SHADER_STAGE_COMPUTE_BIT); VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines._default.pipeline)); + + computeInfo.layout = compute.pipelines.fft_input.pipeline_layout; + computeInfo.stage = load_shader("subgroups_operations/ocean_fft_input.comp", VK_SHADER_STAGE_COMPUTE_BIT); + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines.fft_input.pipeline)); } void SubgroupsOperations::build_compute_command_buffer() @@ -217,12 +223,23 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 1, &memory_barrier, 0, nullptr); } + // fft input data + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines.fft_input.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines.fft_input.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, 32u, 32u, 1u); + } + // add memory barrier - vkCmdDispatch(compute.command_buffer, grid_size, grid_size, 1u); + // add memory barrier + { + // fft calcuation + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, 32u, 32u, 1u); + } if (compute.queue_family_index != ocean.graphics_queue_family_index) { VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); @@ -258,13 +275,25 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) void SubgroupsOperations::load_assets() { - generate_grid(); + // generate_grid(); + auto input_buffer_size = static_cast(grid_size * grid_size * (sizeof(float) * 2u)); // max grid size * max grid size * sizeof(std::complex) - real: float; imag: float + fft_input_buffer = std::make_unique(get_device(), + input_buffer_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + VMA_MEMORY_USAGE_GPU_ONLY); + auto vertex_buffer_size = grid_size * grid_size * sizeof(OceanVertex); + ocean.grid.vertex = std::make_unique(get_device(), + vertex_buffer_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VMA_MEMORY_USAGE_GPU_ONLY); } void SubgroupsOperations::prepare_uniform_buffers() { - camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - compute_ubo = std::make_unique(get_device(), sizeof(ComputeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + compute_ubo = std::make_unique(get_device(), sizeof(ComputeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + update_uniform_buffers(); } @@ -281,6 +310,7 @@ void SubgroupsOperations::generate_grid() point.pos.x = static_cast((static_cast(x) / static_cast(grid_size - 1) * 2 - 1) * (grid_size / 2.0f)); point.pos.z = static_cast((static_cast(z) / static_cast(grid_size - 1) * 2 - 1) * (grid_size / 2.0f)); point.pos.y = 0.0f; + point.normal = glm::vec3(0.0f, 1.0f, 0.0); grid_vertices.push_back(point); } } @@ -319,11 +349,6 @@ void SubgroupsOperations::generate_grid() VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); ocean.grid.index->update(grid_indices.data(), index_buffer_size); - - ocean.grid.vertex = std::make_unique(get_device(), - vertex_buffer_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, - VMA_MEMORY_USAGE_GPU_ONLY); } void SubgroupsOperations::create_semaphore() @@ -336,7 +361,7 @@ void SubgroupsOperations::create_semaphore() void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u), // FFTParametersUbo, CameraUbo, ComputeUbo vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u)}; VkDescriptorPoolCreateInfo descriptor_pool_create_info = vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 2u); @@ -413,7 +438,8 @@ void SubgroupsOperations::create_pipelines() const std::vector vertex_input_bindings = { vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX)}; const std::vector vertex_input_attributes = { - vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos))}; + vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), + vkb::initializers::vertex_input_attribute_description(0u, 1u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, normal))}; VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); @@ -500,7 +526,7 @@ void SubgroupsOperations::build_command_buffers() vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); // draw ocean - { + /*{ vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ui.wireframe ? ocean.pipelines.wireframe.pipeline : ocean.pipelines._default.pipeline); @@ -509,7 +535,7 @@ void SubgroupsOperations::build_command_buffers() vkCmdBindIndexBuffer(cmd_buff, ocean.grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); vkCmdDrawIndexed(cmd_buff, ocean.grid.index_count, 1u, 0u, 0u, 0u); - } + }*/ draw_ui(cmd_buff); @@ -575,6 +601,30 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) } } } + + if (drawer.header("Ocean settings")) + + { + if (drawer.input_float("Amplitude", &ui.amplitude, 0.001f, 3)) + { + // update input for fft + } + if (drawer.input_float("Length", &ui.length, 0.1f, 1)) + { + // update input for fft + } + if (drawer.header("Wind")) + { + if (drawer.input_float("X", &ui.wind.x, 0.5f, 2)) + { + // update input for fft + } + if (drawer.input_float("Y", &ui.wind.y, 0.5f, 2)) + { + // update input for fft + } + } + } } bool SubgroupsOperations::resize(const uint32_t width, const uint32_t height) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 36a07508c..e76bb0d03 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -21,6 +21,13 @@ class SubgroupsOperations : public ApiVulkanSample { + + struct OceanVertex + { + glm::vec3 position; + glm::vec3 normal; + }; + public: SubgroupsOperations(); ~SubgroupsOperations(); @@ -86,39 +93,54 @@ class SubgroupsOperations : public ApiVulkanSample alignas(4) float time; }; + struct FFTParametersUbo + { + alignas(4) float amplitude; + alignas(4) float length; + alignas(8) glm::vec2 wind; + alignas(4) uint32_t grid_size; + }; + struct GuiConfig { - bool wireframe = {false}; + bool wireframe = {false}; + float amplitude = {0.005f}; + float length = {16.0f}; + glm::vec2 wind = {16.0f, 0.0f}; } ui; GridBuffers input_grid; // input buffer for compute shader uint32_t grid_size = {32u}; std::unique_ptr camera_ubo; std::unique_ptr compute_ubo; + std::unique_ptr fft_params_ubo; + + std::unique_ptr fft_input_buffer; // TODO: change name struct { - VkQueue queue; - VkCommandPool command_pool; - VkCommandBuffer command_buffer; - VkSemaphore semaphore; - VkDescriptorSetLayout descriptor_set_layout; - VkDescriptorSet descriptor_set; - uint32_t queue_family_index; + VkQueue queue = {VK_NULL_HANDLE}; + VkCommandPool command_pool = {VK_NULL_HANDLE}; + VkCommandBuffer command_buffer = {VK_NULL_HANDLE}; + VkSemaphore semaphore = {VK_NULL_HANDLE}; + VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; + uint32_t queue_family_index = {-1u}; struct { Pipeline _default; + Pipeline fft_input; } pipelines; } compute; struct { GridBuffers grid; // output (result) buffer for compute shader - uint32_t graphics_queue_family_index; - VkDescriptorSetLayout descriptor_set_layout; - VkDescriptorSet descriptor_set; - VkSemaphore semaphore; + uint32_t graphics_queue_family_index = {-1u}; + VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; + VkSemaphore semaphore = {VK_NULL_HANDLE}; struct { Pipeline _default; diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index 94e2445a5..650ba3eca 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -16,10 +16,12 @@ * limitations under the License. */ + layout (location = 0) in vec3 inNormal; -layout (location = 0) out vec4 outFragColor; + layout (location = 0) out vec4 outFragColor; void main() { - outFragColor = vec4(0.0f, 0.5f, 1.0f, 1.0f); // should be blue -} \ No newline at end of file + vec3 ocean_color = vec3(0.0f, 0.5f, 1.0f) + inNormal; + outFragColor = vec4(ocean_color, 1.0f); +} diff --git a/shaders/subgroups_operations/ocean.vert b/shaders/subgroups_operations/ocean.vert index 909b01b76..86c6a87d5 100644 --- a/shaders/subgroups_operations/ocean.vert +++ b/shaders/subgroups_operations/ocean.vert @@ -17,6 +17,7 @@ */ layout (location = 0) in vec3 inPos; +layout (location = 1) in vec3 inNormal; layout (binding = 0) uniform Ubo { @@ -25,7 +26,10 @@ layout (binding = 0) uniform Ubo mat4 model; } ubo; +layout (location = 0) out vec3 outNormal; + void main() { gl_Position = ubo.projection * ubo.view * ubo.model * vec4(inPos.xyz, 1.0f); + outNormal = inNormal; } diff --git a/shaders/subgroups_operations/ocean_fft.comp b/shaders/subgroups_operations/ocean_fft.comp index 7c221c462..adc90ae5d 100644 --- a/shaders/subgroups_operations/ocean_fft.comp +++ b/shaders/subgroups_operations/ocean_fft.comp @@ -25,37 +25,46 @@ layout (local_size_x = 32, local_size_y = 32, local_size_z = 1) in; struct Vertex { vec3 position; + vec3 normal; }; +struct Complex +{ + float real; + float imag; +}; + +struct FFTInputData +{ + Complex tilde; + Complex tilde_conj; +}; + + layout (set = 0, binding = 0) uniform Ubo { uint grid_size; float time; } ubo; -layout (std140, set = 0, binding = 1) readonly buffer VertexInBuffer +layout (std140, set = 0, binding = 2) readonly buffer FFTInputDataBuffer { - Vertex verticesIn[]; + FFTInputData data[]; }; -layout (std140, set = 0, binding = 2) buffer VertexOutBuffer +layout (std140, set = 0, binding = 1) buffer VertexOutBuffer { Vertex verticesOut[]; }; void main() { - float amplitude = 1.5f; - float wave_length = 25.0f; - float speed = 15.0f; - float k = 2.0f * PI / wave_length; uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; subgroupMemoryBarrierBuffer(); - Vertex inVertex = verticesIn[idx]; + FFTInputData inData = data[idx]; + - inVertex.position.y = amplitude * sin(k * (inVertex.position.x - (speed * ubo.time))) * cos(k * (inVertex.position.z - (speed * ubo.time))); - - verticesOut[idx] = inVertex; + verticesOut[idx].normal = vec3(0.0f); } diff --git a/shaders/subgroups_operations/ocean_fft_input.comp b/shaders/subgroups_operations/ocean_fft_input.comp new file mode 100644 index 000000000..de2931363 --- /dev/null +++ b/shaders/subgroups_operations/ocean_fft_input.comp @@ -0,0 +1,127 @@ +#version 450 +#extension GL_KHR_shader_subgroup_basic : enable + +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +layout (local_size_x = 32, local_size_y = 32, local_size_z = 1) in; + +#define PI 3.141592f +#define GRAVITY 9.81f + +struct Complex +{ + float real; + float imag; +}; + +struct FFTInputData +{ + Complex tilde; + Complex tilde_conj; +}; + +layout (set = 0, binding = 0) uniform FFTParametersUbo +{ + float amplitude; + float len; + vec2 wind; + uint grid_size; +} fftUbo; + +layout (std140, set = 0, binding = 2) buffer FFTInputDataBuffer +{ + FFTInputData data[]; +}; + +Complex complexMultiplication(Complex a, float scalar) +{ + Complex res; + res.real = scalar * a.real; + res.imag = scalar * a.imag; + return res; +} + +Complex conjugate(Complex a) +{ + Complex res; + res.real = a.real; + res.imag = -a.imag; + return res; +} + +float phillips_spectrum(uint n, uint m) +{ + vec2 k = vec2(PI * (2.0f * n - fftUbo.grid_size) / fftUbo.len, PI * (2.0f * m - fftUbo.grid_size) / fftUbo.len); + + float k_length = length(k); + if (k_length < 0.000001f) + return 0.0f; + + float k_len2 = k_length * k_length; + float k_len4 = k_len2 * k_len2; + + float k_dot_wind = dot(normalize(k), normalize(fftUbo.wind)); + float k_dot_wind2 = k_dot_wind * k_dot_wind; + + float wind_len = length(fftUbo.wind); + float L = wind_len * wind_len / GRAVITY; + float L2 = L * L; + + float damping = 0.001f; + float c2 = L2 * damping * damping; + + return fftUbo.amplitude * exp(-1.0f / (k_len2 * L2)) / k_len4 * k_dot_wind2 * exp(-k_len2 * c2); +} + +// source: https://stackoverflow.com/a/74419913 +float random(vec2 st) +{ + return fract(sin(dot(st.xy, vec2(12.9898f, 78.233f))) * 43758.5453123f); +} + +Complex gaussionRndVal(vec2 seed) +{ + float x1, x2, w; + do + { + x1 = 2.0f * random(seed) - 1.0f; + x2 = 2.0f * random(seed) - 1.0f; + w = x1 * x1 + x2 * x2; + } while (w >= 1.0f); + w = sqrt((-2.0f * log(w)) / w); + Complex res; + res.real = x1 * w; + res.imag = x2 * w; + return res; +} + +Complex hTilde(uint n, uint m) +{ + Complex rnd = gaussionRndVal(vec2(n, m)); + return complexMultiplication(rnd, sqrt(phillips_spectrum(n, m) / 2.0f)); +} + +void main() +{ + uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; + subgroupMemoryBarrierBuffer(); + uint n = gl_GlobalInvocationID.x * gl_SubgroupInvocationID; + uint m = gl_GlobalInvocationID.y * gl_SubgroupInvocationID; + data[idx].tilde = hTilde(n, m); // possible issue + data[idx].tilde_conj = conjugate(hTilde(-n, -m)); +} From 741c1c53e795709e5c8b0049bc4474f9f24dad03 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Thu, 6 Jul 2023 15:48:23 +0200 Subject: [PATCH 08/53] Fixed a descriptor set, the function generating random numbers has been fixed --- .../subgroups_operations.cpp | 121 ++++++++++-------- .../subgroups_operations.h | 9 -- shaders/subgroups_operations/ocean_fft.comp | 73 +++++++++-- .../subgroups_operations/ocean_fft_input.comp | 32 +++-- 4 files changed, 150 insertions(+), 85 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 5189de214..c4442b52d 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -44,6 +44,9 @@ SubgroupsOperations::SubgroupsOperations() // Required by VK_KHR_spirv_1_4 add_device_extension(VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME); + // for #extension GL_EXT_debug_printf : enable + add_device_extension(VK_KHR_SHADER_NON_SEMANTIC_INFO_EXTENSION_NAME); + // Targeting SPIR-V version vkb::GLSLCompiler::set_target_environment(glslang::EShTargetSpv, glslang::EShTargetSpv_1_4); @@ -150,9 +153,9 @@ void SubgroupsOperations::create_compute_command_buffer() void SubgroupsOperations::create_compute_descriptor_set_layout() { std::vector set_layout_bindngs = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), // ubo - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), // parameters fft data - // vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) // input fft data + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), // FFTParametersUbo + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), // Vertex data data + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) // FFTInputData fft data }; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); @@ -169,13 +172,15 @@ void SubgroupsOperations::create_compute_descriptor_set() void SubgroupsOperations::update_compute_descriptor() { - VkDescriptorBufferInfo time_ubo_buffer = create_descriptor(*fft_params_ubo); - VkDescriptorBufferInfo input_vertices_buffer = create_descriptor(*fft_input_buffer); - VkDescriptorBufferInfo output_vertices_buffer = create_descriptor(*ocean.grid.vertex); + VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); + VkDescriptorBufferInfo input_fft_data_buffer = create_descriptor(*fft_input_buffer); + VkDescriptorBufferInfo output_vertices_buffer = create_descriptor(*ocean.grid.vertex); + std::vector wirte_descriptor_sets = { - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &time_ubo_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &output_vertices_buffer)}; - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &input_vertices_buffer), + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &fft_params_ubo_buffer), + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &input_fft_data_buffer), + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &output_vertices_buffer) + }; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(wirte_descriptor_sets.size()), wirte_descriptor_sets.data(), 0u, nullptr); } @@ -210,19 +215,19 @@ void SubgroupsOperations::build_compute_command_buffer() VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); - if (compute.queue_family_index != ocean.graphics_queue_family_index) - { - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.srcAccessMask = 0; - memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; - memory_barrier.buffer = ocean.grid.vertex->get_handle(); - memory_barrier.offset = 0; - memory_barrier.size = ocean.grid.vertex->get_size(); - - vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 1, &memory_barrier, 0, nullptr); - } + if (compute.queue_family_index != ocean.graphics_queue_family_index) + { + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.srcAccessMask = 0u; + memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; + memory_barrier.buffer = fft_input_buffer->get_handle(); + memory_barrier.offset = 0u; + memory_barrier.size = fft_input_buffer->get_size(); + + vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); + } // fft input data { vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines.fft_input.pipeline); @@ -230,29 +235,43 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdDispatch(compute.command_buffer, 32u, 32u, 1u); } + + if (compute.queue_family_index != ocean.graphics_queue_family_index) + { + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + memory_barrier.dstAccessMask = 0u; + memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; + memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.buffer = fft_input_buffer->get_handle(); + memory_barrier.offset = 0u; + memory_barrier.size = fft_input_buffer->get_size(); + + vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); + } // add memory barrier // add memory barrier { // fft calcuation - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); + // vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); + // vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, 32u, 32u, 1u); - } - if (compute.queue_family_index != ocean.graphics_queue_family_index) - { - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.srcAccessMask = 0; - memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; - memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.buffer = ocean.grid.vertex->get_handle(); - memory_barrier.offset = 0; - memory_barrier.size = ocean.grid.vertex->get_size(); - - vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, 0, nullptr, 1, &memory_barrier, 0, nullptr); + // vkCmdDispatch(compute.command_buffer, 32u, 32u, 1u); } +// if (compute.queue_family_index != ocean.graphics_queue_family_index) +// { +// VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); +// memory_barrier.srcAccessMask = 0u; +// memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; +// memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; +// memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; +// memory_barrier.buffer = ocean.grid.vertex->get_handle(); +// memory_barrier.offset = 0u; +// memory_barrier.size = ocean.grid.vertex->get_size(); + +// vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); +// } VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); } @@ -276,10 +295,10 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) void SubgroupsOperations::load_assets() { // generate_grid(); - auto input_buffer_size = static_cast(grid_size * grid_size * (sizeof(float) * 2u)); // max grid size * max grid size * sizeof(std::complex) - real: float; imag: float + auto input_buffer_size = static_cast(grid_size * grid_size * (sizeof(float) * 4u)); // max grid size * max grid size * sizeof(std::complex) - real: float; imag: float fft_input_buffer = std::make_unique(get_device(), input_buffer_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VMA_MEMORY_USAGE_GPU_ONLY); auto vertex_buffer_size = grid_size * grid_size * sizeof(OceanVertex); ocean.grid.vertex = std::make_unique(get_device(), @@ -291,7 +310,6 @@ void SubgroupsOperations::load_assets() void SubgroupsOperations::prepare_uniform_buffers() { camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - compute_ubo = std::make_unique(get_device(), sizeof(ComputeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); update_uniform_buffers(); @@ -476,10 +494,12 @@ void SubgroupsOperations::update_uniform_buffers() camera_ubo->convert_and_update(ubo); - ComputeUbo comp_ubo; - comp_ubo.grid_size = grid_size; - comp_ubo.time = float(timer.elapsed()); - compute_ubo->convert_and_update(comp_ubo); + FFTParametersUbo fft_ubo; + fft_ubo.amplitude = ui.amplitude; + fft_ubo.grid_size = grid_size; + fft_ubo.length = ui.length; + fft_ubo.wind = ui.wind; + fft_params_ubo->convert_and_update(fft_ubo); } void SubgroupsOperations::build_command_buffers() @@ -526,7 +546,7 @@ void SubgroupsOperations::build_command_buffers() vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); // draw ocean - /*{ + /*{ vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ui.wireframe ? ocean.pipelines.wireframe.pipeline : ocean.pipelines._default.pipeline); @@ -535,7 +555,7 @@ void SubgroupsOperations::build_command_buffers() vkCmdBindIndexBuffer(cmd_buff, ocean.grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); vkCmdDrawIndexed(cmd_buff, ocean.grid.index_count, 1u, 0u, 0u, 0u); - }*/ + }*/ draw_ui(cmd_buff); @@ -643,16 +663,9 @@ void SubgroupsOperations::render(float delta_time) { return; } - if (!timer.is_running()) - timer.start(); update_uniform_buffers(); draw(); - - auto elapsed_time = static_cast(timer.elapsed()); - - if (elapsed_time >= 10.0f) - timer.lap(); } std::unique_ptr create_subgroups_operations() diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index e76bb0d03..975f9917b 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -87,12 +87,6 @@ class SubgroupsOperations : public ApiVulkanSample alignas(16) glm::mat4 model; }; - struct ComputeUbo - { - alignas(4) uint32_t grid_size; - alignas(4) float time; - }; - struct FFTParametersUbo { alignas(4) float amplitude; @@ -112,9 +106,7 @@ class SubgroupsOperations : public ApiVulkanSample GridBuffers input_grid; // input buffer for compute shader uint32_t grid_size = {32u}; std::unique_ptr camera_ubo; - std::unique_ptr compute_ubo; std::unique_ptr fft_params_ubo; - std::unique_ptr fft_input_buffer; // TODO: change name struct @@ -149,7 +141,6 @@ class SubgroupsOperations : public ApiVulkanSample } ocean; VkPhysicalDeviceSubgroupProperties subgroups_properties; - vkb::Timer timer; }; std::unique_ptr create_subgroups_operations(); diff --git a/shaders/subgroups_operations/ocean_fft.comp b/shaders/subgroups_operations/ocean_fft.comp index adc90ae5d..f602a5410 100644 --- a/shaders/subgroups_operations/ocean_fft.comp +++ b/shaders/subgroups_operations/ocean_fft.comp @@ -1,4 +1,4 @@ -#version 450 + #version 450 #extension GL_KHR_shader_subgroup_basic : enable /* Copyright (c) 2023, Mobica Limited @@ -21,6 +21,8 @@ layout (local_size_x = 32, local_size_y = 32, local_size_z = 1) in; #define PI 3.141592f +#define PI2 6.283184 +#define GRAVITY 9.81f struct Vertex { @@ -40,23 +42,78 @@ struct FFTInputData Complex tilde_conj; }; +Complex complexMultiplication(Complex a, float scalar) +{ + Complex res; + res.real = scalar * a.real; + res.imag = scalar * a.imag; + return res; +} + +Complex conjugate(Complex a) +{ + Complex res; + res.real = a.real; + res.imag = -a.imag; + return res; +} -layout (set = 0, binding = 0) uniform Ubo +layout (set = 0, binding = 0) uniform FFTParametersUbo { - uint grid_size; - float time; -} ubo; + float amplitude; + float len; + vec2 wind; + uint grid_size; +} fftUbo; -layout (std140, set = 0, binding = 2) readonly buffer FFTInputDataBuffer +layout (std140, set = 0, binding = 1) readonly buffer FFTInputDataBuffer { FFTInputData data[]; }; -layout (std140, set = 0, binding = 1) buffer VertexOutBuffer +layout (std140, set = 0, binding = 2) buffer VertexOutBuffer { Vertex verticesOut[]; }; + +// Add another FFTInputData buffer to store the results of the hTilde calculation +// Add 3'th FFTInputData buffer to store fft calculation results - final values + +float dispersion(uint n, uint m) +{ + float w0 = 2.0f * PI / 50.0f; + float x = PI * (2.0f * n - fftUbo.grid_size) / fftUbo.len; + float z = PI * (2.0f * m - fftUbo.grid_size) / fftUbo.len; + return floor(sqrt(GRAVITY * sqrt(x * x + z * z)) / w0) *w0; +} + + +Complex hTilde(float t, uint n, uint m) +{ + uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; + Complex tilde0 = data[idx].tilde + Complex tilde0_conj = data[idx].tilde_conj; + + float omega = dispersion(n, m); + + float _sin = sin(omega); + float _cos = cos(omega); + + Complex c0; + c0.real = _cos; + c0.imag = _sin; + + Complex c1 = conjugate(c0); + + return (tilde0 * c0) + (tilde0_conj * c1); // TODO add a multiply and add functions for Complex struct +} + +void fft() +{ + +} + void main() { uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; @@ -65,6 +122,6 @@ void main() FFTInputData inData = data[idx]; - + verticesOut[idx].position = vec3(0.0f); verticesOut[idx].normal = vec3(0.0f); } diff --git a/shaders/subgroups_operations/ocean_fft_input.comp b/shaders/subgroups_operations/ocean_fft_input.comp index de2931363..03cd72f78 100644 --- a/shaders/subgroups_operations/ocean_fft_input.comp +++ b/shaders/subgroups_operations/ocean_fft_input.comp @@ -21,6 +21,7 @@ layout (local_size_x = 32, local_size_y = 32, local_size_z = 1) in; #define PI 3.141592f +#define PI2 6.283184 #define GRAVITY 9.81f struct Complex @@ -43,7 +44,7 @@ layout (set = 0, binding = 0) uniform FFTParametersUbo uint grid_size; } fftUbo; -layout (std140, set = 0, binding = 2) buffer FFTInputDataBuffer +layout (std140, set = 0, binding = 1) buffer FFTInputDataBuffer { FFTInputData data[]; }; @@ -91,19 +92,20 @@ float phillips_spectrum(uint n, uint m) // source: https://stackoverflow.com/a/74419913 float random(vec2 st) { - return fract(sin(dot(st.xy, vec2(12.9898f, 78.233f))) * 43758.5453123f); + float rndVal = fract(sin(dot(st, vec2(12.9898f, 78.233f))) * 43758.5453123f); + if (rndVal < 0.000001f) return 0.000001f; + return rndVal; } Complex gaussionRndVal(vec2 seed) { - float x1, x2, w; - do - { - x1 = 2.0f * random(seed) - 1.0f; - x2 = 2.0f * random(seed) - 1.0f; - w = x1 * x1 + x2 * x2; - } while (w >= 1.0f); - w = sqrt((-2.0f * log(w)) / w); + float x1, x2, w, w2; + x1 = random(seed); + x1 = -2.0f * log(x1); + x2 = random(seed) * PI2; + + w = sqrt(x1) * cos(x2); + Complex res; res.real = x1 * w; res.imag = x2 * w; @@ -112,7 +114,7 @@ Complex gaussionRndVal(vec2 seed) Complex hTilde(uint n, uint m) { - Complex rnd = gaussionRndVal(vec2(n, m)); + Complex rnd = gaussionRndVal(vec2(abs(n), abs(m))); return complexMultiplication(rnd, sqrt(phillips_spectrum(n, m) / 2.0f)); } @@ -120,8 +122,10 @@ void main() { uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; subgroupMemoryBarrierBuffer(); - uint n = gl_GlobalInvocationID.x * gl_SubgroupInvocationID; - uint m = gl_GlobalInvocationID.y * gl_SubgroupInvocationID; + uint n = gl_GlobalInvocationID.x; + uint m = gl_GlobalInvocationID.y; + + data[idx].tilde = hTilde(n, m); // possible issue - data[idx].tilde_conj = conjugate(hTilde(-n, -m)); + data[idx].tilde_conj = conjugate(data[idx].tilde); } From de6d05481dfcd8600c130f56c5c78a1df6b76624 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Fri, 18 Aug 2023 11:48:39 +0200 Subject: [PATCH 09/53] Transfer of philips spectrum calculation from GPU to CPU. --- .../subgroups_operations.cpp | 223 ++++++++++-------- .../subgroups_operations.h | 20 +- shaders/subgroups_operations/ocean_fft.comp | 38 +-- .../subgroups_operations/ocean_fft_input.comp | 131 ---------- 4 files changed, 172 insertions(+), 240 deletions(-) delete mode 100644 shaders/subgroups_operations/ocean_fft_input.comp diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index c4442b52d..778e17b20 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -18,12 +18,17 @@ #include "subgroups_operations.h" #include +#include + +#define GRAVITY 9.81f + void SubgroupsOperations::Pipeline::destroy(VkDevice device) { if (pipeline != VK_NULL_HANDLE) { vkDestroyPipeline(device, pipeline, nullptr); } + if (pipeline_layout != VK_NULL_HANDLE) { vkDestroyPipelineLayout(device, pipeline_layout, nullptr); @@ -44,8 +49,8 @@ SubgroupsOperations::SubgroupsOperations() // Required by VK_KHR_spirv_1_4 add_device_extension(VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME); - // for #extension GL_EXT_debug_printf : enable - add_device_extension(VK_KHR_SHADER_NON_SEMANTIC_INFO_EXTENSION_NAME); + // TODO: remove when not needed; for #extension GL_EXT_debug_printf : enable + add_device_extension(VK_KHR_SHADER_NON_SEMANTIC_INFO_EXTENSION_NAME); // Targeting SPIR-V version vkb::GLSLCompiler::set_target_environment(glslang::EShTargetSpv, glslang::EShTargetSpv_1_4); @@ -54,7 +59,7 @@ SubgroupsOperations::SubgroupsOperations() camera.type = vkb::CameraType::LookAt; camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); - camera.set_position({-1.98f, -2.24f, -28.55f}); + camera.set_position({-2.0f, -2.25f, -29.0f}); camera.set_rotation({-24.0f, -7.0f, 0.0f}); } @@ -63,7 +68,6 @@ SubgroupsOperations::~SubgroupsOperations() if (device) { compute.pipelines._default.destroy(get_device().get_handle()); - compute.pipelines.fft_input.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); @@ -153,9 +157,9 @@ void SubgroupsOperations::create_compute_command_buffer() void SubgroupsOperations::create_compute_descriptor_set_layout() { std::vector set_layout_bindngs = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), // FFTParametersUbo - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), // Vertex data data - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) // FFTInputData fft data + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), // FFTParametersUbo + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), // Vertex data + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) // FFTInputData fft data }; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); @@ -172,15 +176,14 @@ void SubgroupsOperations::create_compute_descriptor_set() void SubgroupsOperations::update_compute_descriptor() { - VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); - VkDescriptorBufferInfo input_fft_data_buffer = create_descriptor(*fft_input_buffer); - VkDescriptorBufferInfo output_vertices_buffer = create_descriptor(*ocean.grid.vertex); - - std::vector wirte_descriptor_sets = { - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &fft_params_ubo_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &input_fft_data_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &output_vertices_buffer) - }; + VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); + VkDescriptorBufferInfo input_fft_data_buffer = create_descriptor(*fft_input_buffer); + VkDescriptorBufferInfo output_vertices_buffer = create_descriptor(*ocean.grid.vertex); + + std::vector wirte_descriptor_sets = { + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &fft_params_ubo_buffer), + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &input_fft_data_buffer), + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &output_vertices_buffer)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(wirte_descriptor_sets.size()), wirte_descriptor_sets.data(), 0u, nullptr); } @@ -192,7 +195,6 @@ void SubgroupsOperations::preapre_compute_pipeline_layout() compute_pipeline_layout_info.pSetLayouts = &compute.descriptor_set_layout; VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &compute.pipelines._default.pipeline_layout)); - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &compute.pipelines.fft_input.pipeline_layout)); } void SubgroupsOperations::prepare_compute_pipeline() @@ -203,10 +205,6 @@ void SubgroupsOperations::prepare_compute_pipeline() computeInfo.stage = load_shader("subgroups_operations/ocean_fft.comp", VK_SHADER_STAGE_COMPUTE_BIT); VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines._default.pipeline)); - - computeInfo.layout = compute.pipelines.fft_input.pipeline_layout; - computeInfo.stage = load_shader("subgroups_operations/ocean_fft_input.comp", VK_SHADER_STAGE_COMPUTE_BIT); - VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines.fft_input.pipeline)); } void SubgroupsOperations::build_compute_command_buffer() @@ -215,63 +213,40 @@ void SubgroupsOperations::build_compute_command_buffer() VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); - if (compute.queue_family_index != ocean.graphics_queue_family_index) - { - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.srcAccessMask = 0u; - memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; - memory_barrier.buffer = fft_input_buffer->get_handle(); - memory_barrier.offset = 0u; - memory_barrier.size = fft_input_buffer->get_size(); - - vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); - } - // fft input data + if (compute.queue_family_index != ocean.graphics_queue_family_index) + { + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.srcAccessMask = 0u; + memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; + memory_barrier.buffer = fft_input_buffer->get_handle(); + memory_barrier.offset = 0u; + memory_barrier.size = fft_input_buffer->get_size(); + + vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); + } + // fft calculation { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines.fft_input.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines.fft_input.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); vkCmdDispatch(compute.command_buffer, 32u, 32u, 1u); } - if (compute.queue_family_index != ocean.graphics_queue_family_index) - { - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - memory_barrier.dstAccessMask = 0u; - memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; - memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.buffer = fft_input_buffer->get_handle(); - memory_barrier.offset = 0u; - memory_barrier.size = fft_input_buffer->get_size(); - - vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); - } - // add memory barrier - - // add memory barrier + if (compute.queue_family_index != ocean.graphics_queue_family_index) { - // fft calcuation - // vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); - // vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); - - // vkCmdDispatch(compute.command_buffer, 32u, 32u, 1u); + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + memory_barrier.dstAccessMask = 0u; + memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; + memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.buffer = fft_input_buffer->get_handle(); + memory_barrier.offset = 0u; + memory_barrier.size = fft_input_buffer->get_size(); + + vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); } -// if (compute.queue_family_index != ocean.graphics_queue_family_index) -// { -// VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); -// memory_barrier.srcAccessMask = 0u; -// memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; -// memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; -// memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; -// memory_barrier.buffer = ocean.grid.vertex->get_handle(); -// memory_barrier.offset = 0u; -// memory_barrier.size = ocean.grid.vertex->get_size(); - -// vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); -// } VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); } @@ -295,7 +270,7 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) void SubgroupsOperations::load_assets() { // generate_grid(); - auto input_buffer_size = static_cast(grid_size * grid_size * (sizeof(float) * 4u)); // max grid size * max grid size * sizeof(std::complex) - real: float; imag: float + auto input_buffer_size = static_cast(grid_size * grid_size * (sizeof(float) * 4u)); // max grid size * max grid size * sizeof(std::complex) - real: float; imag: float fft_input_buffer = std::make_unique(get_device(), input_buffer_size, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, @@ -305,12 +280,76 @@ void SubgroupsOperations::load_assets() vertex_buffer_size, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, VMA_MEMORY_USAGE_GPU_ONLY); + size_t log_2_n = log(grid_size) / log(2); + auto helpBuffer1Size = VkDeviceSize(log_2_n * (sizeof(float) * 4) * grid_size); + helpBuffers.buffer1 = std::make_unique(get_device(), helpBuffer1Size, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + VMA_MEMORY_USAGE_GPU_ONLY); + + h_tilde_0.clear(); + h_hilde_0_conj.clear(); + for (uint32_t m = 0; m < grid_size + 1U; ++m) + { + for (uint32_t n = 0; n < grid_size + 1U; ++n) + { + h_tilde_0.push_back(hTilde_0(n, m)); + h_hilde_0_conj.push_back(std::conj(hTilde_0(-n, -m))); + } + } +} + +float SubgroupsOperations::phillips_spectrum(int32_t n, int32_t m) +{ + glm::vec2 k(glm::pi() * (2.0f * n - grid_size) / ui.length, glm::pi() * (2.0f * m - grid_size) / ui.length); + + float k_len = glm::length(k); + if (k_len < 0.000001f) + return 0.0f; + + float k_len2 = k_len * k_len; + float k_len4 = k_len2 * k_len2; + + float k_dot_w = glm::dot(glm::normalize(k), glm::normalize(ui.wind)); + float k_dot_w2 = k_dot_w * k_dot_w; + + float w_len = glm::length(ui.wind); + float L = w_len * w_len / GRAVITY; + float L2 = L * L; + + float damping = 0.001f; + float l2 = L2 * damping * damping; + + return ui.amplitude * glm::exp(-1.0f / (k_len2 * L2)) / k_len4 * k_dot_w2 * glm::exp(-k_len2 * l2); +} + +std::complex SubgroupsOperations::rndGaussian() +{ + float x1, x2, w; + auto rndVal = []() -> float { + std::random_device rndDevice; + std::mt19937 mt(rndDevice()); + std::uniform_real_distribution dis(0.0f, 1.0f); + return dis(mt); + }; + do + { + x1 = 2.0f * rndVal() - 1.0f; + x2 = 2.0f * rndVal() - 1.0f; + w = x1 * x1 + x2 * x2; + } while (w >= 1.0f); + w = std::sqrt((-2.0f * std::log(w)) / w); + return {x1 * w, x2 * w}; +} + +std::complex SubgroupsOperations::hTilde_0(uint32_t n, uint32_t m) +{ + std::complex rnd = rndGaussian(); + return rnd * std::sqrt(phillips_spectrum(n, m) / 2.0f); } void SubgroupsOperations::prepare_uniform_buffers() { - camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); update_uniform_buffers(); } @@ -494,12 +533,12 @@ void SubgroupsOperations::update_uniform_buffers() camera_ubo->convert_and_update(ubo); - FFTParametersUbo fft_ubo; - fft_ubo.amplitude = ui.amplitude; - fft_ubo.grid_size = grid_size; - fft_ubo.length = ui.length; - fft_ubo.wind = ui.wind; - fft_params_ubo->convert_and_update(fft_ubo); + FFTParametersUbo fft_ubo; + fft_ubo.amplitude = ui.amplitude; + fft_ubo.grid_size = grid_size; + fft_ubo.length = ui.length; + fft_ubo.wind = ui.wind; + fft_params_ubo->convert_and_update(fft_ubo); } void SubgroupsOperations::build_command_buffers() @@ -546,16 +585,16 @@ void SubgroupsOperations::build_command_buffers() vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); // draw ocean - /*{ - vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); - vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ui.wireframe ? ocean.pipelines.wireframe.pipeline : ocean.pipelines._default.pipeline); + /*{ + vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); + vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ui.wireframe ? ocean.pipelines.wireframe.pipeline : ocean.pipelines._default.pipeline); - VkDeviceSize offset[] = {0u}; - vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, ocean.grid.vertex->get(), offset); - vkCmdBindIndexBuffer(cmd_buff, ocean.grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); + VkDeviceSize offset[] = {0u}; + vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, ocean.grid.vertex->get(), offset); + vkCmdBindIndexBuffer(cmd_buff, ocean.grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); - vkCmdDrawIndexed(cmd_buff, ocean.grid.index_count, 1u, 0u, 0u, 0u); - }*/ + vkCmdDrawIndexed(cmd_buff, ocean.grid.index_count, 1u, 0u, 0u, 0u); + }*/ draw_ui(cmd_buff); @@ -625,21 +664,21 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) if (drawer.header("Ocean settings")) { - if (drawer.input_float("Amplitude", &ui.amplitude, 0.001f, 3)) + if (drawer.input_float("Amplitude", &ui.amplitude, 0.001f, 3u)) { // update input for fft } - if (drawer.input_float("Length", &ui.length, 0.1f, 1)) + if (drawer.input_float("Length", &ui.length, 0.1f, 1u)) { // update input for fft } if (drawer.header("Wind")) { - if (drawer.input_float("X", &ui.wind.x, 0.5f, 2)) + if (drawer.input_float("X", &ui.wind.x, 0.5f, 2u)) { // update input for fft } - if (drawer.input_float("Y", &ui.wind.y, 0.5f, 2)) + if (drawer.input_float("Y", &ui.wind.y, 0.5f, 2u)) { // update input for fft } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 975f9917b..4cf899d85 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -18,10 +18,10 @@ #pragma once #include "api_vulkan_sample.h" +#include class SubgroupsOperations : public ApiVulkanSample { - struct OceanVertex { glm::vec3 position; @@ -65,6 +65,12 @@ class SubgroupsOperations : public ApiVulkanSample void update_uniform_buffers(); void update_compute_descriptor(); + // ocean stuff + float phillips_spectrum(int32_t n, int32_t m); + std::complex hTilde_0(uint32_t n, uint32_t m); + std::complex rndGaussian(); + + struct Pipeline { void destroy(VkDevice device); @@ -91,8 +97,8 @@ class SubgroupsOperations : public ApiVulkanSample { alignas(4) float amplitude; alignas(4) float length; - alignas(8) glm::vec2 wind; alignas(4) uint32_t grid_size; + alignas(8) glm::vec2 wind; }; struct GuiConfig @@ -109,6 +115,15 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr fft_params_ubo; std::unique_ptr fft_input_buffer; // TODO: change name + std::vector> h_tilde_0; + std::vector> h_hilde_0_conj; + + struct + { + std::unique_ptr buffer1; + std::unique_ptr W; + } helpBuffers; + struct { VkQueue queue = {VK_NULL_HANDLE}; @@ -122,7 +137,6 @@ class SubgroupsOperations : public ApiVulkanSample struct { Pipeline _default; - Pipeline fft_input; } pipelines; } compute; diff --git a/shaders/subgroups_operations/ocean_fft.comp b/shaders/subgroups_operations/ocean_fft.comp index f602a5410..dd12cf1d1 100644 --- a/shaders/subgroups_operations/ocean_fft.comp +++ b/shaders/subgroups_operations/ocean_fft.comp @@ -1,4 +1,4 @@ - #version 450 +#version 450 #extension GL_KHR_shader_subgroup_basic : enable /* Copyright (c) 2023, Mobica Limited @@ -18,10 +18,9 @@ * limitations under the License. */ -layout (local_size_x = 32, local_size_y = 32, local_size_z = 1) in; +layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; #define PI 3.141592f -#define PI2 6.283184 #define GRAVITY 9.81f struct Vertex @@ -42,7 +41,7 @@ struct FFTInputData Complex tilde_conj; }; -Complex complexMultiplication(Complex a, float scalar) +Complex complexMultiplicationByScalar(Complex a, float scalar) { Complex res; res.real = scalar * a.real; @@ -50,6 +49,22 @@ Complex complexMultiplication(Complex a, float scalar) return res; } +Complex complexMultiplication(Complex a, Complex b) +{ + Complex res; + res.real = a.real * b.real; + res.imag = a.imag * b.imag; + return res; +} + +Complex complexSum(Complex a, Complex b) +{ + Complex res; + res.real = a.real + b.real; + res.imag = a.imag + b.imag; + return res; +} + Complex conjugate(Complex a) { Complex res; @@ -62,8 +77,8 @@ layout (set = 0, binding = 0) uniform FFTParametersUbo { float amplitude; float len; - vec2 wind; uint grid_size; + vec2 wind; } fftUbo; layout (std140, set = 0, binding = 1) readonly buffer FFTInputDataBuffer @@ -76,7 +91,6 @@ layout (std140, set = 0, binding = 2) buffer VertexOutBuffer Vertex verticesOut[]; }; - // Add another FFTInputData buffer to store the results of the hTilde calculation // Add 3'th FFTInputData buffer to store fft calculation results - final values @@ -88,25 +102,21 @@ float dispersion(uint n, uint m) return floor(sqrt(GRAVITY * sqrt(x * x + z * z)) / w0) *w0; } - Complex hTilde(float t, uint n, uint m) { uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; - Complex tilde0 = data[idx].tilde + Complex tilde0 = data[idx].tilde; Complex tilde0_conj = data[idx].tilde_conj; float omega = dispersion(n, m); - float _sin = sin(omega); - float _cos = cos(omega); - Complex c0; - c0.real = _cos; - c0.imag = _sin; + c0.real = cos(omega); + c0.imag = sin(omega); Complex c1 = conjugate(c0); - return (tilde0 * c0) + (tilde0_conj * c1); // TODO add a multiply and add functions for Complex struct + return complexSum(complexMultiplication(tilde0, c0), complexMultiplication(tilde0_conj, c1)); } void fft() diff --git a/shaders/subgroups_operations/ocean_fft_input.comp b/shaders/subgroups_operations/ocean_fft_input.comp deleted file mode 100644 index 03cd72f78..000000000 --- a/shaders/subgroups_operations/ocean_fft_input.comp +++ /dev/null @@ -1,131 +0,0 @@ -#version 450 -#extension GL_KHR_shader_subgroup_basic : enable - -/* Copyright (c) 2023, Mobica Limited - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -layout (local_size_x = 32, local_size_y = 32, local_size_z = 1) in; - -#define PI 3.141592f -#define PI2 6.283184 -#define GRAVITY 9.81f - -struct Complex -{ - float real; - float imag; -}; - -struct FFTInputData -{ - Complex tilde; - Complex tilde_conj; -}; - -layout (set = 0, binding = 0) uniform FFTParametersUbo -{ - float amplitude; - float len; - vec2 wind; - uint grid_size; -} fftUbo; - -layout (std140, set = 0, binding = 1) buffer FFTInputDataBuffer -{ - FFTInputData data[]; -}; - -Complex complexMultiplication(Complex a, float scalar) -{ - Complex res; - res.real = scalar * a.real; - res.imag = scalar * a.imag; - return res; -} - -Complex conjugate(Complex a) -{ - Complex res; - res.real = a.real; - res.imag = -a.imag; - return res; -} - -float phillips_spectrum(uint n, uint m) -{ - vec2 k = vec2(PI * (2.0f * n - fftUbo.grid_size) / fftUbo.len, PI * (2.0f * m - fftUbo.grid_size) / fftUbo.len); - - float k_length = length(k); - if (k_length < 0.000001f) - return 0.0f; - - float k_len2 = k_length * k_length; - float k_len4 = k_len2 * k_len2; - - float k_dot_wind = dot(normalize(k), normalize(fftUbo.wind)); - float k_dot_wind2 = k_dot_wind * k_dot_wind; - - float wind_len = length(fftUbo.wind); - float L = wind_len * wind_len / GRAVITY; - float L2 = L * L; - - float damping = 0.001f; - float c2 = L2 * damping * damping; - - return fftUbo.amplitude * exp(-1.0f / (k_len2 * L2)) / k_len4 * k_dot_wind2 * exp(-k_len2 * c2); -} - -// source: https://stackoverflow.com/a/74419913 -float random(vec2 st) -{ - float rndVal = fract(sin(dot(st, vec2(12.9898f, 78.233f))) * 43758.5453123f); - if (rndVal < 0.000001f) return 0.000001f; - return rndVal; -} - -Complex gaussionRndVal(vec2 seed) -{ - float x1, x2, w, w2; - x1 = random(seed); - x1 = -2.0f * log(x1); - x2 = random(seed) * PI2; - - w = sqrt(x1) * cos(x2); - - Complex res; - res.real = x1 * w; - res.imag = x2 * w; - return res; -} - -Complex hTilde(uint n, uint m) -{ - Complex rnd = gaussionRndVal(vec2(abs(n), abs(m))); - return complexMultiplication(rnd, sqrt(phillips_spectrum(n, m) / 2.0f)); -} - -void main() -{ - uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; - subgroupMemoryBarrierBuffer(); - uint n = gl_GlobalInvocationID.x; - uint m = gl_GlobalInvocationID.y; - - - data[idx].tilde = hTilde(n, m); // possible issue - data[idx].tilde_conj = conjugate(data[idx].tilde); -} From f052073f8ee54f0343cd416d1e81f4d7cc67d49c Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 6 Sep 2023 18:38:12 +0200 Subject: [PATCH 10/53] Refactoring and changes in graphic pipeline --- .../subgroups_operations.cpp | 195 ++++++++++-------- .../subgroups_operations.h | 24 +-- shaders/subgroups_operations/ocean.frag | 3 +- shaders/subgroups_operations/ocean.tesc | 27 +++ shaders/subgroups_operations/ocean.tese | 23 +++ shaders/subgroups_operations/ocean.vert | 3 - shaders/subgroups_operations/ocean_fft.comp | 35 ++-- 7 files changed, 190 insertions(+), 120 deletions(-) create mode 100644 shaders/subgroups_operations/ocean.tesc create mode 100644 shaders/subgroups_operations/ocean.tese diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 778e17b20..86c295aa9 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -157,10 +157,11 @@ void SubgroupsOperations::create_compute_command_buffer() void SubgroupsOperations::create_compute_descriptor_set_layout() { std::vector set_layout_bindngs = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), // FFTParametersUbo - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), // Vertex data - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) // FFTInputData fft data - }; + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 3u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, VK_SHADER_STAGE_COMPUTE_BIT, 4u)}; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); @@ -176,14 +177,20 @@ void SubgroupsOperations::create_compute_descriptor_set() void SubgroupsOperations::update_compute_descriptor() { - VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); - VkDescriptorBufferInfo input_fft_data_buffer = create_descriptor(*fft_input_buffer); - VkDescriptorBufferInfo output_vertices_buffer = create_descriptor(*ocean.grid.vertex); + VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); + VkDescriptorBufferInfo fft_input_h_tilde0_data_buffer = create_descriptor(*fft_buffers.fft_input_htilde0); + VkDescriptorBufferInfo fft_input_h_tilde0_conj_data_buffer = create_descriptor(*fft_buffers.fft_input_htilde0_conj); + VkDescriptorBufferInfo fft_input_weight_data_buffer = create_descriptor(*fft_buffers.fft_input_weight); + // VkDescriptorBufferInfo fft_input_image = create_descriptor(*fft_buffers.fft_height_map_image->get_vk_image()); std::vector wirte_descriptor_sets = { vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &fft_params_ubo_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &input_fft_data_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &output_vertices_buffer)}; + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &fft_input_h_tilde0_data_buffer), + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &fft_input_h_tilde0_conj_data_buffer), + vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 3u, &fft_input_weight_data_buffer), + // vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 4u, &fft_input_image) + + }; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(wirte_descriptor_sets.size()), wirte_descriptor_sets.data(), 0u, nullptr); } @@ -220,9 +227,9 @@ void SubgroupsOperations::build_compute_command_buffer() memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; - memory_barrier.buffer = fft_input_buffer->get_handle(); + memory_barrier.buffer = fft_buffers.fft_input_htilde0->get_handle(); memory_barrier.offset = 0u; - memory_barrier.size = fft_input_buffer->get_size(); + memory_barrier.size = fft_buffers.fft_input_htilde0->get_size(); vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); } @@ -241,9 +248,9 @@ void SubgroupsOperations::build_compute_command_buffer() memory_barrier.dstAccessMask = 0u; memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.buffer = fft_input_buffer->get_handle(); + memory_barrier.buffer = fft_buffers.fft_input_htilde0->get_handle(); memory_barrier.offset = 0u; - memory_barrier.size = fft_input_buffer->get_size(); + memory_barrier.size = fft_buffers.fft_input_htilde0->get_size(); vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); } @@ -258,6 +265,15 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) gpu.get_mutable_requested_features().fillModeNonSolid = VK_TRUE; } + if (gpu.get_features().tessellationShader) + { + gpu.get_mutable_requested_features().tessellationShader = VK_TRUE; + } + else + { + throw vkb::VulkanException(VK_ERROR_FEATURE_NOT_PRESENT, "Selected GPU does not support tessellation shaders!"); + } + subgroups_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; subgroups_properties.pNext = VK_NULL_HANDLE; @@ -269,32 +285,52 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) void SubgroupsOperations::load_assets() { - // generate_grid(); - auto input_buffer_size = static_cast(grid_size * grid_size * (sizeof(float) * 4u)); // max grid size * max grid size * sizeof(std::complex) - real: float; imag: float - fft_input_buffer = std::make_unique(get_device(), - input_buffer_size, - VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, - VMA_MEMORY_USAGE_GPU_ONLY); - auto vertex_buffer_size = grid_size * grid_size * sizeof(OceanVertex); - ocean.grid.vertex = std::make_unique(get_device(), - vertex_buffer_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, - VMA_MEMORY_USAGE_GPU_ONLY); - size_t log_2_n = log(grid_size) / log(2); - auto helpBuffer1Size = VkDeviceSize(log_2_n * (sizeof(float) * 4) * grid_size); - helpBuffers.buffer1 = std::make_unique(get_device(), helpBuffer1Size, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, - VMA_MEMORY_USAGE_GPU_ONLY); + generate_plane(); + // generate fft inputs h_tilde_0.clear(); - h_hilde_0_conj.clear(); + h_tilde_0_conj.clear(); for (uint32_t m = 0; m < grid_size + 1U; ++m) { for (uint32_t n = 0; n < grid_size + 1U; ++n) { h_tilde_0.push_back(hTilde_0(n, m)); - h_hilde_0_conj.push_back(std::conj(hTilde_0(-n, -m))); + h_tilde_0_conj.push_back(std::conj(hTilde_0(-n, -m))); } } + + // calculate weights + auto log_2_N = glm::log(grid_size) / glm::log(2.0f); + uint32_t pow2 = 1U; + for (uint32_t i = 0U; i < log_2_N; ++i) + { + for (uint32_t j = 0U; j < pow2; ++j) + { + weights.push_back(calculate_weight(j, pow2 * 2)); + } + pow2 *= 2; + } + + auto fft_input_htilde0_size = static_cast(h_tilde_0.size() * sizeof(std::complex)); + fft_buffers.fft_input_htilde0 = std::make_unique(get_device(), + fft_input_htilde0_size, + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); + auto fft_input_htilde0_conj_size = static_cast(h_tilde_0_conj.size() * sizeof(std::complex)); + + fft_buffers.fft_input_htilde0_conj = std::make_unique(get_device(), + fft_input_htilde0_conj_size, + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); + + auto fft_input_weights_size = static_cast(weights.size() * sizeof(std::complex)); + fft_buffers.fft_input_weight = std::make_unique(get_device(), fft_input_weights_size, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + + fft_buffers.fft_input_htilde0->update(h_tilde_0.data(), h_tilde_0.size()); + fft_buffers.fft_input_htilde0_conj->update(h_tilde_0_conj.data(), h_tilde_0_conj.size()); + fft_buffers.fft_input_weight->update(weights.data(), weights.size()); + + // fft_buffers.fft_height_map_image = std::make_unique(grid_size * grid_size); } float SubgroupsOperations::phillips_spectrum(int32_t n, int32_t m) @@ -321,6 +357,13 @@ float SubgroupsOperations::phillips_spectrum(int32_t n, int32_t m) return ui.amplitude * glm::exp(-1.0f / (k_len2 * L2)) / k_len4 * k_dot_w2 * glm::exp(-k_len2 * l2); } +std::complex SubgroupsOperations::calculate_weight(uint32_t x, uint32_t n) +{ + const auto pi2 = glm::pi() * 2.0f; + return { + glm::cos(pi2 * x / n), glm::sin(pi2 * x / n)}; +} + std::complex SubgroupsOperations::rndGaussian() { float x1, x2, w; @@ -354,58 +397,35 @@ void SubgroupsOperations::prepare_uniform_buffers() update_uniform_buffers(); } -void SubgroupsOperations::generate_grid() +void SubgroupsOperations::generate_plane() { - std::vector grid_vertices; - std::vector grid_indices; + std::vector plane_vertices; - for (uint32_t z = 0u; z <= grid_size; ++z) - { - for (uint32_t x = 0u; x <= grid_size; ++x) - { - Vertex point = {}; - point.pos.x = static_cast((static_cast(x) / static_cast(grid_size - 1) * 2 - 1) * (grid_size / 2.0f)); - point.pos.z = static_cast((static_cast(z) / static_cast(grid_size - 1) * 2 - 1) * (grid_size / 2.0f)); - point.pos.y = 0.0f; - point.normal = glm::vec3(0.0f, 1.0f, 0.0); - grid_vertices.push_back(point); - } - } + Vertex v; + v.pos = {10.0f, 10.0f, 0.0f}; + plane_vertices.push_back(v); + v.pos = {-10.0f, 10.0f, 0.0f}; + plane_vertices.push_back(v); + v.pos = {-10.0f, -10.0f, 0.0f}; + plane_vertices.push_back(v); + v.pos = {10.0f, -10.0f, 0.0f}; + plane_vertices.push_back(v); - for (uint32_t z = 0u; z < grid_size; ++z) - { - for (uint32_t x = 0u; x < grid_size; ++x) - { - uint32_t i0 = z * (grid_size + 1u) + x; - uint32_t i1 = i0 + 1u; - uint32_t i2 = i0 + (grid_size + 1u); - uint32_t i3 = i2 + 1u; - - grid_indices.push_back(i0); - grid_indices.push_back(i2); - grid_indices.push_back(i1); - - grid_indices.push_back(i1); - grid_indices.push_back(i2); - grid_indices.push_back(i3); - } - } + std::vector indices = {0, 1, 2, 2, 3, 0}; - auto vertex_buffer_size = vkb::to_u32(grid_vertices.size() * sizeof(Vertex)); - auto index_buffer_size = vkb::to_u32(grid_indices.size() * sizeof(uint32_t)); - ocean.grid.index_count = vkb::to_u32(grid_indices.size()); + auto vertex_buffer_size = vkb::to_u32(plane_vertices.size() * sizeof(Vertex)); + auto index_buffer_size = vkb::to_u32(indices.size() * sizeof(uint32_t)); + ocean.grid.index_count = vkb::to_u32(indices.size()); - input_grid.vertex = std::make_unique(get_device(), - vertex_buffer_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, - VMA_MEMORY_USAGE_CPU_TO_GPU); - input_grid.vertex->update(grid_vertices.data(), vertex_buffer_size); + ocean.grid.vertex = std::make_unique(get_device(), vertex_buffer_size, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); ocean.grid.index = std::make_unique(get_device(), index_buffer_size, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - ocean.grid.index->update(grid_indices.data(), index_buffer_size); + + ocean.grid.vertex->update(plane_vertices.data(), vertex_buffer_size); + ocean.grid.index->update(indices.data(), index_buffer_size); } void SubgroupsOperations::create_semaphore() @@ -489,14 +509,18 @@ void SubgroupsOperations::create_pipelines() dynamic_state_enables.data(), static_cast(dynamic_state_enables.size()), 0u); - std::array shader_stages; - shader_stages[0] = load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT); - shader_stages[1] = load_shader("subgroups_operations/ocean.frag", VK_SHADER_STAGE_FRAGMENT_BIT); + VkPipelineTessellationStateCreateInfo tessellation_state = vkb::initializers::pipeline_tessellation_state_create_info(4); + + std::array shader_stages; + shader_stages[0] = load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT); + shader_stages[1] = load_shader("subgroups_operations/ocean.frag", VK_SHADER_STAGE_FRAGMENT_BIT); + shader_stages[2] = load_shader("subgroups_operations/ocean.tesc", VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT); + shader_stages[3] = load_shader("subgroups_operations/ocean.tese", VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT); + const std::vector vertex_input_bindings = { vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX)}; const std::vector vertex_input_attributes = { - vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), - vkb::initializers::vertex_input_attribute_description(0u, 1u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, normal))}; + vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos))}; VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); @@ -512,6 +536,7 @@ void SubgroupsOperations::create_pipelines() pipeline_create_info.pViewportState = &viewport_state; pipeline_create_info.pDepthStencilState = &depth_stencil_state; pipeline_create_info.pDynamicState = &dynamic_state; + pipeline_create_info.pTessellationState = &tessellation_state; pipeline_create_info.stageCount = static_cast(shader_stages.size()); pipeline_create_info.pStages = shader_stages.data(); VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &ocean.pipelines._default.pipeline)); @@ -585,16 +610,16 @@ void SubgroupsOperations::build_command_buffers() vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); // draw ocean - /*{ - vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); - vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ui.wireframe ? ocean.pipelines.wireframe.pipeline : ocean.pipelines._default.pipeline); + { + vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); + vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ui.wireframe ? ocean.pipelines.wireframe.pipeline : ocean.pipelines._default.pipeline); - VkDeviceSize offset[] = {0u}; - vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, ocean.grid.vertex->get(), offset); - vkCmdBindIndexBuffer(cmd_buff, ocean.grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); + VkDeviceSize offset[] = {0u}; + vkCmdBindVertexBuffers(cmd_buff, 0u, 1u, ocean.grid.vertex->get(), offset); + vkCmdBindIndexBuffer(cmd_buff, ocean.grid.index->get_handle(), VkDeviceSize(0), VK_INDEX_TYPE_UINT32); - vkCmdDrawIndexed(cmd_buff, ocean.grid.index_count, 1u, 0u, 0u, 0u); - }*/ + vkCmdDrawIndexed(cmd_buff, ocean.grid.index_count, 1u, 0u, 0u, 0u); + } draw_ui(cmd_buff); diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 4cf899d85..daf1ef713 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -25,7 +25,6 @@ class SubgroupsOperations : public ApiVulkanSample struct OceanVertex { glm::vec3 position; - glm::vec3 normal; }; public: @@ -54,7 +53,7 @@ class SubgroupsOperations : public ApiVulkanSample void build_compute_command_buffer(); - void generate_grid(); + void generate_plane(); void prepare_uniform_buffers(); void setup_descriptor_pool(); void create_semaphore(); @@ -66,10 +65,10 @@ class SubgroupsOperations : public ApiVulkanSample void update_compute_descriptor(); // ocean stuff - float phillips_spectrum(int32_t n, int32_t m); + float phillips_spectrum(int32_t n, int32_t m); std::complex hTilde_0(uint32_t n, uint32_t m); std::complex rndGaussian(); - + std::complex calculate_weight(uint32_t x, uint32_t n); struct Pipeline { @@ -109,20 +108,21 @@ class SubgroupsOperations : public ApiVulkanSample glm::vec2 wind = {16.0f, 0.0f}; } ui; - GridBuffers input_grid; // input buffer for compute shader - uint32_t grid_size = {32u}; + uint32_t grid_size = {128u}; std::unique_ptr camera_ubo; std::unique_ptr fft_params_ubo; - std::unique_ptr fft_input_buffer; // TODO: change name std::vector> h_tilde_0; - std::vector> h_hilde_0_conj; + std::vector> h_tilde_0_conj; + std::vector> weights; struct { - std::unique_ptr buffer1; - std::unique_ptr W; - } helpBuffers; + std::unique_ptr fft_input_htilde0; + std::unique_ptr fft_input_htilde0_conj; + std::unique_ptr fft_input_weight; + std::unique_ptr fft_height_map_image; + } fft_buffers; struct { @@ -142,7 +142,7 @@ class SubgroupsOperations : public ApiVulkanSample struct { - GridBuffers grid; // output (result) buffer for compute shader + GridBuffers grid; uint32_t graphics_queue_family_index = {-1u}; VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index 650ba3eca..58416b076 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -16,12 +16,11 @@ * limitations under the License. */ - layout (location = 0) in vec3 inNormal; layout (location = 0) out vec4 outFragColor; void main() { - vec3 ocean_color = vec3(0.0f, 0.5f, 1.0f) + inNormal; + vec3 ocean_color = vec3(0.0f, 0.5f, 1.0f); outFragColor = vec4(ocean_color, 1.0f); } diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc new file mode 100644 index 000000000..eba26d06d --- /dev/null +++ b/shaders/subgroups_operations/ocean.tesc @@ -0,0 +1,27 @@ +#version 450 + +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + layout (vertices = 4) out; + + + void main() + { + gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position; + + } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese new file mode 100644 index 000000000..ef606d6ee --- /dev/null +++ b/shaders/subgroups_operations/ocean.tese @@ -0,0 +1,23 @@ +#version 450 + +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + layout(quads, equal_spacing, cw) in; + + void main() + { + } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.vert b/shaders/subgroups_operations/ocean.vert index 86c6a87d5..9d6b773a6 100644 --- a/shaders/subgroups_operations/ocean.vert +++ b/shaders/subgroups_operations/ocean.vert @@ -17,7 +17,6 @@ */ layout (location = 0) in vec3 inPos; -layout (location = 1) in vec3 inNormal; layout (binding = 0) uniform Ubo { @@ -26,10 +25,8 @@ layout (binding = 0) uniform Ubo mat4 model; } ubo; -layout (location = 0) out vec3 outNormal; void main() { gl_Position = ubo.projection * ubo.view * ubo.model * vec4(inPos.xyz, 1.0f); - outNormal = inNormal; } diff --git a/shaders/subgroups_operations/ocean_fft.comp b/shaders/subgroups_operations/ocean_fft.comp index dd12cf1d1..78b62b6cb 100644 --- a/shaders/subgroups_operations/ocean_fft.comp +++ b/shaders/subgroups_operations/ocean_fft.comp @@ -18,7 +18,7 @@ * limitations under the License. */ -layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; +layout (local_size_x = 4, local_size_y = 4, local_size_z = 2) in; #define PI 3.141592f #define GRAVITY 9.81f @@ -26,7 +26,7 @@ layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; struct Vertex { vec3 position; - vec3 normal; +// vec3 normal; }; struct Complex @@ -35,12 +35,6 @@ struct Complex float imag; }; -struct FFTInputData -{ - Complex tilde; - Complex tilde_conj; -}; - Complex complexMultiplicationByScalar(Complex a, float scalar) { Complex res; @@ -81,16 +75,25 @@ layout (set = 0, binding = 0) uniform FFTParametersUbo vec2 wind; } fftUbo; -layout (std140, set = 0, binding = 1) readonly buffer FFTInputDataBuffer +layout (std140, set = 0, binding = 1) readonly buffer FFTInputHTilda0_DataBuffer +{ + Complex htilde0[]; +}; + + +layout (std140, set = 0, binding = 2) readonly buffer FFTInputHTilda0Conj_DataBuffer { - FFTInputData data[]; + Complex htilde0_conj[]; }; -layout (std140, set = 0, binding = 2) buffer VertexOutBuffer +layout (std140, set = 0, binding = 3) readonly buffer FFTInputWeight_DataBuffer { - Vertex verticesOut[]; + Complex weight[]; }; + +layout (binding = 4, rgba32f) writeonly uniform image2D result_texture; + // Add another FFTInputData buffer to store the results of the hTilde calculation // Add 3'th FFTInputData buffer to store fft calculation results - final values @@ -105,8 +108,8 @@ float dispersion(uint n, uint m) Complex hTilde(float t, uint n, uint m) { uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; - Complex tilde0 = data[idx].tilde; - Complex tilde0_conj = data[idx].tilde_conj; + Complex tilde0 ; //= data[idx].tilde; + Complex tilde0_conj; // = data[idx].tilde_conj; float omega = dispersion(n, m); @@ -130,8 +133,4 @@ void main() subgroupMemoryBarrierBuffer(); - FFTInputData inData = data[idx]; - - verticesOut[idx].position = vec3(0.0f); - verticesOut[idx].normal = vec3(0.0f); } From a7a2b143f7c781ba4f6a2ac04541736a7b834050 Mon Sep 17 00:00:00 2001 From: Krzysztof-Dmitruk-Mobica Date: Tue, 12 Sep 2023 12:00:05 +0200 Subject: [PATCH 11/53] Add the FBAttachment type --- .../subgroups_operations.cpp | 38 +++++++++++++++++++ .../subgroups_operations.h | 15 ++++++++ 2 files changed, 53 insertions(+) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 86c295aa9..bb8c0f090 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -736,3 +736,41 @@ std::unique_ptr create_subgroups_operations() { return std::make_unique(); } + +void SubgroupsOperations::createFBAttachement(VkFormat format, uint32_t width, uint32_t height, FBAttachment *attachment) { + attachment->format = format; + + VkImageCreateInfo image = vkb::initializers::image_create_info(); + image.imageType = VK_IMAGE_TYPE_2D; + image.format = format; + image.extent.width = width; + image.extent.height = height; + image.extent.depth = 1; + image.mipLevels = 1; + image.arrayLayers = 1; + image.samples = VK_SAMPLE_COUNT_1_BIT; + image.tiling = VK_IMAGE_TILING_OPTIMAL; + image.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT; + + VkMemoryAllocateInfo memory_allocate_info = vkb::initializers::memory_allocate_info(); + VkMemoryRequirements memory_requirements; + + VK_CHECK(vkCreateImage(get_device().get_handle(), &image, nullptr, &attachment->image)); + vkGetImageMemoryRequirements(get_device().get_handle(), attachment->image, &memory_requirements); + memory_allocate_info.allocationSize = memory_requirements.size; + memory_allocate_info.memoryTypeIndex = get_device().get_memory_type(memory_requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + VK_CHECK(vkAllocateMemory(get_device().get_handle(), &memory_allocate_info, nullptr, &attachment->memory)); + VK_CHECK(vkBindImageMemory(get_device().get_handle(), attachment->image, attachment->memory, 0)); + + VkImageViewCreateInfo image_view_create_info = vkb::initializers::image_view_create_info(); + image_view_create_info.viewType = VK_IMAGE_VIEW_TYPE_2D; + image_view_create_info.format = format; + image_view_create_info.subresourceRange = {}; + image_view_create_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_view_create_info.subresourceRange.baseMipLevel = 0; + image_view_create_info.subresourceRange.levelCount = 1; + image_view_create_info.subresourceRange.baseArrayLayer = 0; + image_view_create_info.subresourceRange.layerCount = 1; + image_view_create_info.image = attachment->image; + VK_CHECK(vkCreateImageView(get_device().get_handle(), &image_view_create_info, nullptr, &attachment->view)); +} diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index daf1ef713..0d905067f 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -116,6 +116,21 @@ class SubgroupsOperations : public ApiVulkanSample std::vector> h_tilde_0_conj; std::vector> weights; + struct FBAttachment + { + VkImage image; + VkDeviceMemory memory; + VkImageView view; + VkFormat format; + void destroy(VkDevice device) + { + vkDestroyImageView(device, view, nullptr); + vkDestroyImage(device, image, nullptr); + vkFreeMemory(device, memory, nullptr); + }; + }; + void createFBAttachement(VkFormat format, uint32_t width, uint32_t height, FBAttachment *result); + struct { std::unique_ptr fft_input_htilde0; From bdb3b31cafc881c66ec5d89c37de1a4ef1584ffb Mon Sep 17 00:00:00 2001 From: Krzysztof-Dmitruk-Mobica Date: Wed, 20 Sep 2023 13:46:34 +0200 Subject: [PATCH 12/53] Add butterfly precomp shader and texture --- .../subgroups_operations/CMakeLists.txt | 3 +- .../subgroups_operations.cpp | 156 +++++++++++++----- .../subgroups_operations.h | 46 ++++-- .../butterfly_precomp.comp | 68 ++++++++ 4 files changed, 220 insertions(+), 53 deletions(-) create mode 100644 shaders/subgroups_operations/butterfly_precomp.comp diff --git a/samples/extensions/subgroups_operations/CMakeLists.txt b/samples/extensions/subgroups_operations/CMakeLists.txt index d7b0a128b..417b2d041 100644 --- a/samples/extensions/subgroups_operations/CMakeLists.txt +++ b/samples/extensions/subgroups_operations/CMakeLists.txt @@ -28,4 +28,5 @@ add_sample( SHADER_FILES_GLSL "subgroups_operations/ocean.vert" "subgroups_operations/ocean.frag" - "subgroups_operations/ocean_fft.comp") + "subgroups_operations/ocean_fft.comp" + "subgroups_operations/butterfly_precomp.comp") diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index bb8c0f090..96930f78c 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -21,6 +21,7 @@ #include #define GRAVITY 9.81f +#define DISPLACEMENT_MAP_DIM 256 void SubgroupsOperations::Pipeline::destroy(VkDevice device) { @@ -99,6 +100,8 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) create_descriptor_set(); create_pipelines(); + create_butterfly_texture(); + build_command_buffers(); // signal semaphore @@ -287,6 +290,8 @@ void SubgroupsOperations::load_assets() { generate_plane(); + log_2_N = log(DISPLACEMENT_MAP_DIM) / log(2); + // generate fft inputs h_tilde_0.clear(); h_tilde_0_conj.clear(); @@ -439,9 +444,11 @@ void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u), // FFTParametersUbo, CameraUbo, ComputeUbo - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u)}; + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 3u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u), + }; VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 2u); + vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 3u); VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } @@ -737,40 +744,113 @@ std::unique_ptr create_subgroups_operations() return std::make_unique(); } -void SubgroupsOperations::createFBAttachement(VkFormat format, uint32_t width, uint32_t height, FBAttachment *attachment) { - attachment->format = format; - - VkImageCreateInfo image = vkb::initializers::image_create_info(); - image.imageType = VK_IMAGE_TYPE_2D; - image.format = format; - image.extent.width = width; - image.extent.height = height; - image.extent.depth = 1; - image.mipLevels = 1; - image.arrayLayers = 1; - image.samples = VK_SAMPLE_COUNT_1_BIT; - image.tiling = VK_IMAGE_TILING_OPTIMAL; - image.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT; - - VkMemoryAllocateInfo memory_allocate_info = vkb::initializers::memory_allocate_info(); - VkMemoryRequirements memory_requirements; - - VK_CHECK(vkCreateImage(get_device().get_handle(), &image, nullptr, &attachment->image)); - vkGetImageMemoryRequirements(get_device().get_handle(), attachment->image, &memory_requirements); - memory_allocate_info.allocationSize = memory_requirements.size; - memory_allocate_info.memoryTypeIndex = get_device().get_memory_type(memory_requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - VK_CHECK(vkAllocateMemory(get_device().get_handle(), &memory_allocate_info, nullptr, &attachment->memory)); - VK_CHECK(vkBindImageMemory(get_device().get_handle(), attachment->image, attachment->memory, 0)); - - VkImageViewCreateInfo image_view_create_info = vkb::initializers::image_view_create_info(); - image_view_create_info.viewType = VK_IMAGE_VIEW_TYPE_2D; - image_view_create_info.format = format; - image_view_create_info.subresourceRange = {}; - image_view_create_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - image_view_create_info.subresourceRange.baseMipLevel = 0; - image_view_create_info.subresourceRange.levelCount = 1; - image_view_create_info.subresourceRange.baseArrayLayer = 0; - image_view_create_info.subresourceRange.layerCount = 1; - image_view_create_info.image = attachment->image; - VK_CHECK(vkCreateImageView(get_device().get_handle(), &image_view_create_info, nullptr, &attachment->view)); +// TODO: move out usage to the function arguments +void SubgroupsOperations::createFBAttachement(VkFormat format, uint32_t width, uint32_t height, FBAttachment &attachment) +{ + attachment.format = format; + + VkImageCreateInfo image = vkb::initializers::image_create_info(); + image.imageType = VK_IMAGE_TYPE_2D; + image.format = format; + image.extent.width = width; + image.extent.height = height; + image.extent.depth = 1; + image.mipLevels = 1; + image.arrayLayers = 1; + image.samples = VK_SAMPLE_COUNT_1_BIT; + image.tiling = VK_IMAGE_TILING_OPTIMAL; + image.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT | VK_IMAGE_USAGE_STORAGE_BIT; + + VkMemoryAllocateInfo memory_allocate_info = vkb::initializers::memory_allocate_info(); + VkMemoryRequirements memory_requirements; + + VK_CHECK(vkCreateImage(get_device().get_handle(), &image, nullptr, &attachment.image)); + vkGetImageMemoryRequirements(get_device().get_handle(), attachment.image, &memory_requirements); + memory_allocate_info.allocationSize = memory_requirements.size; + memory_allocate_info.memoryTypeIndex = get_device().get_memory_type(memory_requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + VK_CHECK(vkAllocateMemory(get_device().get_handle(), &memory_allocate_info, nullptr, &attachment.memory)); + VK_CHECK(vkBindImageMemory(get_device().get_handle(), attachment.image, attachment.memory, 0)); + + VkImageViewCreateInfo image_view_create_info = vkb::initializers::image_view_create_info(); + image_view_create_info.viewType = VK_IMAGE_VIEW_TYPE_2D; + image_view_create_info.format = format; + image_view_create_info.subresourceRange = {}; + image_view_create_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_view_create_info.subresourceRange.baseMipLevel = 0; + image_view_create_info.subresourceRange.levelCount = 1; + image_view_create_info.subresourceRange.baseArrayLayer = 0; + image_view_create_info.subresourceRange.layerCount = 1; + image_view_create_info.image = attachment.image; + VK_CHECK(vkCreateImageView(get_device().get_handle(), &image_view_create_info, nullptr, &attachment.view)); +} + +uint32_t SubgroupsOperations::reverse(uint32_t i) +{ + uint32_t res = 0; + for (int j = 0; j < log_2_N; j++) + { + res = (res << 1) + (i & 1); + i >>= 1; + } + return res; +} + +void SubgroupsOperations::create_butterfly_texture() +{ + VkCommandPool commandPool = compute.command_pool; + VkCommandBuffer command_buffer = compute.command_buffer; + + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u)}; + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &precompute.descriptor_set_layout)); + + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &precompute.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &precompute.descriptor_set)); + + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &precompute.descriptor_set_layout; + + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &precompute.pipeline.pipeline_layout)); + + VkComputePipelineCreateInfo computeInfo = {}; + computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + computeInfo.layout = precompute.pipeline.pipeline_layout; + computeInfo.stage = load_shader("subgroups_operations/butterfly_precomp.comp", VK_SHADER_STAGE_COMPUTE_BIT); + + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &precompute.pipeline.pipeline)); + + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, log_2_N, DISPLACEMENT_MAP_DIM, butterfly_precomp); + + std::array bit_reverse_arr; + for (uint32_t i = 0; i < DISPLACEMENT_MAP_DIM; ++i) + bit_reverse_arr[i] = reverse(i); + + bit_reverse_buffer = std::make_unique(get_device(), sizeof(uint32_t) * bit_reverse_arr.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + + VkDescriptorBufferInfo bit_reverse_descriptor = create_descriptor(*bit_reverse_buffer); + bit_reverse_buffer->convert_and_update(bit_reverse_arr.data(), sizeof(uint32_t) * bit_reverse_arr.size()); + + VkDescriptorImageInfo image_descriptor{}; + image_descriptor.imageView = butterfly_precomp.view; + image_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor.sampler = nullptr; + + std::vector write_descriptor_sets = { + vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor), + vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &bit_reverse_descriptor)}; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); + + VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); + VK_CHECK(vkBeginCommandBuffer(command_buffer, &begin_info)); + + vkCmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline); + vkCmdBindDescriptorSets(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline_layout, 0u, 1u, &precompute.descriptor_set, 0u, nullptr); + + vkCmdDispatch(command_buffer, 32u, 32u, 1u); + + VK_CHECK(vkEndCommandBuffer(command_buffer)); } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 0d905067f..4cd7a3d0d 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -61,6 +61,8 @@ class SubgroupsOperations : public ApiVulkanSample void create_descriptor_set(); void create_pipelines(); + void create_butterfly_texture(); + void update_uniform_buffers(); void update_compute_descriptor(); @@ -116,20 +118,25 @@ class SubgroupsOperations : public ApiVulkanSample std::vector> h_tilde_0_conj; std::vector> weights; - struct FBAttachment - { - VkImage image; - VkDeviceMemory memory; - VkImageView view; - VkFormat format; - void destroy(VkDevice device) - { - vkDestroyImageView(device, view, nullptr); - vkDestroyImage(device, image, nullptr); - vkFreeMemory(device, memory, nullptr); - }; - }; - void createFBAttachement(VkFormat format, uint32_t width, uint32_t height, FBAttachment *result); + struct FBAttachment + { + VkImage image; + VkDeviceMemory memory; + VkImageView view; + VkFormat format; + void destroy(VkDevice device) + { + vkDestroyImageView(device, view, nullptr); + vkDestroyImage(device, image, nullptr); + vkFreeMemory(device, memory, nullptr); + }; + } butterfly_precomp; + + uint32_t log_2_N; + + std::unique_ptr bit_reverse_buffer; + + void createFBAttachement(VkFormat format, uint32_t width, uint32_t height, FBAttachment &result); struct { @@ -152,9 +159,17 @@ class SubgroupsOperations : public ApiVulkanSample struct { Pipeline _default; + } pipelines; } compute; + struct + { + VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; + Pipeline pipeline; + } precompute; + struct { GridBuffers grid; @@ -170,6 +185,9 @@ class SubgroupsOperations : public ApiVulkanSample } ocean; VkPhysicalDeviceSubgroupProperties subgroups_properties; + + private: + uint32_t reverse(uint32_t i); }; std::unique_ptr create_subgroups_operations(); diff --git a/shaders/subgroups_operations/butterfly_precomp.comp b/shaders/subgroups_operations/butterfly_precomp.comp new file mode 100644 index 000000000..e8debeb90 --- /dev/null +++ b/shaders/subgroups_operations/butterfly_precomp.comp @@ -0,0 +1,68 @@ +#version 460 core + +#define PI_32F 3.14159265358979f +#define DISPLACEMENT_MAP_DIM 256 + +layout (local_size_x = 8, local_size_y = 32) in; + +layout (binding = 0, rgba32f) writeonly uniform image2D u_butterfly_precomp; + +layout (std430, binding = 1) buffer indices +{ + int data[]; +} +bit_reversed_indices; + + +void main() +{ + vec2 pos = gl_GlobalInvocationID.xy; + + int N = DISPLACEMENT_MAP_DIM; + + // Twiddle factor exponent, Thesis, Section 4.2.6, Eq 4.6 + float k = mod(pos.y * (float(N) / pow(2, pos.x + 1)), N); + + // Thesis, Section 4.2.6, Eq 4.7 + int butterfly_span = int(pow(2, pos.x)); + int butterfly_wing = 0; + + if ((mod(pos.y, pow(2, pos.x + 1))) < butterfly_span) butterfly_wing = 1; + + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + + vec4 result; + result.x = cos(2.f * PI_32F * k / float(N)); // Twiddle factor real part + result.y = sin(2.f * PI_32F * k / float(N)); // Twiddle factor imaginary part + + // Store the sample indices for the next stage + if (pos.x == 0) + { + if (butterfly_wing == 1) + { + result.z = bit_reversed_indices.data[int(pos.y)]; + result.w = bit_reversed_indices.data[int(pos.y + 1)]; + } + else + { + result.z = bit_reversed_indices.data[int(pos.y - 1)]; + result.w = bit_reversed_indices.data[int(pos.y)]; + } + } + else + { + if (butterfly_wing == 1) + { + result.z = pos.y; + result.w = pos.y + butterfly_span; + } + else + { + result.z = pos.y - butterfly_span; + result.w = pos.y; + } + } + + imageStore(u_butterfly_precomp, pixel_pos, result); +} + From fd6d9705eebeb00dfc584971848d5caea5855c67 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Thu, 21 Sep 2023 15:42:17 +0200 Subject: [PATCH 13/53] Add tilde textures and pipeline --- .../subgroups_operations/CMakeLists.txt | 3 +- .../subgroups_operations.cpp | 126 ++++++++++++++++-- .../subgroups_operations.h | 25 +++- shaders/subgroups_operations/fft_tilde_h.comp | 122 +++++++++++++++++ 4 files changed, 257 insertions(+), 19 deletions(-) create mode 100644 shaders/subgroups_operations/fft_tilde_h.comp diff --git a/samples/extensions/subgroups_operations/CMakeLists.txt b/samples/extensions/subgroups_operations/CMakeLists.txt index 417b2d041..7c89e8da2 100644 --- a/samples/extensions/subgroups_operations/CMakeLists.txt +++ b/samples/extensions/subgroups_operations/CMakeLists.txt @@ -29,4 +29,5 @@ add_sample( "subgroups_operations/ocean.vert" "subgroups_operations/ocean.frag" "subgroups_operations/ocean_fft.comp" - "subgroups_operations/butterfly_precomp.comp") + "subgroups_operations/butterfly_precomp.comp" + "subgroups_operations/fft_tilde_h.comp") diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 96930f78c..b1aebecdd 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -68,6 +68,9 @@ SubgroupsOperations::~SubgroupsOperations() { if (device) { + precompute.pipeline.destroy(get_device().get_handle()); + tildas.pipeline.destroy(get_device().get_handle()); + compute.pipelines._default.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); @@ -100,7 +103,8 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) create_descriptor_set(); create_pipelines(); - create_butterfly_texture(); + create_tildas(); + //create_butterfly_texture(); build_command_buffers(); @@ -120,11 +124,11 @@ void SubgroupsOperations::prepare_compute() create_compute_queue(); create_compute_command_pool(); create_compute_command_buffer(); - create_compute_descriptor_set_layout(); - create_compute_descriptor_set(); - preapre_compute_pipeline_layout(); - prepare_compute_pipeline(); - build_compute_command_buffer(); +// create_compute_descriptor_set_layout(); +// create_compute_descriptor_set(); +// preapre_compute_pipeline_layout(); +// prepare_compute_pipeline(); +// build_compute_command_buffer(); } void SubgroupsOperations::create_compute_queue() @@ -175,7 +179,7 @@ void SubgroupsOperations::create_compute_descriptor_set() VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1u); VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); - update_compute_descriptor(); + // update_compute_descriptor(); } void SubgroupsOperations::update_compute_descriptor() @@ -286,6 +290,94 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); } +void SubgroupsOperations::create_tildas() +{ + fft_buffers.fft_tilde_h_kt_dx = std::make_unique(); + fft_buffers.fft_tilde_h_kt_dy = std::make_unique(); + fft_buffers.fft_tilde_h_kt_dz = std::make_unique(); + + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dx); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dy); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dz); + + + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 3u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 4u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 5u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u) + + }; + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &tildas.descriptor_set_layout)); + + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &tildas.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &tildas.descriptor_set)); + + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &tildas.descriptor_set_layout; + + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &tildas.pipeline.pipeline_layout)); + + VkComputePipelineCreateInfo computeInfo = vkb::initializers::compute_pipeline_create_info(tildas.pipeline.pipeline_layout); + computeInfo.stage = load_shader("subgroups_operations/fft_tilde_h.comp", VK_SHADER_STAGE_COMPUTE_BIT); + + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &tildas.pipeline.pipeline)); + + // update descriptor set + + auto htilde_0_descriptor = create_descriptor(*fft_buffers.fft_input_htilde0); + auto htilde_conj_0_descriptor = create_descriptor(*fft_buffers.fft_input_htilde0_conj); + + VkDescriptorImageInfo image_dx_descriptor{}; + image_dx_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dx->view; + image_dx_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_dx_descriptor.sampler = nullptr; + + VkDescriptorImageInfo image_dy_descriptor{}; + image_dy_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dy->view; + image_dy_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_dy_descriptor.sampler = nullptr; + + VkDescriptorImageInfo image_dz_descriptor{}; + image_dz_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dz->view; + image_dz_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_dz_descriptor.sampler = nullptr; + + VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); + VkDescriptorBufferInfo fft_time_ubo_buffer = create_descriptor(*fft_time_ubo); + + std::vector write_descriptor_sets = { + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 0u, &htilde_0_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &htilde_conj_0_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_dx_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 3u, &image_dy_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &image_dz_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &fft_params_ubo_buffer), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 6u, &fft_time_ubo_buffer) + }; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); + + + + VkCommandBuffer command_buffer = compute.command_buffer; + + VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); + VK_CHECK(vkBeginCommandBuffer(command_buffer, &begin_info)); + + vkCmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline); + vkCmdBindDescriptorSets(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline_layout, 0u, 1u, &tildas.descriptor_set, 0u, nullptr); + + vkCmdDispatch(command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + + VK_CHECK(vkEndCommandBuffer(command_buffer)); +} + void SubgroupsOperations::load_assets() { generate_plane(); @@ -331,9 +423,9 @@ void SubgroupsOperations::load_assets() auto fft_input_weights_size = static_cast(weights.size() * sizeof(std::complex)); fft_buffers.fft_input_weight = std::make_unique(get_device(), fft_input_weights_size, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - fft_buffers.fft_input_htilde0->update(h_tilde_0.data(), h_tilde_0.size()); - fft_buffers.fft_input_htilde0_conj->update(h_tilde_0_conj.data(), h_tilde_0_conj.size()); - fft_buffers.fft_input_weight->update(weights.data(), weights.size()); + fft_buffers.fft_input_htilde0->convert_and_update(h_tilde_0.data()); + fft_buffers.fft_input_htilde0_conj->convert_and_update(h_tilde_0_conj.data()); + fft_buffers.fft_input_weight->convert_and_update(weights.data()); // fft_buffers.fft_height_map_image = std::make_unique(grid_size * grid_size); } @@ -398,6 +490,7 @@ void SubgroupsOperations::prepare_uniform_buffers() { camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); update_uniform_buffers(); } @@ -443,12 +536,12 @@ void SubgroupsOperations::create_semaphore() void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u), // FFTParametersUbo, CameraUbo, ComputeUbo - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 3u), - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 10u), // FFTParametersUbo, CameraUbo, ComputeUbo + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 10u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u) }; VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 3u); + vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 4u); VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } @@ -571,6 +664,11 @@ void SubgroupsOperations::update_uniform_buffers() fft_ubo.length = ui.length; fft_ubo.wind = ui.wind; fft_params_ubo->convert_and_update(fft_ubo); + + TimeUbo t; + t.time = 0.36541365841351354135135f; + + fft_time_ubo->convert_and_update(t); } void SubgroupsOperations::build_command_buffers() diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 4cd7a3d0d..ed260a906 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -61,6 +61,7 @@ class SubgroupsOperations : public ApiVulkanSample void create_descriptor_set(); void create_pipelines(); + void create_tildas(); void create_butterfly_texture(); void update_uniform_buffers(); @@ -102,17 +103,23 @@ class SubgroupsOperations : public ApiVulkanSample alignas(8) glm::vec2 wind; }; + struct TimeUbo + { + alignas(4) float time; + }; + struct GuiConfig { bool wireframe = {false}; - float amplitude = {0.005f}; - float length = {16.0f}; - glm::vec2 wind = {16.0f, 0.0f}; + float amplitude = {4.0f}; + float length = {1000.0f}; + glm::vec2 wind = {10.0f, 10.0f}; } ui; - uint32_t grid_size = {128u}; + uint32_t grid_size = {256u}; std::unique_ptr camera_ubo; std::unique_ptr fft_params_ubo; + std::unique_ptr fft_time_ubo; std::vector> h_tilde_0; std::vector> h_tilde_0_conj; @@ -144,6 +151,9 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr fft_input_htilde0_conj; std::unique_ptr fft_input_weight; std::unique_ptr fft_height_map_image; + std::unique_ptr fft_tilde_h_kt_dx; + std::unique_ptr fft_tilde_h_kt_dy; + std::unique_ptr fft_tilde_h_kt_dz; } fft_buffers; struct @@ -163,6 +173,13 @@ class SubgroupsOperations : public ApiVulkanSample } pipelines; } compute; + struct + { + VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; + Pipeline pipeline; + } tildas; + struct { VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; diff --git a/shaders/subgroups_operations/fft_tilde_h.comp b/shaders/subgroups_operations/fft_tilde_h.comp new file mode 100644 index 000000000..f05c41fdf --- /dev/null +++ b/shaders/subgroups_operations/fft_tilde_h.comp @@ -0,0 +1,122 @@ +#version 450 +#extension GL_KHR_shader_subgroup_basic : enable + +#define PI 3.141592f +#define GRAVITY 9.81f + +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; + +layout (std140, binding = 0) readonly buffer TildeH0K +{ + vec2 data[]; +} +tilde_h0_k; + +layout (std140, binding = 1) readonly buffer TildeH0MinusK +{ + vec2 data[]; +} +tilde_h0_minus_k; + +layout (binding = 2, rgba32f) writeonly uniform image2D tilde_h_kt_dx; +layout (binding = 3, rgba32f) writeonly uniform image2D tilde_h_kt_dy; +layout (binding = 4, rgba32f) writeonly uniform image2D tilde_h_kt_dz; + +layout (binding = 5) uniform FFTParametersUbo +{ + float amplitude; + float len; + uint grid_size; + vec2 wind; +} fftUbo; + +layout (binding = 6) uniform Time +{ + float time; +} t; + +struct Complex +{ + float real; + float imag; +}; + +Complex complex_add(Complex c1, Complex c2) +{ + Complex res; + res.real = c1.real + c2.real; + res.imag = c1.imag + c2.imag; + return res; +} + +Complex complex_multiply(Complex c1, Complex c2) +{ + Complex res; + res.real = c1.real * c2.real - c1.imag * c2.imag; + res.imag = c1.real * c2.imag + c1.imag * c2.real; + return res; +} + +Complex complex_conj(Complex c) +{ + Complex res; + res.real = c.real; + res.imag = -c.imag; + return res; +} + +void main() +{ + float L = fftUbo.len; + + vec2 uv = gl_GlobalInvocationID.xy; + + vec2 k = vec2( (2.0f * PI * uv.x) / L, (2.0f * PI * uv.y) / L); + + float k_mag = length(k); + if (k_mag < 0.00001f) k_mag = 0.00001f; + + float w = sqrt(GRAVITY * k_mag); + + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + + uint idx = pixel_pos.x + pixel_pos.y; + vec2 h0_k = tilde_h0_k.data[idx]; + + Complex amp = Complex(h0_k.x, h0_k.y); + Complex exp_iwt = Complex(cos(w * t.time), sin(w * t.time)); + + vec2 h0_minus_k = tilde_h0_minus_k.data[idx]; + + Complex amp_conj = Complex(h0_minus_k.x, h0_minus_k.y); + Complex exp_minus_iwt = Complex(cos(w * t.time), -sin(w * t.time)); + + + Complex h_k_t_dy = complex_add( complex_multiply(amp, exp_iwt), complex_multiply(amp_conj, exp_minus_iwt)); + imageStore(tilde_h_kt_dy, pixel_pos, vec4(h_k_t_dy.real, h_k_t_dy.imag, 0.0f, 1.0f)); + + Complex dx = Complex(0.0f, -k.x / k_mag); + Complex h_k_t_dx = complex_multiply(dx, h_k_t_dy); + imageStore(tilde_h_kt_dx, pixel_pos, vec4(h_k_t_dx.real, h_k_t_dx.imag, 0.0f, 1.0f)); + + Complex dz = Complex(0.0f, -k.y / k_mag); + Complex h_k_t_dz = complex_multiply(dz, h_k_t_dy); + imageStore(tilde_h_kt_dz, pixel_pos, vec4(h_k_t_dz.real, h_k_t_dz.imag, 0.0f, 1.0f)); +} From 047e7f53e8f9f59044bd89442f8346b8effc34be Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Tue, 26 Sep 2023 09:21:32 +0200 Subject: [PATCH 14/53] Refactor and add butterfly and tild texture generation --- .../subgroups_operations.cpp | 228 +++++++----------- .../subgroups_operations.h | 43 ++-- shaders/subgroups_operations/fft_tilde_h.comp | 24 +- 3 files changed, 113 insertions(+), 182 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index b1aebecdd..e4e36e1a6 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -68,8 +68,8 @@ SubgroupsOperations::~SubgroupsOperations() { if (device) { - precompute.pipeline.destroy(get_device().get_handle()); - tildas.pipeline.destroy(get_device().get_handle()); + precompute.pipeline.destroy(get_device().get_handle()); + tildas.pipeline.destroy(get_device().get_handle()); compute.pipelines._default.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); @@ -103,8 +103,9 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) create_descriptor_set(); create_pipelines(); - create_tildas(); - //create_butterfly_texture(); + create_tildas(); + create_butterfly_texture(); + build_compute_command_buffer(); build_command_buffers(); @@ -124,11 +125,6 @@ void SubgroupsOperations::prepare_compute() create_compute_queue(); create_compute_command_pool(); create_compute_command_buffer(); -// create_compute_descriptor_set_layout(); -// create_compute_descriptor_set(); -// preapre_compute_pipeline_layout(); -// prepare_compute_pipeline(); -// build_compute_command_buffer(); } void SubgroupsOperations::create_compute_queue() @@ -161,27 +157,6 @@ void SubgroupsOperations::create_compute_command_buffer() VK_CHECK(vkCreateSemaphore(get_device().get_handle(), &semaphore_create_info, nullptr, &compute.semaphore)); } -void SubgroupsOperations::create_compute_descriptor_set_layout() -{ - std::vector set_layout_bindngs = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 3u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, VK_SHADER_STAGE_COMPUTE_BIT, 4u)}; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); - - VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &compute.descriptor_set_layout)); -} - -void SubgroupsOperations::create_compute_descriptor_set() -{ - VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &compute.descriptor_set_layout, 1u); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &compute.descriptor_set)); - - // update_compute_descriptor(); -} - void SubgroupsOperations::update_compute_descriptor() { VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); @@ -201,26 +176,6 @@ void SubgroupsOperations::update_compute_descriptor() vkUpdateDescriptorSets(get_device().get_handle(), static_cast(wirte_descriptor_sets.size()), wirte_descriptor_sets.data(), 0u, nullptr); } -void SubgroupsOperations::preapre_compute_pipeline_layout() -{ - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; - compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - compute_pipeline_layout_info.setLayoutCount = 1u; - compute_pipeline_layout_info.pSetLayouts = &compute.descriptor_set_layout; - - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &compute.pipelines._default.pipeline_layout)); -} - -void SubgroupsOperations::prepare_compute_pipeline() -{ - VkComputePipelineCreateInfo computeInfo = {}; - computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; - computeInfo.layout = compute.pipelines._default.pipeline_layout; - computeInfo.stage = load_shader("subgroups_operations/ocean_fft.comp", VK_SHADER_STAGE_COMPUTE_BIT); - - VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &compute.pipelines._default.pipeline)); -} - void SubgroupsOperations::build_compute_command_buffer() { // record command @@ -240,12 +195,20 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); } - // fft calculation + // buttle fly texture + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline_layout, 0u, 1u, &precompute.descriptor_set, 0u, nullptr); + + vkCmdDispatch(compute.command_buffer, 8u, DISPLACEMENT_MAP_DIM, 1u); + } + + // tildas textures { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute.pipelines._default.pipeline_layout, 0u, 1u, &compute.descriptor_set, 0u, nullptr); + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline_layout, 0u, 1u, &tildas.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, 32u, 32u, 1u); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); } if (compute.queue_family_index != ocean.graphics_queue_family_index) @@ -292,90 +255,74 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) void SubgroupsOperations::create_tildas() { - fft_buffers.fft_tilde_h_kt_dx = std::make_unique(); - fft_buffers.fft_tilde_h_kt_dy = std::make_unique(); - fft_buffers.fft_tilde_h_kt_dz = std::make_unique(); - - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dx); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dy); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dz); - - - std::vector set_layout_bindngs = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 3u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 4u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 5u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u) - - }; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); - VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &tildas.descriptor_set_layout)); + fft_buffers.fft_tilde_h_kt_dx = std::make_unique(); + fft_buffers.fft_tilde_h_kt_dy = std::make_unique(); + fft_buffers.fft_tilde_h_kt_dz = std::make_unique(); - VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &tildas.descriptor_set_layout, 1u); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &tildas.descriptor_set)); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dx); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dy); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dz); - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; - compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - compute_pipeline_layout_info.setLayoutCount = 1u; - compute_pipeline_layout_info.pSetLayouts = &tildas.descriptor_set_layout; - - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &tildas.pipeline.pipeline_layout)); - - VkComputePipelineCreateInfo computeInfo = vkb::initializers::compute_pipeline_create_info(tildas.pipeline.pipeline_layout); - computeInfo.stage = load_shader("subgroups_operations/fft_tilde_h.comp", VK_SHADER_STAGE_COMPUTE_BIT); - - VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &tildas.pipeline.pipeline)); - - // update descriptor set + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 3u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 4u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 5u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u) - auto htilde_0_descriptor = create_descriptor(*fft_buffers.fft_input_htilde0); - auto htilde_conj_0_descriptor = create_descriptor(*fft_buffers.fft_input_htilde0_conj); + }; + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &tildas.descriptor_set_layout)); - VkDescriptorImageInfo image_dx_descriptor{}; - image_dx_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dx->view; - image_dx_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_dx_descriptor.sampler = nullptr; + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &tildas.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &tildas.descriptor_set)); - VkDescriptorImageInfo image_dy_descriptor{}; - image_dy_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dy->view; - image_dy_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_dy_descriptor.sampler = nullptr; + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &tildas.descriptor_set_layout; - VkDescriptorImageInfo image_dz_descriptor{}; - image_dz_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dz->view; - image_dz_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_dz_descriptor.sampler = nullptr; + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &tildas.pipeline.pipeline_layout)); - VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); - VkDescriptorBufferInfo fft_time_ubo_buffer = create_descriptor(*fft_time_ubo); + VkComputePipelineCreateInfo computeInfo = vkb::initializers::compute_pipeline_create_info(tildas.pipeline.pipeline_layout); + computeInfo.stage = load_shader("subgroups_operations/fft_tilde_h.comp", VK_SHADER_STAGE_COMPUTE_BIT); - std::vector write_descriptor_sets = { - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 0u, &htilde_0_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &htilde_conj_0_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_dx_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 3u, &image_dy_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &image_dz_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &fft_params_ubo_buffer), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 6u, &fft_time_ubo_buffer) - }; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &tildas.pipeline.pipeline)); + // update descriptor set + VkDescriptorBufferInfo htilde_0_descriptor = create_descriptor(*fft_buffers.fft_input_htilde0); + VkDescriptorBufferInfo htilde_conj_0_descriptor = create_descriptor(*fft_buffers.fft_input_htilde0_conj); - VkCommandBuffer command_buffer = compute.command_buffer; + VkDescriptorImageInfo image_dx_descriptor{}; + image_dx_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dx->view; + image_dx_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_dx_descriptor.sampler = nullptr; - VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); - VK_CHECK(vkBeginCommandBuffer(command_buffer, &begin_info)); + VkDescriptorImageInfo image_dy_descriptor{}; + image_dy_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dy->view; + image_dy_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_dy_descriptor.sampler = nullptr; - vkCmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline); - vkCmdBindDescriptorSets(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline_layout, 0u, 1u, &tildas.descriptor_set, 0u, nullptr); + VkDescriptorImageInfo image_dz_descriptor{}; + image_dz_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dz->view; + image_dz_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_dz_descriptor.sampler = nullptr; - vkCmdDispatch(command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); + VkDescriptorBufferInfo fft_time_ubo_buffer = create_descriptor(*fft_time_ubo); - VK_CHECK(vkEndCommandBuffer(command_buffer)); + std::vector write_descriptor_sets = { + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 0u, &htilde_0_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &htilde_conj_0_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_dx_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 3u, &image_dy_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &image_dz_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &fft_params_ubo_buffer), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 6u, &fft_time_ubo_buffer)}; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } void SubgroupsOperations::load_assets() @@ -423,9 +370,9 @@ void SubgroupsOperations::load_assets() auto fft_input_weights_size = static_cast(weights.size() * sizeof(std::complex)); fft_buffers.fft_input_weight = std::make_unique(get_device(), fft_input_weights_size, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - fft_buffers.fft_input_htilde0->convert_and_update(h_tilde_0.data()); - fft_buffers.fft_input_htilde0_conj->convert_and_update(h_tilde_0_conj.data()); - fft_buffers.fft_input_weight->convert_and_update(weights.data()); + fft_buffers.fft_input_htilde0->update(h_tilde_0.data(), fft_input_htilde0_size); + fft_buffers.fft_input_htilde0_conj->update(h_tilde_0_conj.data(), fft_input_htilde0_conj_size); + fft_buffers.fft_input_weight->update(weights.data(), fft_input_weights_size); // fft_buffers.fft_height_map_image = std::make_unique(grid_size * grid_size); } @@ -436,7 +383,7 @@ float SubgroupsOperations::phillips_spectrum(int32_t n, int32_t m) float k_len = glm::length(k); if (k_len < 0.000001f) - return 0.0f; + return 0.000001f; float k_len2 = k_len * k_len; float k_len4 = k_len2 * k_len2; @@ -490,7 +437,7 @@ void SubgroupsOperations::prepare_uniform_buffers() { camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); update_uniform_buffers(); } @@ -536,12 +483,11 @@ void SubgroupsOperations::create_semaphore() void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 10u), // FFTParametersUbo, CameraUbo, ComputeUbo - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 10u), - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u) - }; + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 10u), // FFTParametersUbo, CameraUbo, ComputeUbo + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 10u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u)}; VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 4u); + vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 4u); VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } @@ -665,10 +611,10 @@ void SubgroupsOperations::update_uniform_buffers() fft_ubo.wind = ui.wind; fft_params_ubo->convert_and_update(fft_ubo); - TimeUbo t; - t.time = 0.36541365841351354135135f; + TimeUbo t; + t.time = 92340.36541365841351354135135f; - fft_time_ubo->convert_and_update(t); + fft_time_ubo->convert_and_update(t); } void SubgroupsOperations::build_command_buffers() @@ -941,14 +887,4 @@ void SubgroupsOperations::create_butterfly_texture() vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor), vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &bit_reverse_descriptor)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); - - VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); - VK_CHECK(vkBeginCommandBuffer(command_buffer, &begin_info)); - - vkCmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline); - vkCmdBindDescriptorSets(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline_layout, 0u, 1u, &precompute.descriptor_set, 0u, nullptr); - - vkCmdDispatch(command_buffer, 32u, 32u, 1u); - - VK_CHECK(vkEndCommandBuffer(command_buffer)); } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index ed260a906..bcb38e5f3 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -45,11 +45,6 @@ class SubgroupsOperations : public ApiVulkanSample void create_compute_queue(); void create_compute_command_pool(); void create_compute_command_buffer(); - void create_compute_descriptor_set_layout(); - void create_compute_descriptor_set(); - - void preapre_compute_pipeline_layout(); - void prepare_compute_pipeline(); void build_compute_command_buffer(); @@ -61,7 +56,7 @@ class SubgroupsOperations : public ApiVulkanSample void create_descriptor_set(); void create_pipelines(); - void create_tildas(); + void create_tildas(); void create_butterfly_texture(); void update_uniform_buffers(); @@ -103,23 +98,23 @@ class SubgroupsOperations : public ApiVulkanSample alignas(8) glm::vec2 wind; }; - struct TimeUbo - { - alignas(4) float time; - }; + struct TimeUbo + { + alignas(4) float time; + }; struct GuiConfig { bool wireframe = {false}; - float amplitude = {4.0f}; - float length = {1000.0f}; - glm::vec2 wind = {10.0f, 10.0f}; + float amplitude = {5.0f}; + float length = {1000.0f}; + glm::vec2 wind = {100.0f, 0.0f}; } ui; - uint32_t grid_size = {256u}; + uint32_t grid_size = {256u}; std::unique_ptr camera_ubo; std::unique_ptr fft_params_ubo; - std::unique_ptr fft_time_ubo; + std::unique_ptr fft_time_ubo; std::vector> h_tilde_0; std::vector> h_tilde_0_conj; @@ -151,9 +146,9 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr fft_input_htilde0_conj; std::unique_ptr fft_input_weight; std::unique_ptr fft_height_map_image; - std::unique_ptr fft_tilde_h_kt_dx; - std::unique_ptr fft_tilde_h_kt_dy; - std::unique_ptr fft_tilde_h_kt_dz; + std::unique_ptr fft_tilde_h_kt_dx; + std::unique_ptr fft_tilde_h_kt_dy; + std::unique_ptr fft_tilde_h_kt_dz; } fft_buffers; struct @@ -173,12 +168,12 @@ class SubgroupsOperations : public ApiVulkanSample } pipelines; } compute; - struct - { - VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; - VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; - Pipeline pipeline; - } tildas; + struct + { + VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; + Pipeline pipeline; + } tildas; struct { diff --git a/shaders/subgroups_operations/fft_tilde_h.comp b/shaders/subgroups_operations/fft_tilde_h.comp index f05c41fdf..d29a207d6 100644 --- a/shaders/subgroups_operations/fft_tilde_h.comp +++ b/shaders/subgroups_operations/fft_tilde_h.comp @@ -25,13 +25,13 @@ layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; layout (std140, binding = 0) readonly buffer TildeH0K { - vec2 data[]; + vec2 data[]; } tilde_h0_k; layout (std140, binding = 1) readonly buffer TildeH0MinusK { - vec2 data[]; + vec2 data[]; } tilde_h0_minus_k; @@ -84,9 +84,9 @@ Complex complex_conj(Complex c) void main() { - float L = fftUbo.len; + float L = fftUbo.len; - vec2 uv = gl_GlobalInvocationID.xy; + vec2 uv = vec2(gl_GlobalInvocationID.xy) - (fftUbo.grid_size / 2.0f); vec2 k = vec2( (2.0f * PI * uv.x) / L, (2.0f * PI * uv.y) / L); @@ -97,26 +97,26 @@ void main() ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - uint idx = pixel_pos.x + pixel_pos.y; - vec2 h0_k = tilde_h0_k.data[idx]; + uint idx = pixel_pos.x + pixel_pos.y; + vec2 h0_k = tilde_h0_k.data[idx]; Complex amp = Complex(h0_k.x, h0_k.y); - Complex exp_iwt = Complex(cos(w * t.time), sin(w * t.time)); + Complex exp_iwt = Complex(cos(w * t.time), sin(w * t.time)); - vec2 h0_minus_k = tilde_h0_minus_k.data[idx]; + vec2 h0_minus_k = tilde_h0_minus_k.data[idx]; Complex amp_conj = Complex(h0_minus_k.x, h0_minus_k.y); - Complex exp_minus_iwt = Complex(cos(w * t.time), -sin(w * t.time)); + Complex exp_minus_iwt = Complex(cos(w * t.time), -sin(w * t.time)); Complex h_k_t_dy = complex_add( complex_multiply(amp, exp_iwt), complex_multiply(amp_conj, exp_minus_iwt)); - imageStore(tilde_h_kt_dy, pixel_pos, vec4(h_k_t_dy.real, h_k_t_dy.imag, 0.0f, 1.0f)); + imageStore(tilde_h_kt_dy, pixel_pos, vec4(h_k_t_dy.real, h_k_t_dy.imag, 0.0f, 1.0f)); Complex dx = Complex(0.0f, -k.x / k_mag); Complex h_k_t_dx = complex_multiply(dx, h_k_t_dy); - imageStore(tilde_h_kt_dx, pixel_pos, vec4(h_k_t_dx.real, h_k_t_dx.imag, 0.0f, 1.0f)); + imageStore(tilde_h_kt_dx, pixel_pos, vec4(h_k_t_dx.real, h_k_t_dx.imag, 0.0f, 1.0f)); Complex dz = Complex(0.0f, -k.y / k_mag); Complex h_k_t_dz = complex_multiply(dz, h_k_t_dy); - imageStore(tilde_h_kt_dz, pixel_pos, vec4(h_k_t_dz.real, h_k_t_dz.imag, 0.0f, 1.0f)); + imageStore(tilde_h_kt_dz, pixel_pos, vec4(h_k_t_dz.real, h_k_t_dz.imag, 0.0f, 1.0f)); } From 9c311648a5d8b8bfcf703868b0af0bda24d8c9c8 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Tue, 26 Sep 2023 10:00:11 +0200 Subject: [PATCH 15/53] Some fixes --- .../subgroups_operations.cpp | 18 +++++------------- .../subgroups_operations.h | 8 +++++--- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index e4e36e1a6..fd9fb7a68 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -21,7 +21,6 @@ #include #define GRAVITY 9.81f -#define DISPLACEMENT_MAP_DIM 256 void SubgroupsOperations::Pipeline::destroy(VkDevice device) { @@ -328,8 +327,7 @@ void SubgroupsOperations::create_tildas() void SubgroupsOperations::load_assets() { generate_plane(); - - log_2_N = log(DISPLACEMENT_MAP_DIM) / log(2); + log_2_N = glm::log(grid_size) / glm::log(2.0f); // generate fft inputs h_tilde_0.clear(); @@ -344,7 +342,6 @@ void SubgroupsOperations::load_assets() } // calculate weights - auto log_2_N = glm::log(grid_size) / glm::log(2.0f); uint32_t pow2 = 1U; for (uint32_t i = 0U; i < log_2_N; ++i) { @@ -373,8 +370,6 @@ void SubgroupsOperations::load_assets() fft_buffers.fft_input_htilde0->update(h_tilde_0.data(), fft_input_htilde0_size); fft_buffers.fft_input_htilde0_conj->update(h_tilde_0_conj.data(), fft_input_htilde0_conj_size); fft_buffers.fft_input_weight->update(weights.data(), fft_input_weights_size); - - // fft_buffers.fft_height_map_image = std::make_unique(grid_size * grid_size); } float SubgroupsOperations::phillips_spectrum(int32_t n, int32_t m) @@ -423,14 +418,14 @@ std::complex SubgroupsOperations::rndGaussian() x2 = 2.0f * rndVal() - 1.0f; w = x1 * x1 + x2 * x2; } while (w >= 1.0f); - w = std::sqrt((-2.0f * std::log(w)) / w); + w = glm::sqrt((-2.0f * glm::log(w)) / w); return {x1 * w, x2 * w}; } std::complex SubgroupsOperations::hTilde_0(uint32_t n, uint32_t m) { std::complex rnd = rndGaussian(); - return rnd * std::sqrt(phillips_spectrum(n, m) / 2.0f); + return rnd * glm::sqrt(phillips_spectrum(n, m) / 2.0f); } void SubgroupsOperations::prepare_uniform_buffers() @@ -611,10 +606,7 @@ void SubgroupsOperations::update_uniform_buffers() fft_ubo.wind = ui.wind; fft_params_ubo->convert_and_update(fft_ubo); - TimeUbo t; - t.time = 92340.36541365841351354135135f; - - fft_time_ubo->convert_and_update(t); + fft_time_ubo->convert_and_update(fftTime); } void SubgroupsOperations::build_command_buffers() @@ -778,7 +770,7 @@ void SubgroupsOperations::render(float delta_time) { return; } - + fftTime.time = delta_time; update_uniform_buffers(); draw(); } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index bcb38e5f3..26d44cd4f 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -20,6 +20,8 @@ #include "api_vulkan_sample.h" #include +#define DISPLACEMENT_MAP_DIM 256u + class SubgroupsOperations : public ApiVulkanSample { struct OceanVertex @@ -100,8 +102,8 @@ class SubgroupsOperations : public ApiVulkanSample struct TimeUbo { - alignas(4) float time; - }; + alignas(4) float time = {0.0f}; + } fftTime; struct GuiConfig { @@ -111,7 +113,7 @@ class SubgroupsOperations : public ApiVulkanSample glm::vec2 wind = {100.0f, 0.0f}; } ui; - uint32_t grid_size = {256u}; + uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; std::unique_ptr camera_ubo; std::unique_ptr fft_params_ubo; std::unique_ptr fft_time_ubo; From 2ab5a23dff6c4961f3cf2c9f7d85a232388d4d07 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Tue, 26 Sep 2023 11:42:11 +0200 Subject: [PATCH 16/53] Fix access to input buffers --- .../subgroups_operations/subgroups_operations.cpp | 6 +++--- shaders/subgroups_operations/fft_tilde_h.comp | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index fd9fb7a68..81321c7d2 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -332,9 +332,9 @@ void SubgroupsOperations::load_assets() // generate fft inputs h_tilde_0.clear(); h_tilde_0_conj.clear(); - for (uint32_t m = 0; m < grid_size + 1U; ++m) + for (uint32_t m = 0; m < grid_size; ++m) { - for (uint32_t n = 0; n < grid_size + 1U; ++n) + for (uint32_t n = 0; n < grid_size; ++n) { h_tilde_0.push_back(hTilde_0(n, m)); h_tilde_0_conj.push_back(std::conj(hTilde_0(-n, -m))); @@ -342,7 +342,7 @@ void SubgroupsOperations::load_assets() } // calculate weights - uint32_t pow2 = 1U; + uint32_t pow2 = 1U; for (uint32_t i = 0U; i < log_2_N; ++i) { for (uint32_t j = 0U; j < pow2; ++j) diff --git a/shaders/subgroups_operations/fft_tilde_h.comp b/shaders/subgroups_operations/fft_tilde_h.comp index d29a207d6..826c4bf20 100644 --- a/shaders/subgroups_operations/fft_tilde_h.comp +++ b/shaders/subgroups_operations/fft_tilde_h.comp @@ -96,8 +96,9 @@ void main() float w = sqrt(GRAVITY * k_mag); ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + uvec2 uv_idx = uvec2(gl_GlobalInvocationID.xy * 0.5f); - uint idx = pixel_pos.x + pixel_pos.y; + uint idx = uv_idx.x * uv_idx.y; vec2 h0_k = tilde_h0_k.data[idx]; Complex amp = Complex(h0_k.x, h0_k.y); From 2e26f5c0b7c3ee1a20266769baa606a4a1c1e7eb Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Thu, 28 Sep 2023 13:57:56 +0200 Subject: [PATCH 17/53] Add fft pipeline for rows --- .../subgroups_operations/CMakeLists.txt | 3 +- .../subgroups_operations.cpp | 217 ++++++++++++++---- .../subgroups_operations.h | 35 ++- shaders/subgroups_operations/fft.comp | 139 +++++++++++ 4 files changed, 344 insertions(+), 50 deletions(-) create mode 100644 shaders/subgroups_operations/fft.comp diff --git a/samples/extensions/subgroups_operations/CMakeLists.txt b/samples/extensions/subgroups_operations/CMakeLists.txt index 7c89e8da2..1bfa3f3bb 100644 --- a/samples/extensions/subgroups_operations/CMakeLists.txt +++ b/samples/extensions/subgroups_operations/CMakeLists.txt @@ -30,4 +30,5 @@ add_sample( "subgroups_operations/ocean.frag" "subgroups_operations/ocean_fft.comp" "subgroups_operations/butterfly_precomp.comp" - "subgroups_operations/fft_tilde_h.comp") + "subgroups_operations/fft_tilde_h.comp" + "subgroups_operations/fft.comp") diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 81321c7d2..44ddd9456 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -67,11 +67,22 @@ SubgroupsOperations::~SubgroupsOperations() { if (device) { + fft_buffers.fft_tilde_h_kt_dx->destroy(get_device().get_handle()); + fft_buffers.fft_tilde_h_kt_dy->destroy(get_device().get_handle()); + fft_buffers.fft_tilde_h_kt_dz->destroy(get_device().get_handle()); + butterfly_precomp.destroy(get_device().get_handle()); + precompute.pipeline.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), precompute.descriptor_set_layout, nullptr); + tildas.pipeline.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), tildas.descriptor_set_layout, nullptr); + + + fft.pipelines.horizontal.destroy(get_device().get_handle()); + fft.pipelines.vertical.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), fft.descriptor_set_layout, nullptr); - compute.pipelines._default.destroy(get_device().get_handle()); - vkDestroyDescriptorSetLayout(get_device().get_handle(), compute.descriptor_set_layout, nullptr); vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); @@ -104,6 +115,8 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) create_tildas(); create_butterfly_texture(); + create_fft(); + build_compute_command_buffer(); build_command_buffers(); @@ -156,25 +169,6 @@ void SubgroupsOperations::create_compute_command_buffer() VK_CHECK(vkCreateSemaphore(get_device().get_handle(), &semaphore_create_info, nullptr, &compute.semaphore)); } -void SubgroupsOperations::update_compute_descriptor() -{ - VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); - VkDescriptorBufferInfo fft_input_h_tilde0_data_buffer = create_descriptor(*fft_buffers.fft_input_htilde0); - VkDescriptorBufferInfo fft_input_h_tilde0_conj_data_buffer = create_descriptor(*fft_buffers.fft_input_htilde0_conj); - VkDescriptorBufferInfo fft_input_weight_data_buffer = create_descriptor(*fft_buffers.fft_input_weight); - // VkDescriptorBufferInfo fft_input_image = create_descriptor(*fft_buffers.fft_height_map_image->get_vk_image()); - - std::vector wirte_descriptor_sets = { - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &fft_params_ubo_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &fft_input_h_tilde0_data_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &fft_input_h_tilde0_conj_data_buffer), - vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 3u, &fft_input_weight_data_buffer), - // vkb::initializers::write_descriptor_set(compute.descriptor_set, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 4u, &fft_input_image) - - }; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(wirte_descriptor_sets.size()), wirte_descriptor_sets.data(), 0u, nullptr); -} - void SubgroupsOperations::build_compute_command_buffer() { // record command @@ -209,6 +203,38 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); } + auto calculateFft = [this](){ + // update ubo + for (uint32_t i = 0; i < log_2_N; ++i) + { + FFTPage nrPage; + nrPage.page = i; + fft_page_ubo->convert_and_update(nrPage); // upload information about nr of page + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + }; + + // fft horizontal; for Y axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); + calculateFft(); + } + + // fft horizontal; for X axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); + calculateFft(); + } + + // fft horizontal; for Z axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); + calculateFft(); + } + if (compute.queue_family_index != ocean.graphics_queue_family_index) { @@ -258,9 +284,9 @@ void SubgroupsOperations::create_tildas() fft_buffers.fft_tilde_h_kt_dy = std::make_unique(); fft_buffers.fft_tilde_h_kt_dz = std::make_unique(); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dx); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dy); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, DISPLACEMENT_MAP_DIM, DISPLACEMENT_MAP_DIM, *fft_buffers.fft_tilde_h_kt_dz); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dx); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dy); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dz); std::vector set_layout_bindngs = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), @@ -433,7 +459,11 @@ void SubgroupsOperations::prepare_uniform_buffers() camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - + fft_page_ubo = std::make_unique(get_device(), sizeof(FFTPage), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + // tmp + FFTPage s; + s.page = 0; + fft_page_ubo->convert_and_update(s); update_uniform_buffers(); } @@ -478,11 +508,11 @@ void SubgroupsOperations::create_semaphore() void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 10u), // FFTParametersUbo, CameraUbo, ComputeUbo - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 10u), - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u)}; + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 20u), // FFTParametersUbo, CameraUbo, ComputeUbo + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 20u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 20u)}; VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 4u); + vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 10u); VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } @@ -790,9 +820,9 @@ void SubgroupsOperations::createFBAttachement(VkFormat format, uint32_t width, u image.format = format; image.extent.width = width; image.extent.height = height; - image.extent.depth = 1; - image.mipLevels = 1; - image.arrayLayers = 1; + image.extent.depth = 1u; + image.mipLevels = 1u; + image.arrayLayers = 1u; image.samples = VK_SAMPLE_COUNT_1_BIT; image.tiling = VK_IMAGE_TILING_OPTIMAL; image.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT | VK_IMAGE_USAGE_STORAGE_BIT; @@ -812,10 +842,10 @@ void SubgroupsOperations::createFBAttachement(VkFormat format, uint32_t width, u image_view_create_info.format = format; image_view_create_info.subresourceRange = {}; image_view_create_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - image_view_create_info.subresourceRange.baseMipLevel = 0; - image_view_create_info.subresourceRange.levelCount = 1; - image_view_create_info.subresourceRange.baseArrayLayer = 0; - image_view_create_info.subresourceRange.layerCount = 1; + image_view_create_info.subresourceRange.baseMipLevel = 0u; + image_view_create_info.subresourceRange.levelCount = 1u; + image_view_create_info.subresourceRange.baseArrayLayer = 0u; + image_view_create_info.subresourceRange.layerCount = 1u; image_view_create_info.image = attachment.image; VK_CHECK(vkCreateImageView(get_device().get_handle(), &image_view_create_info, nullptr, &attachment.view)); } @@ -833,9 +863,6 @@ uint32_t SubgroupsOperations::reverse(uint32_t i) void SubgroupsOperations::create_butterfly_texture() { - VkCommandPool commandPool = compute.command_pool; - VkCommandBuffer command_buffer = compute.command_buffer; - std::vector set_layout_bindngs = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u)}; @@ -880,3 +907,115 @@ void SubgroupsOperations::create_butterfly_texture() vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &bit_reverse_descriptor)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } + +void SubgroupsOperations::create_fft() +{ + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 3u) + }; + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft.descriptor_set_layout)); + + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_y)); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_x)); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_z)); + + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &fft.descriptor_set_layout; + + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &fft.pipelines.horizontal.pipeline_layout)); + + VkComputePipelineCreateInfo computeInfo = {}; + computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + computeInfo.layout = fft.pipelines.horizontal.pipeline_layout; + computeInfo.stage = load_shader("subgroups_operations/fft.comp", VK_SHADER_STAGE_COMPUTE_BIT); + + std::array specialization_map_entries; + VkSpecializationInfo spec_info; + uint32_t direction = 0; + specialization_map_entries[0] = vkb::initializers::specialization_map_entry(0, 0, sizeof(uint32_t)); + spec_info = vkb::initializers::specialization_info(static_cast(specialization_map_entries.size()), + specialization_map_entries.data(), + sizeof(uint32_t), + &direction); + computeInfo.stage.pSpecializationInfo = &spec_info; + + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft.pipelines.horizontal.pipeline)); + + fft.tilde_axis_y = std::make_unique(); + fft.tilde_axis_x = std::make_unique(); + fft.tilde_axis_z = std::make_unique(); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_y); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_x); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_z); + + VkDescriptorImageInfo image_descriptor_battlefly{}; + image_descriptor_battlefly.imageView = butterfly_precomp.view; + image_descriptor_battlefly.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_battlefly.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_tilda_y{}; + image_descriptor_tilda_y.imageView = fft_buffers.fft_tilde_h_kt_dy->view; + image_descriptor_tilda_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilda_y.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_tilde_axis_y{}; + image_descriptor_tilde_axis_y.imageView = fft.tilde_axis_y->view; + image_descriptor_tilde_axis_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilde_axis_y.sampler = nullptr; + + auto fft_page_descriptor = create_descriptor(*fft_page_ubo); + + std::vector write_descriptor_sets_asix_y = { + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_y), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_y), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_y.size()), write_descriptor_sets_asix_y.data(), 0u, nullptr); + + + VkDescriptorImageInfo image_descriptor_tilda_x{}; + image_descriptor_tilda_x.imageView = fft_buffers.fft_tilde_h_kt_dx->view; + image_descriptor_tilda_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilda_x.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_tilde_axis_x{}; + image_descriptor_tilde_axis_x.imageView = fft.tilde_axis_x->view; + image_descriptor_tilde_axis_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilde_axis_x.sampler = nullptr; + + std::vector write_descriptor_sets_asix_x = { + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_x), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_x), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_x.size()), write_descriptor_sets_asix_x.data(), 0u, nullptr); + + + VkDescriptorImageInfo image_descriptor_tilda_z{}; + image_descriptor_tilda_z.imageView = fft_buffers.fft_tilde_h_kt_dz->view; + image_descriptor_tilda_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilda_z.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_tilde_axis_z{}; + image_descriptor_tilde_axis_z.imageView = fft.tilde_axis_z->view; + image_descriptor_tilde_axis_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilde_axis_z.sampler = nullptr; + + std::vector write_descriptor_sets_asix_z = { + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_z), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_z), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_z.size()), write_descriptor_sets_asix_z.data(), 0u, nullptr); + +} diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 26d44cd4f..a806b8e12 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -60,9 +60,9 @@ class SubgroupsOperations : public ApiVulkanSample void create_tildas(); void create_butterfly_texture(); + void create_fft(); void update_uniform_buffers(); - void update_compute_descriptor(); // ocean stuff float phillips_spectrum(int32_t n, int32_t m); @@ -100,6 +100,11 @@ class SubgroupsOperations : public ApiVulkanSample alignas(8) glm::vec2 wind; }; + struct FFTPage + { + alignas(4) int32_t page = {0}; + }; + struct TimeUbo { alignas(4) float time = {0.0f}; @@ -117,6 +122,8 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr camera_ubo; std::unique_ptr fft_params_ubo; std::unique_ptr fft_time_ubo; + std::unique_ptr fft_page_ubo; + std::vector> h_tilde_0; std::vector> h_tilde_0_conj; @@ -147,7 +154,6 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr fft_input_htilde0; std::unique_ptr fft_input_htilde0_conj; std::unique_ptr fft_input_weight; - std::unique_ptr fft_height_map_image; std::unique_ptr fft_tilde_h_kt_dx; std::unique_ptr fft_tilde_h_kt_dy; std::unique_ptr fft_tilde_h_kt_dz; @@ -159,16 +165,25 @@ class SubgroupsOperations : public ApiVulkanSample VkCommandPool command_pool = {VK_NULL_HANDLE}; VkCommandBuffer command_buffer = {VK_NULL_HANDLE}; VkSemaphore semaphore = {VK_NULL_HANDLE}; - VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; - VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; - uint32_t queue_family_index = {-1u}; + uint32_t queue_family_index = {-1u}; + } compute; - struct - { - Pipeline _default; - } pipelines; - } compute; + struct + { + VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set_axis_y = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set_axis_x = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set_axis_z = {VK_NULL_HANDLE}; + struct + { + Pipeline horizontal; + Pipeline vertical; + } pipelines; + std::unique_ptr tilde_axis_y; + std::unique_ptr tilde_axis_x; + std::unique_ptr tilde_axis_z; + } fft; struct { diff --git a/shaders/subgroups_operations/fft.comp b/shaders/subgroups_operations/fft.comp new file mode 100644 index 000000000..1b8238200 --- /dev/null +++ b/shaders/subgroups_operations/fft.comp @@ -0,0 +1,139 @@ +#version 450 +#extension GL_KHR_shader_subgroup_basic : enable + +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; + +layout (constant_id = 0) const int direction = 0; + +layout (binding = 0, rgba32f) readonly uniform image2D u_butterfly_precomp; +layout (binding = 1, rgba32f) uniform image2D u_pingpong0; +layout (binding = 2, rgba32f) uniform image2D u_pingpong1; + +layout (binding = 3) uniform FftPage +{ + int page; +} fftUbo; + +struct Complex +{ + float real; + float imag; +}; + +Complex complex_add(Complex c1, Complex c2) +{ + Complex res; + res.real = c1.real + c2.real; + res.imag = c1.imag + c2.imag; + return res; +} + +Complex complex_multiply(Complex c1, Complex c2) +{ + Complex res; + res.real = c1.real * c2.real - c1.imag * c2.imag; + res.imag = c1.real * c2.imag + c1.imag * c2.real; + return res; +} + +Complex complex_conj(Complex c) +{ + Complex res; + res.real = c.real; + res.imag = -c.imag; + return res; +} + + +void HorizontalButterflies(in ivec2 pixel_pos) +{ + vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(fftUbo.page, pixel_pos.x)); + bool type = bool(fftUbo.page % 2); + if (type == false) + { + vec2 a_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; + vec2 b_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + + imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + } + else + { + vec2 a_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; + vec2 b_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + + imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + } +} + +void VerticalButterfiles(in ivec2 pixel_pos) +{ + vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(fftUbo.page, pixel_pos.y)); + bool type = bool(fftUbo.page % 2); + + if (type == false) + { + vec2 a_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; + vec2 b_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + + imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + } + else + { + vec2 a_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; + vec2 b_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + + imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + } +} + +void main() +{ + ivec2 uv = ivec2(gl_GlobalInvocationID.xy); + if (direction == 0) HorizontalButterflies(uv); + else if (direction == 1) VerticalButterfiles(uv); +} From cf16c81f1aa1da00148c7ace015a666d20df5f5a Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Fri, 29 Sep 2023 09:43:01 +0200 Subject: [PATCH 18/53] Add vertical fft calculation --- .../subgroups_operations.cpp | 351 +++++++++--------- .../subgroups_operations.h | 54 ++- shaders/subgroups_operations/fft.comp | 26 +- 3 files changed, 225 insertions(+), 206 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 44ddd9456..3b4385b85 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -67,21 +67,20 @@ SubgroupsOperations::~SubgroupsOperations() { if (device) { - fft_buffers.fft_tilde_h_kt_dx->destroy(get_device().get_handle()); - fft_buffers.fft_tilde_h_kt_dy->destroy(get_device().get_handle()); - fft_buffers.fft_tilde_h_kt_dz->destroy(get_device().get_handle()); - butterfly_precomp.destroy(get_device().get_handle()); + fft_buffers.fft_tilde_h_kt_dx->destroy(get_device().get_handle()); + fft_buffers.fft_tilde_h_kt_dy->destroy(get_device().get_handle()); + fft_buffers.fft_tilde_h_kt_dz->destroy(get_device().get_handle()); + butterfly_precomp.destroy(get_device().get_handle()); precompute.pipeline.destroy(get_device().get_handle()); - vkDestroyDescriptorSetLayout(get_device().get_handle(), precompute.descriptor_set_layout, nullptr); + vkDestroyDescriptorSetLayout(get_device().get_handle(), precompute.descriptor_set_layout, nullptr); tildas.pipeline.destroy(get_device().get_handle()); - vkDestroyDescriptorSetLayout(get_device().get_handle(), tildas.descriptor_set_layout, nullptr); + vkDestroyDescriptorSetLayout(get_device().get_handle(), tildas.descriptor_set_layout, nullptr); - - fft.pipelines.horizontal.destroy(get_device().get_handle()); - fft.pipelines.vertical.destroy(get_device().get_handle()); - vkDestroyDescriptorSetLayout(get_device().get_handle(), fft.descriptor_set_layout, nullptr); + fft.pipelines.horizontal.destroy(get_device().get_handle()); + fft.pipelines.vertical.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), fft.descriptor_set_layout, nullptr); vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); @@ -115,7 +114,7 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) create_tildas(); create_butterfly_texture(); - create_fft(); + create_fft(); build_compute_command_buffer(); @@ -203,38 +202,48 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); } - auto calculateFft = [this](){ - // update ubo - for (uint32_t i = 0; i < log_2_N; ++i) - { - FFTPage nrPage; - nrPage.page = i; - fft_page_ubo->convert_and_update(nrPage); // upload information about nr of page - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } - }; - - // fft horizontal; for Y axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); - calculateFft(); - } - - // fft horizontal; for X axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); - calculateFft(); - } - - // fft horizontal; for Z axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); - calculateFft(); - } + // fft horizontal; for Y axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + // fft horizontal; for X axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + // fft horizontal; for Z axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + // fft vertical; for Y axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + // fft vertical; for X axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + // fft vertical; for Z axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } if (compute.queue_family_index != ocean.graphics_queue_family_index) { @@ -284,9 +293,9 @@ void SubgroupsOperations::create_tildas() fft_buffers.fft_tilde_h_kt_dy = std::make_unique(); fft_buffers.fft_tilde_h_kt_dz = std::make_unique(); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dx); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dy); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dz); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dx); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dy); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dz); std::vector set_layout_bindngs = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), @@ -459,11 +468,11 @@ void SubgroupsOperations::prepare_uniform_buffers() camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - fft_page_ubo = std::make_unique(get_device(), sizeof(FFTPage), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - // tmp - FFTPage s; - s.page = 0; - fft_page_ubo->convert_and_update(s); + fft_page_ubo = std::make_unique(get_device(), sizeof(FFTPage), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + // tmp + FFTPage s; + s.page = 0; + fft_page_ubo->convert_and_update(s); update_uniform_buffers(); } @@ -508,11 +517,11 @@ void SubgroupsOperations::create_semaphore() void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 20u), // FFTParametersUbo, CameraUbo, ComputeUbo - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 20u), - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 20u)}; + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 20u), // FFTParametersUbo, CameraUbo, ComputeUbo + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 20u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 20u)}; VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 10u); + vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 10u); VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } @@ -637,6 +646,9 @@ void SubgroupsOperations::update_uniform_buffers() fft_params_ubo->convert_and_update(fft_ubo); fft_time_ubo->convert_and_update(fftTime); + FFTPage pageSize; + pageSize.page = static_cast(log_2_N); + fft_page_ubo->convert_and_update(pageSize); } void SubgroupsOperations::build_command_buffers() @@ -820,9 +832,9 @@ void SubgroupsOperations::createFBAttachement(VkFormat format, uint32_t width, u image.format = format; image.extent.width = width; image.extent.height = height; - image.extent.depth = 1u; - image.mipLevels = 1u; - image.arrayLayers = 1u; + image.extent.depth = 1u; + image.mipLevels = 1u; + image.arrayLayers = 1u; image.samples = VK_SAMPLE_COUNT_1_BIT; image.tiling = VK_IMAGE_TILING_OPTIMAL; image.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT | VK_IMAGE_USAGE_STORAGE_BIT; @@ -842,10 +854,10 @@ void SubgroupsOperations::createFBAttachement(VkFormat format, uint32_t width, u image_view_create_info.format = format; image_view_create_info.subresourceRange = {}; image_view_create_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - image_view_create_info.subresourceRange.baseMipLevel = 0u; - image_view_create_info.subresourceRange.levelCount = 1u; - image_view_create_info.subresourceRange.baseArrayLayer = 0u; - image_view_create_info.subresourceRange.layerCount = 1u; + image_view_create_info.subresourceRange.baseMipLevel = 0u; + image_view_create_info.subresourceRange.levelCount = 1u; + image_view_create_info.subresourceRange.baseArrayLayer = 0u; + image_view_create_info.subresourceRange.layerCount = 1u; image_view_create_info.image = attachment.image; VK_CHECK(vkCreateImageView(get_device().get_handle(), &image_view_create_info, nullptr, &attachment.view)); } @@ -910,112 +922,115 @@ void SubgroupsOperations::create_butterfly_texture() void SubgroupsOperations::create_fft() { - std::vector set_layout_bindngs = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 3u) - }; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); - VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft.descriptor_set_layout)); - - VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft.descriptor_set_layout, 1u); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_y)); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_x)); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_z)); - - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; - compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - compute_pipeline_layout_info.setLayoutCount = 1u; - compute_pipeline_layout_info.pSetLayouts = &fft.descriptor_set_layout; - - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &fft.pipelines.horizontal.pipeline_layout)); - - VkComputePipelineCreateInfo computeInfo = {}; - computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; - computeInfo.layout = fft.pipelines.horizontal.pipeline_layout; - computeInfo.stage = load_shader("subgroups_operations/fft.comp", VK_SHADER_STAGE_COMPUTE_BIT); - - std::array specialization_map_entries; - VkSpecializationInfo spec_info; - uint32_t direction = 0; - specialization_map_entries[0] = vkb::initializers::specialization_map_entry(0, 0, sizeof(uint32_t)); - spec_info = vkb::initializers::specialization_info(static_cast(specialization_map_entries.size()), - specialization_map_entries.data(), - sizeof(uint32_t), - &direction); - computeInfo.stage.pSpecializationInfo = &spec_info; - - VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft.pipelines.horizontal.pipeline)); - - fft.tilde_axis_y = std::make_unique(); - fft.tilde_axis_x = std::make_unique(); - fft.tilde_axis_z = std::make_unique(); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_y); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_x); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_z); - - VkDescriptorImageInfo image_descriptor_battlefly{}; - image_descriptor_battlefly.imageView = butterfly_precomp.view; - image_descriptor_battlefly.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_battlefly.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_tilda_y{}; - image_descriptor_tilda_y.imageView = fft_buffers.fft_tilde_h_kt_dy->view; - image_descriptor_tilda_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilda_y.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_tilde_axis_y{}; - image_descriptor_tilde_axis_y.imageView = fft.tilde_axis_y->view; - image_descriptor_tilde_axis_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilde_axis_y.sampler = nullptr; - - auto fft_page_descriptor = create_descriptor(*fft_page_ubo); - - std::vector write_descriptor_sets_asix_y = { - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_y), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_y), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; - - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_y.size()), write_descriptor_sets_asix_y.data(), 0u, nullptr); - - - VkDescriptorImageInfo image_descriptor_tilda_x{}; - image_descriptor_tilda_x.imageView = fft_buffers.fft_tilde_h_kt_dx->view; - image_descriptor_tilda_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilda_x.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_tilde_axis_x{}; - image_descriptor_tilde_axis_x.imageView = fft.tilde_axis_x->view; - image_descriptor_tilde_axis_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilde_axis_x.sampler = nullptr; - - std::vector write_descriptor_sets_asix_x = { - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_x), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_x), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; - - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_x.size()), write_descriptor_sets_asix_x.data(), 0u, nullptr); - - - VkDescriptorImageInfo image_descriptor_tilda_z{}; - image_descriptor_tilda_z.imageView = fft_buffers.fft_tilde_h_kt_dz->view; - image_descriptor_tilda_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilda_z.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_tilde_axis_z{}; - image_descriptor_tilde_axis_z.imageView = fft.tilde_axis_z->view; - image_descriptor_tilde_axis_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilde_axis_z.sampler = nullptr; - - std::vector write_descriptor_sets_asix_z = { - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_z), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_z), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; - - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_z.size()), write_descriptor_sets_asix_z.data(), 0u, nullptr); + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 3u)}; + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft.descriptor_set_layout)); + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_y)); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_x)); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_z)); + + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &fft.descriptor_set_layout; + + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &fft.pipelines.horizontal.pipeline_layout)); + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &fft.pipelines.vertical.pipeline_layout)); + + VkComputePipelineCreateInfo computeInfo = {}; + computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + computeInfo.layout = fft.pipelines.horizontal.pipeline_layout; + computeInfo.stage = load_shader("subgroups_operations/fft.comp", VK_SHADER_STAGE_COMPUTE_BIT); + + std::array specialization_map_entries; + VkSpecializationInfo spec_info; + uint32_t direction = 0u; + specialization_map_entries[0] = vkb::initializers::specialization_map_entry(0u, 0u, sizeof(uint32_t)); + spec_info = vkb::initializers::specialization_info(static_cast(specialization_map_entries.size()), + specialization_map_entries.data(), + sizeof(uint32_t), + &direction); + computeInfo.stage.pSpecializationInfo = &spec_info; + + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft.pipelines.horizontal.pipeline)); + + direction = 1u; + computeInfo.layout = fft.pipelines.vertical.pipeline_layout; + + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft.pipelines.vertical.pipeline)); + + + fft.tilde_axis_y = std::make_unique(); + fft.tilde_axis_x = std::make_unique(); + fft.tilde_axis_z = std::make_unique(); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_y); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_x); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_z); + + VkDescriptorImageInfo image_descriptor_battlefly{}; + image_descriptor_battlefly.imageView = butterfly_precomp.view; + image_descriptor_battlefly.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_battlefly.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_tilda_y{}; + image_descriptor_tilda_y.imageView = fft_buffers.fft_tilde_h_kt_dy->view; + image_descriptor_tilda_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilda_y.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_tilde_axis_y{}; + image_descriptor_tilde_axis_y.imageView = fft.tilde_axis_y->view; + image_descriptor_tilde_axis_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilde_axis_y.sampler = nullptr; + + auto fft_page_descriptor = create_descriptor(*fft_page_ubo); + + std::vector write_descriptor_sets_asix_y = { + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_y), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_y), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_y.size()), write_descriptor_sets_asix_y.data(), 0u, nullptr); + + VkDescriptorImageInfo image_descriptor_tilda_x{}; + image_descriptor_tilda_x.imageView = fft_buffers.fft_tilde_h_kt_dx->view; + image_descriptor_tilda_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilda_x.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_tilde_axis_x{}; + image_descriptor_tilde_axis_x.imageView = fft.tilde_axis_x->view; + image_descriptor_tilde_axis_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilde_axis_x.sampler = nullptr; + + std::vector write_descriptor_sets_asix_x = { + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_x), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_x), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_x.size()), write_descriptor_sets_asix_x.data(), 0u, nullptr); + + VkDescriptorImageInfo image_descriptor_tilda_z{}; + image_descriptor_tilda_z.imageView = fft_buffers.fft_tilde_h_kt_dz->view; + image_descriptor_tilda_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilda_z.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_tilde_axis_z{}; + image_descriptor_tilde_axis_z.imageView = fft.tilde_axis_z->view; + image_descriptor_tilde_axis_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_tilde_axis_z.sampler = nullptr; + + std::vector write_descriptor_sets_asix_z = { + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_z), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_z), + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_z.size()), write_descriptor_sets_asix_z.data(), 0u, nullptr); } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index a806b8e12..bbcbe83d8 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -60,7 +60,7 @@ class SubgroupsOperations : public ApiVulkanSample void create_tildas(); void create_butterfly_texture(); - void create_fft(); + void create_fft(); void update_uniform_buffers(); @@ -100,10 +100,10 @@ class SubgroupsOperations : public ApiVulkanSample alignas(8) glm::vec2 wind; }; - struct FFTPage - { - alignas(4) int32_t page = {0}; - }; + struct FFTPage + { + alignas(4) int32_t page = {0u}; + }; struct TimeUbo { @@ -122,8 +122,7 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr camera_ubo; std::unique_ptr fft_params_ubo; std::unique_ptr fft_time_ubo; - std::unique_ptr fft_page_ubo; - + std::unique_ptr fft_page_ubo; std::vector> h_tilde_0; std::vector> h_tilde_0_conj; @@ -161,29 +160,28 @@ class SubgroupsOperations : public ApiVulkanSample struct { - VkQueue queue = {VK_NULL_HANDLE}; - VkCommandPool command_pool = {VK_NULL_HANDLE}; - VkCommandBuffer command_buffer = {VK_NULL_HANDLE}; - VkSemaphore semaphore = {VK_NULL_HANDLE}; - uint32_t queue_family_index = {-1u}; + VkQueue queue = {VK_NULL_HANDLE}; + VkCommandPool command_pool = {VK_NULL_HANDLE}; + VkCommandBuffer command_buffer = {VK_NULL_HANDLE}; + VkSemaphore semaphore = {VK_NULL_HANDLE}; + uint32_t queue_family_index = {-1u}; } compute; - - struct - { - VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; - VkDescriptorSet descriptor_set_axis_y = {VK_NULL_HANDLE}; - VkDescriptorSet descriptor_set_axis_x = {VK_NULL_HANDLE}; - VkDescriptorSet descriptor_set_axis_z = {VK_NULL_HANDLE}; - struct - { - Pipeline horizontal; - Pipeline vertical; - } pipelines; - std::unique_ptr tilde_axis_y; - std::unique_ptr tilde_axis_x; - std::unique_ptr tilde_axis_z; - } fft; + struct + { + VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set_axis_y = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set_axis_x = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set_axis_z = {VK_NULL_HANDLE}; + struct + { + Pipeline horizontal; + Pipeline vertical; + } pipelines; + std::unique_ptr tilde_axis_y; + std::unique_ptr tilde_axis_x; + std::unique_ptr tilde_axis_z; + } fft; struct { diff --git a/shaders/subgroups_operations/fft.comp b/shaders/subgroups_operations/fft.comp index 1b8238200..5e88a5fd3 100644 --- a/shaders/subgroups_operations/fft.comp +++ b/shaders/subgroups_operations/fft.comp @@ -28,7 +28,7 @@ layout (binding = 2, rgba32f) uniform image2D u_pingpong1; layout (binding = 3) uniform FftPage { - int page; + int pages; } fftUbo; struct Complex @@ -64,9 +64,10 @@ Complex complex_conj(Complex c) void HorizontalButterflies(in ivec2 pixel_pos) { - vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(fftUbo.page, pixel_pos.x)); - bool type = bool(fftUbo.page % 2); - if (type == false) + for (int i = 0; i < fftUbo.pages; ++i) + { + vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(i, pixel_pos.x)); + if ((i % 2) == 0) { vec2 a_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; vec2 b_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; @@ -77,7 +78,7 @@ void HorizontalButterflies(in ivec2 pixel_pos) // Buttefly operation Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - + subgroupMemoryBarrierImage(); imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); } else @@ -91,17 +92,19 @@ void HorizontalButterflies(in ivec2 pixel_pos) // Buttefly operation Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - + subgroupMemoryBarrierImage(); imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); } + } } void VerticalButterfiles(in ivec2 pixel_pos) { - vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(fftUbo.page, pixel_pos.y)); - bool type = bool(fftUbo.page % 2); + for (int i = 0; i < fftUbo.pages; ++i) + { + vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(i, pixel_pos.y)); - if (type == false) + if ((i % 2) == 0) { vec2 a_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; vec2 b_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; @@ -112,6 +115,7 @@ void VerticalButterfiles(in ivec2 pixel_pos) // Buttefly operation Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + subgroupMemoryBarrierImage(); imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); } @@ -126,14 +130,16 @@ void VerticalButterfiles(in ivec2 pixel_pos) // Buttefly operation Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + subgroupMemoryBarrierImage(); imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); } + } } void main() { - ivec2 uv = ivec2(gl_GlobalInvocationID.xy); + ivec2 uv = ivec2(gl_GlobalInvocationID.xy * 0.5f); if (direction == 0) HorizontalButterflies(uv); else if (direction == 1) VerticalButterfiles(uv); } From e4b6eee958e088cb5363f64fe2c1412281021072 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Fri, 29 Sep 2023 11:52:49 +0200 Subject: [PATCH 19/53] Add invert fft and clean-up --- .../subgroups_operations/CMakeLists.txt | 4 +- .../subgroups_operations.cpp | 147 ++++++++++++++++-- .../subgroups_operations.h | 22 +++ shaders/subgroups_operations/fft_invert.comp | 64 ++++++++ shaders/subgroups_operations/ocean_fft.comp | 136 ---------------- 5 files changed, 226 insertions(+), 147 deletions(-) create mode 100644 shaders/subgroups_operations/fft_invert.comp delete mode 100644 shaders/subgroups_operations/ocean_fft.comp diff --git a/samples/extensions/subgroups_operations/CMakeLists.txt b/samples/extensions/subgroups_operations/CMakeLists.txt index 1bfa3f3bb..0bdd3e306 100644 --- a/samples/extensions/subgroups_operations/CMakeLists.txt +++ b/samples/extensions/subgroups_operations/CMakeLists.txt @@ -28,7 +28,9 @@ add_sample( SHADER_FILES_GLSL "subgroups_operations/ocean.vert" "subgroups_operations/ocean.frag" - "subgroups_operations/ocean_fft.comp" + "subgroups_operations/ocean.tesc" + "subgroups_operations/ocean.tese" + "subgroups_operations/fft_invert.comp" "subgroups_operations/butterfly_precomp.comp" "subgroups_operations/fft_tilde_h.comp" "subgroups_operations/fft.comp") diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 3b4385b85..ba68e8539 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -115,6 +115,7 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) create_tildas(); create_butterfly_texture(); create_fft(); + create_fft_inversion(); build_compute_command_buffer(); @@ -245,6 +246,25 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); } + ////////////////////////////////////// fft inverse + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set_asix_y, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set_asix_x, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set_asix_z, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + if (compute.queue_family_index != ocean.graphics_queue_family_index) { VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); @@ -469,10 +489,7 @@ void SubgroupsOperations::prepare_uniform_buffers() fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_page_ubo = std::make_unique(get_device(), sizeof(FFTPage), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - // tmp - FFTPage s; - s.page = 0; - fft_page_ubo->convert_and_update(s); + invert_fft_ubo = std::make_unique(get_device(), sizeof(FFTInvert), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); update_uniform_buffers(); } @@ -636,19 +653,24 @@ void SubgroupsOperations::update_uniform_buffers() ubo.view = camera.matrices.view; ubo.projection = camera.matrices.perspective; - camera_ubo->convert_and_update(ubo); - FFTParametersUbo fft_ubo; fft_ubo.amplitude = ui.amplitude; fft_ubo.grid_size = grid_size; fft_ubo.length = ui.length; fft_ubo.wind = ui.wind; - fft_params_ubo->convert_and_update(fft_ubo); - fft_time_ubo->convert_and_update(fftTime); FFTPage pageSize; pageSize.page = static_cast(log_2_N); + + FFTInvert invertFft; + invertFft.grid_size = grid_size; + invertFft.page_idx = log_2_N % 2; + + camera_ubo->convert_and_update(ubo); + fft_params_ubo->convert_and_update(fft_ubo); + fft_time_ubo->convert_and_update(fftTime); fft_page_ubo->convert_and_update(pageSize); + invert_fft_ubo->convert_and_update(invertFft); } void SubgroupsOperations::build_command_buffers() @@ -960,12 +982,11 @@ void SubgroupsOperations::create_fft() VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft.pipelines.horizontal.pipeline)); - direction = 1u; + direction = 1u; computeInfo.layout = fft.pipelines.vertical.pipeline_layout; VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft.pipelines.vertical.pipeline)); - fft.tilde_axis_y = std::make_unique(); fft.tilde_axis_x = std::make_unique(); fft.tilde_axis_z = std::make_unique(); @@ -1034,3 +1055,109 @@ void SubgroupsOperations::create_fft() vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_z.size()), write_descriptor_sets_asix_z.data(), 0u, nullptr); } + +void SubgroupsOperations::create_fft_inversion() +{ + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 3u)}; + + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft_inversion.descriptor_set_layout)); + + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft_inversion.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft_inversion.descriptor_set_asix_y)); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft_inversion.descriptor_set_asix_x)); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft_inversion.descriptor_set_asix_z)); + + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &fft_inversion.descriptor_set_layout; + + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &fft_inversion.pipeline.pipeline_layout)); + + VkComputePipelineCreateInfo computeInfo = {}; + computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + computeInfo.layout = fft_inversion.pipeline.pipeline_layout; + computeInfo.stage = load_shader("subgroups_operations/fft_invert.comp", VK_SHADER_STAGE_COMPUTE_BIT); + + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft_inversion.pipeline.pipeline)); + + fft_buffers.fft_displacement_y = std::make_unique(); + fft_buffers.fft_displacement_x = std::make_unique(); + fft_buffers.fft_displacement_z = std::make_unique(); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_displacement_y); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_displacement_x); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_displacement_z); + + VkDescriptorImageInfo image_descriptor_displacment_axis_y{}; + image_descriptor_displacment_axis_y.imageView = fft_buffers.fft_displacement_y->view; + image_descriptor_displacment_axis_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_displacment_axis_y.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_pingpong0_axis_y{}; + image_descriptor_pingpong0_axis_y.imageView = fft_buffers.fft_tilde_h_kt_dy->view; + image_descriptor_pingpong0_axis_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_pingpong0_axis_y.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_pingpong1_axis_y{}; + image_descriptor_pingpong1_axis_y.imageView = fft.tilde_axis_y->view; + image_descriptor_pingpong1_axis_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_pingpong1_axis_y.sampler = nullptr; + + auto fft_page_descriptor = create_descriptor(*invert_fft_ubo); + + std::vector write_descriptor_sets_axis_y = { + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_displacment_axis_y), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_pingpong0_axis_y), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_pingpong1_axis_y), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_y, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_axis_y.size()), write_descriptor_sets_axis_y.data(), 0u, nullptr); + + VkDescriptorImageInfo image_descriptor_displacment_axis_x{}; + image_descriptor_displacment_axis_x.imageView = fft_buffers.fft_displacement_x->view; + image_descriptor_displacment_axis_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_displacment_axis_x.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_pingpong0_axis_x{}; + image_descriptor_pingpong0_axis_x.imageView = fft_buffers.fft_tilde_h_kt_dx->view; + image_descriptor_pingpong0_axis_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_pingpong0_axis_x.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_pingpong1_axis_x{}; + image_descriptor_pingpong1_axis_x.imageView = fft.tilde_axis_x->view; + image_descriptor_pingpong1_axis_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_pingpong1_axis_x.sampler = nullptr; + + std::vector write_descriptor_sets_axis_x = { + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_displacment_axis_x), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_pingpong0_axis_x), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_pingpong1_axis_x), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_x, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_axis_x.size()), write_descriptor_sets_axis_x.data(), 0u, nullptr); + + VkDescriptorImageInfo image_descriptor_displacment_axis_z{}; + image_descriptor_displacment_axis_z.imageView = fft_buffers.fft_displacement_z->view; + image_descriptor_displacment_axis_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_displacment_axis_z.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_pingpong0_axis_z{}; + image_descriptor_pingpong0_axis_z.imageView = fft_buffers.fft_tilde_h_kt_dz->view; + image_descriptor_pingpong0_axis_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_pingpong0_axis_z.sampler = nullptr; + + VkDescriptorImageInfo image_descriptor_pingpong1_axis_z{}; + image_descriptor_pingpong1_axis_z.imageView = fft.tilde_axis_z->view; + image_descriptor_pingpong1_axis_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor_pingpong1_axis_z.sampler = nullptr; + + std::vector write_descriptor_sets_axis_z = { + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_displacment_axis_z), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_pingpong0_axis_z), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_pingpong1_axis_z), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_z, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_axis_z.size()), write_descriptor_sets_axis_z.data(), 0u, nullptr); +} diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index bbcbe83d8..c4b2c4198 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -61,6 +61,7 @@ class SubgroupsOperations : public ApiVulkanSample void create_tildas(); void create_butterfly_texture(); void create_fft(); + void create_fft_inversion(); void update_uniform_buffers(); @@ -105,6 +106,12 @@ class SubgroupsOperations : public ApiVulkanSample alignas(4) int32_t page = {0u}; }; + struct FFTInvert + { + alignas(4) int32_t page_idx = {-1}; + alignas(4) uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; + }; + struct TimeUbo { alignas(4) float time = {0.0f}; @@ -123,6 +130,7 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr fft_params_ubo; std::unique_ptr fft_time_ubo; std::unique_ptr fft_page_ubo; + std::unique_ptr invert_fft_ubo; std::vector> h_tilde_0; std::vector> h_tilde_0_conj; @@ -156,6 +164,10 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr fft_tilde_h_kt_dx; std::unique_ptr fft_tilde_h_kt_dy; std::unique_ptr fft_tilde_h_kt_dz; + std::unique_ptr fft_displacement_y; + std::unique_ptr fft_displacement_x; + std::unique_ptr fft_displacement_z; + } fft_buffers; struct @@ -183,6 +195,16 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr tilde_axis_z; } fft; + struct + { + VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set_asix_y = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set_asix_x = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set_asix_z = {VK_NULL_HANDLE}; + + Pipeline pipeline; + } fft_inversion; + struct { VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; diff --git a/shaders/subgroups_operations/fft_invert.comp b/shaders/subgroups_operations/fft_invert.comp new file mode 100644 index 000000000..d6a14b17c --- /dev/null +++ b/shaders/subgroups_operations/fft_invert.comp @@ -0,0 +1,64 @@ +#version 450 +#extension GL_KHR_shader_subgroup_basic : enable + +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0, rgba32f) writeonly uniform image2D u_displacement; + +layout (binding = 1, rgba32f) readonly uniform image2D u_pingpong0_y; +layout (binding = 2, rgba32f) readonly uniform image2D u_pingpong1_y; + +//layout (binding = 3, rgba32f) readonly uniform image2D u_pingpong0_x; +//layout (binding = 4, rgba32f) readonly uniform image2D u_pingpong1_x; +// +//layout (binding = 5, rgba32f) readonly uniform image2D u_pingpong0_z; +//layout (binding = 6, rgba32f) readonly uniform image2D u_pingpong1_z; + +layout (binding = 3) uniform InvertFft +{ + int pong_idx; + uint grid_size; +} fftUbo; + +void main() +{ + + uint N = fftUbo.grid_size; + + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy * 0.5f); + + float perms[2] = { 1.0, -1.0 }; + int index = int(mod(pixel_pos.x + pixel_pos.y, 2)); + float perm = perms[index]; + uint pingpong_index = fftUbo.pong_idx; + if (pingpong_index == 0) + { + float h_y = imageLoad(u_pingpong0_y, pixel_pos).r; + // float h_x = imageLoad(u_pingpong0_x, pixel_pos).r; + // float h_z = imageLoad(u_pingpong0_z, pixel_pos).r; + // imageStore(u_displacement, pixel_pos, vec4(perm * (h_y / float(N * N)), perm * (h_x / float(N * N)), perm * (h_z / float(N * N)), 1.0f)); + imageStore(u_displacement, pixel_pos, vec4(perm * (h_y / float(N * N)), 0.0f, 0.0f, 1.0f)); + } + else if (pingpong_index == 1) + { + float h = imageLoad(u_pingpong1_y, pixel_pos).r; + imageStore(u_displacement, pixel_pos, vec4(perm * (h / float(N * N)), 0, 0, 1)); + } +} diff --git a/shaders/subgroups_operations/ocean_fft.comp b/shaders/subgroups_operations/ocean_fft.comp deleted file mode 100644 index 78b62b6cb..000000000 --- a/shaders/subgroups_operations/ocean_fft.comp +++ /dev/null @@ -1,136 +0,0 @@ -#version 450 -#extension GL_KHR_shader_subgroup_basic : enable - -/* Copyright (c) 2023, Mobica Limited - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -layout (local_size_x = 4, local_size_y = 4, local_size_z = 2) in; - -#define PI 3.141592f -#define GRAVITY 9.81f - -struct Vertex -{ - vec3 position; -// vec3 normal; -}; - -struct Complex -{ - float real; - float imag; -}; - -Complex complexMultiplicationByScalar(Complex a, float scalar) -{ - Complex res; - res.real = scalar * a.real; - res.imag = scalar * a.imag; - return res; -} - -Complex complexMultiplication(Complex a, Complex b) -{ - Complex res; - res.real = a.real * b.real; - res.imag = a.imag * b.imag; - return res; -} - -Complex complexSum(Complex a, Complex b) -{ - Complex res; - res.real = a.real + b.real; - res.imag = a.imag + b.imag; - return res; -} - -Complex conjugate(Complex a) -{ - Complex res; - res.real = a.real; - res.imag = -a.imag; - return res; -} - -layout (set = 0, binding = 0) uniform FFTParametersUbo -{ - float amplitude; - float len; - uint grid_size; - vec2 wind; -} fftUbo; - -layout (std140, set = 0, binding = 1) readonly buffer FFTInputHTilda0_DataBuffer -{ - Complex htilde0[]; -}; - - -layout (std140, set = 0, binding = 2) readonly buffer FFTInputHTilda0Conj_DataBuffer -{ - Complex htilde0_conj[]; -}; - -layout (std140, set = 0, binding = 3) readonly buffer FFTInputWeight_DataBuffer -{ - Complex weight[]; -}; - - -layout (binding = 4, rgba32f) writeonly uniform image2D result_texture; - -// Add another FFTInputData buffer to store the results of the hTilde calculation -// Add 3'th FFTInputData buffer to store fft calculation results - final values - -float dispersion(uint n, uint m) -{ - float w0 = 2.0f * PI / 50.0f; - float x = PI * (2.0f * n - fftUbo.grid_size) / fftUbo.len; - float z = PI * (2.0f * m - fftUbo.grid_size) / fftUbo.len; - return floor(sqrt(GRAVITY * sqrt(x * x + z * z)) / w0) *w0; -} - -Complex hTilde(float t, uint n, uint m) -{ - uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; - Complex tilde0 ; //= data[idx].tilde; - Complex tilde0_conj; // = data[idx].tilde_conj; - - float omega = dispersion(n, m); - - Complex c0; - c0.real = cos(omega); - c0.imag = sin(omega); - - Complex c1 = conjugate(c0); - - return complexSum(complexMultiplication(tilde0, c0), complexMultiplication(tilde0_conj, c1)); -} - -void fft() -{ - -} - -void main() -{ - uint idx = gl_GlobalInvocationID.x * gl_SubgroupInvocationID + gl_GlobalInvocationID.y * gl_SubgroupInvocationID + gl_GlobalInvocationID.z * gl_SubgroupInvocationID; - - subgroupMemoryBarrierBuffer(); - -} From b4880503b935090f19b9555b8e38efce58e6f5c5 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 4 Oct 2023 09:35:54 +0200 Subject: [PATCH 20/53] Fix validation layer errors and fix tessellation shaders --- .../subgroups_operations.cpp | 28 +++++++++++++++++-- shaders/subgroups_operations/ocean.tesc | 10 +++++++ shaders/subgroups_operations/ocean.tese | 22 ++++++++++++++- shaders/subgroups_operations/ocean.vert | 2 +- 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index ba68e8539..5526bd54a 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -545,7 +545,9 @@ void SubgroupsOperations::setup_descriptor_pool() void SubgroupsOperations::create_descriptor_set_layout() { std::vector set_layout_bindings = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT, 0u)}; + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, + 0u)}; VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); @@ -570,7 +572,7 @@ void SubgroupsOperations::create_pipelines() { VkPipelineInputAssemblyStateCreateInfo input_assembly_state = vkb::initializers::pipeline_input_assembly_state_create_info( - VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, + VK_PRIMITIVE_TOPOLOGY_PATCH_LIST, 0u, VK_FALSE); VkPipelineRasterizationStateCreateInfo rasterization_state = @@ -860,6 +862,7 @@ void SubgroupsOperations::createFBAttachement(VkFormat format, uint32_t width, u image.samples = VK_SAMPLE_COUNT_1_BIT; image.tiling = VK_IMAGE_TILING_OPTIMAL; image.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT | VK_IMAGE_USAGE_STORAGE_BIT; + image.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; VkMemoryAllocateInfo memory_allocate_info = vkb::initializers::memory_allocate_info(); VkMemoryRequirements memory_requirements; @@ -882,6 +885,27 @@ void SubgroupsOperations::createFBAttachement(VkFormat format, uint32_t width, u image_view_create_info.subresourceRange.layerCount = 1u; image_view_create_info.image = attachment.image; VK_CHECK(vkCreateImageView(get_device().get_handle(), &image_view_create_info, nullptr, &attachment.view)); + + VkImageMemoryBarrier imgMemBarrier = vkb::initializers::image_memory_barrier(); + imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_GENERAL; + imgMemBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + imgMemBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + imgMemBarrier.image = attachment.image; + imgMemBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + imgMemBarrier.subresourceRange.baseArrayLayer = 0u; + imgMemBarrier.subresourceRange.levelCount = 1u; + imgMemBarrier.subresourceRange.baseArrayLayer = 0u; + imgMemBarrier.subresourceRange.layerCount = 1u; + imgMemBarrier.srcAccessMask = 0u; + imgMemBarrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + + VkPipelineStageFlagBits srcStage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; + VkPipelineStageFlagBits dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT; + + VkCommandBuffer cmd = get_device().create_command_buffer(VK_COMMAND_BUFFER_LEVEL_PRIMARY, true); + vkCmdPipelineBarrier(cmd, srcStage, dstStage, 0u, 0u, nullptr, 0u, nullptr, 1u, &imgMemBarrier); + get_device().flush_command_buffer(cmd, queue, true); } uint32_t SubgroupsOperations::reverse(uint32_t i) diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index eba26d06d..0f49159d4 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -24,4 +24,14 @@ { gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position; + if (gl_InvocationID == 0) + { + gl_TessLevelOuter[0] = 1.0f; + gl_TessLevelOuter[1] = 1.0f; + gl_TessLevelOuter[2] = 1.0f; + gl_TessLevelOuter[3] = 1.0f; + + gl_TessLevelInner[0] = 1.0f; + gl_TessLevelInner[1] = 1.0f; + } } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index ef606d6ee..05a4a46c7 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -16,8 +16,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - layout(quads, equal_spacing, cw) in; + layout(quads, equal_spacing, ccw) in; + + + layout (binding = 0) uniform Ubo +{ + mat4 projection; + mat4 view; + mat4 model; +} ubo; + void main() { + vec4 p00 = gl_in[0].gl_Position; + vec4 p01 = gl_in[1].gl_Position; + vec4 p10 = gl_in[2].gl_Position; + vec4 p11 = gl_in[3].gl_Position; + + vec4 p0 = (p01 - p00) * gl_TessCoord.x + p00; + vec4 p1 = (p11 - p10) * gl_TessCoord.x + p10; + + vec4 p = (p1 - p0) * gl_TessCoord.y + p0; + + gl_Position = ubo.projection * ubo.view * ubo.model * p; } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.vert b/shaders/subgroups_operations/ocean.vert index 9d6b773a6..a65d54dd6 100644 --- a/shaders/subgroups_operations/ocean.vert +++ b/shaders/subgroups_operations/ocean.vert @@ -28,5 +28,5 @@ layout (binding = 0) uniform Ubo void main() { - gl_Position = ubo.projection * ubo.view * ubo.model * vec4(inPos.xyz, 1.0f); + gl_Position = vec4(inPos.xyz, 1.0f); } From 4e74a2954f6776840c9a28dc5fae7c085392b1ad Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 4 Oct 2023 11:59:30 +0200 Subject: [PATCH 21/53] some fixes --- .../subgroups_operations.cpp | 193 +++++------------- .../subgroups_operations.h | 12 +- shaders/subgroups_operations/fft_invert.comp | 25 +-- shaders/subgroups_operations/ocean.tesc | 10 +- shaders/subgroups_operations/ocean.tese | 11 +- 5 files changed, 80 insertions(+), 171 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 5526bd54a..d3b64601a 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -70,6 +70,7 @@ SubgroupsOperations::~SubgroupsOperations() fft_buffers.fft_tilde_h_kt_dx->destroy(get_device().get_handle()); fft_buffers.fft_tilde_h_kt_dy->destroy(get_device().get_handle()); fft_buffers.fft_tilde_h_kt_dz->destroy(get_device().get_handle()); + fft_buffers.fft_displacement->destroy(get_device().get_handle()); butterfly_precomp.destroy(get_device().get_handle()); precompute.pipeline.destroy(get_device().get_handle()); @@ -78,6 +79,12 @@ SubgroupsOperations::~SubgroupsOperations() tildas.pipeline.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), tildas.descriptor_set_layout, nullptr); + fft_inversion.pipeline.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), fft_inversion.descriptor_set_layout, nullptr); + + fft.tilde_axis_x->destroy(get_device().get_handle()); + fft.tilde_axis_y->destroy(get_device().get_handle()); + fft.tilde_axis_z->destroy(get_device().get_handle()); fft.pipelines.horizontal.destroy(get_device().get_handle()); fft.pipelines.vertical.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), fft.descriptor_set_layout, nullptr); @@ -246,22 +253,10 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); } - ////////////////////////////////////// fft inverse - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set_asix_y, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } - - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set_asix_x, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } - + // fft inverse { vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set_asix_z, 0u, nullptr); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set, 0u, nullptr); vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); } @@ -345,25 +340,12 @@ void SubgroupsOperations::create_tildas() VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &tildas.pipeline.pipeline)); - // update descriptor set - VkDescriptorBufferInfo htilde_0_descriptor = create_descriptor(*fft_buffers.fft_input_htilde0); VkDescriptorBufferInfo htilde_conj_0_descriptor = create_descriptor(*fft_buffers.fft_input_htilde0_conj); - VkDescriptorImageInfo image_dx_descriptor{}; - image_dx_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dx->view; - image_dx_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_dx_descriptor.sampler = nullptr; - - VkDescriptorImageInfo image_dy_descriptor{}; - image_dy_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dy->view; - image_dy_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_dy_descriptor.sampler = nullptr; - - VkDescriptorImageInfo image_dz_descriptor{}; - image_dz_descriptor.imageView = fft_buffers.fft_tilde_h_kt_dz->view; - image_dz_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_dz_descriptor.sampler = nullptr; + VkDescriptorImageInfo image_dx_descriptor = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dx); + VkDescriptorImageInfo image_dy_descriptor = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dy); + VkDescriptorImageInfo image_dz_descriptor = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dz); VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); VkDescriptorBufferInfo fft_time_ubo_buffer = create_descriptor(*fft_time_ubo); @@ -955,10 +937,7 @@ void SubgroupsOperations::create_butterfly_texture() VkDescriptorBufferInfo bit_reverse_descriptor = create_descriptor(*bit_reverse_buffer); bit_reverse_buffer->convert_and_update(bit_reverse_arr.data(), sizeof(uint32_t) * bit_reverse_arr.size()); - VkDescriptorImageInfo image_descriptor{}; - image_descriptor.imageView = butterfly_precomp.view; - image_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor.sampler = nullptr; + VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor), @@ -1018,20 +997,9 @@ void SubgroupsOperations::create_fft() createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_x); createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_z); - VkDescriptorImageInfo image_descriptor_battlefly{}; - image_descriptor_battlefly.imageView = butterfly_precomp.view; - image_descriptor_battlefly.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_battlefly.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_tilda_y{}; - image_descriptor_tilda_y.imageView = fft_buffers.fft_tilde_h_kt_dy->view; - image_descriptor_tilda_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilda_y.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_tilde_axis_y{}; - image_descriptor_tilde_axis_y.imageView = fft.tilde_axis_y->view; - image_descriptor_tilde_axis_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilde_axis_y.sampler = nullptr; + VkDescriptorImageInfo image_descriptor_battlefly = create_fb_descriptor(butterfly_precomp); + VkDescriptorImageInfo image_descriptor_tilda_y = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dy); + VkDescriptorImageInfo image_descriptor_tilde_axis_y = create_fb_descriptor(*fft.tilde_axis_y); auto fft_page_descriptor = create_descriptor(*fft_page_ubo); @@ -1043,15 +1011,8 @@ void SubgroupsOperations::create_fft() vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_y.size()), write_descriptor_sets_asix_y.data(), 0u, nullptr); - VkDescriptorImageInfo image_descriptor_tilda_x{}; - image_descriptor_tilda_x.imageView = fft_buffers.fft_tilde_h_kt_dx->view; - image_descriptor_tilda_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilda_x.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_tilde_axis_x{}; - image_descriptor_tilde_axis_x.imageView = fft.tilde_axis_x->view; - image_descriptor_tilde_axis_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilde_axis_x.sampler = nullptr; + VkDescriptorImageInfo image_descriptor_tilda_x = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dx); + VkDescriptorImageInfo image_descriptor_tilde_axis_x = create_fb_descriptor(*fft.tilde_axis_x); std::vector write_descriptor_sets_asix_x = { vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), @@ -1061,15 +1022,8 @@ void SubgroupsOperations::create_fft() vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_x.size()), write_descriptor_sets_asix_x.data(), 0u, nullptr); - VkDescriptorImageInfo image_descriptor_tilda_z{}; - image_descriptor_tilda_z.imageView = fft_buffers.fft_tilde_h_kt_dz->view; - image_descriptor_tilda_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilda_z.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_tilde_axis_z{}; - image_descriptor_tilde_axis_z.imageView = fft.tilde_axis_z->view; - image_descriptor_tilde_axis_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_tilde_axis_z.sampler = nullptr; + VkDescriptorImageInfo image_descriptor_tilda_z = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dz); + VkDescriptorImageInfo image_descriptor_tilde_axis_z = create_fb_descriptor(*fft.tilde_axis_z); std::vector write_descriptor_sets_asix_z = { vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), @@ -1086,15 +1040,17 @@ void SubgroupsOperations::create_fft_inversion() vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 3u)}; + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 3u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 4u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 5u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 6u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 7u)}; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft_inversion.descriptor_set_layout)); VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft_inversion.descriptor_set_layout, 1u); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft_inversion.descriptor_set_asix_y)); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft_inversion.descriptor_set_asix_x)); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft_inversion.descriptor_set_asix_z)); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft_inversion.descriptor_set)); VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; @@ -1110,78 +1066,37 @@ void SubgroupsOperations::create_fft_inversion() VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft_inversion.pipeline.pipeline)); - fft_buffers.fft_displacement_y = std::make_unique(); - fft_buffers.fft_displacement_x = std::make_unique(); - fft_buffers.fft_displacement_z = std::make_unique(); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_displacement_y); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_displacement_x); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_displacement_z); + fft_buffers.fft_displacement = std::make_unique(); - VkDescriptorImageInfo image_descriptor_displacment_axis_y{}; - image_descriptor_displacment_axis_y.imageView = fft_buffers.fft_displacement_y->view; - image_descriptor_displacment_axis_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_displacment_axis_y.sampler = nullptr; + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_displacement); - VkDescriptorImageInfo image_descriptor_pingpong0_axis_y{}; - image_descriptor_pingpong0_axis_y.imageView = fft_buffers.fft_tilde_h_kt_dy->view; - image_descriptor_pingpong0_axis_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_pingpong0_axis_y.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_pingpong1_axis_y{}; - image_descriptor_pingpong1_axis_y.imageView = fft.tilde_axis_y->view; - image_descriptor_pingpong1_axis_y.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_pingpong1_axis_y.sampler = nullptr; + VkDescriptorImageInfo image_descriptor_displacment_axis = create_fb_descriptor(*fft_buffers.fft_displacement); + VkDescriptorImageInfo image_descriptor_pingpong0_axis_y = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dy); + VkDescriptorImageInfo image_descriptor_pingpong1_axis_y = create_fb_descriptor(*fft.tilde_axis_y); + VkDescriptorImageInfo image_descriptor_pingpong0_axis_x = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dx); + VkDescriptorImageInfo image_descriptor_pingpong1_axis_x = create_fb_descriptor(*fft.tilde_axis_x); + VkDescriptorImageInfo image_descriptor_pingpong0_axis_z = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dz); + VkDescriptorImageInfo image_descriptor_pingpong1_axis_z = create_fb_descriptor(*fft.tilde_axis_z); auto fft_page_descriptor = create_descriptor(*invert_fft_ubo); - std::vector write_descriptor_sets_axis_y = { - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_displacment_axis_y), - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_pingpong0_axis_y), - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_pingpong1_axis_y), - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_y, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_axis_y.size()), write_descriptor_sets_axis_y.data(), 0u, nullptr); - - VkDescriptorImageInfo image_descriptor_displacment_axis_x{}; - image_descriptor_displacment_axis_x.imageView = fft_buffers.fft_displacement_x->view; - image_descriptor_displacment_axis_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_displacment_axis_x.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_pingpong0_axis_x{}; - image_descriptor_pingpong0_axis_x.imageView = fft_buffers.fft_tilde_h_kt_dx->view; - image_descriptor_pingpong0_axis_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_pingpong0_axis_x.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_pingpong1_axis_x{}; - image_descriptor_pingpong1_axis_x.imageView = fft.tilde_axis_x->view; - image_descriptor_pingpong1_axis_x.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_pingpong1_axis_x.sampler = nullptr; - - std::vector write_descriptor_sets_axis_x = { - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_displacment_axis_x), - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_pingpong0_axis_x), - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_pingpong1_axis_x), - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_x, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_axis_x.size()), write_descriptor_sets_axis_x.data(), 0u, nullptr); - - VkDescriptorImageInfo image_descriptor_displacment_axis_z{}; - image_descriptor_displacment_axis_z.imageView = fft_buffers.fft_displacement_z->view; - image_descriptor_displacment_axis_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_displacment_axis_z.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_pingpong0_axis_z{}; - image_descriptor_pingpong0_axis_z.imageView = fft_buffers.fft_tilde_h_kt_dz->view; - image_descriptor_pingpong0_axis_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_pingpong0_axis_z.sampler = nullptr; - - VkDescriptorImageInfo image_descriptor_pingpong1_axis_z{}; - image_descriptor_pingpong1_axis_z.imageView = fft.tilde_axis_z->view; - image_descriptor_pingpong1_axis_z.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor_pingpong1_axis_z.sampler = nullptr; - - std::vector write_descriptor_sets_axis_z = { - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_displacment_axis_z), - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_pingpong0_axis_z), - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_pingpong1_axis_z), - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set_asix_z, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_axis_z.size()), write_descriptor_sets_axis_z.data(), 0u, nullptr); + std::vector write_descriptor_sets = { + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_displacment_axis), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_pingpong0_axis_y), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_pingpong1_axis_y), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 3u, &image_descriptor_pingpong0_axis_x), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &image_descriptor_pingpong1_axis_x), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 5u, &image_descriptor_pingpong0_axis_z), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 6u, &image_descriptor_pingpong1_axis_z), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 7u, &fft_page_descriptor)}; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } + +VkDescriptorImageInfo SubgroupsOperations::create_fb_descriptor(FBAttachment &attachment) +{ + VkDescriptorImageInfo image_descriptor{}; + image_descriptor.imageView = attachment.view; + image_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor.sampler = nullptr; + return image_descriptor; +} \ No newline at end of file diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index c4b2c4198..cedc66bf5 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -65,7 +65,6 @@ class SubgroupsOperations : public ApiVulkanSample void update_uniform_buffers(); - // ocean stuff float phillips_spectrum(int32_t n, int32_t m); std::complex hTilde_0(uint32_t n, uint32_t m); std::complex rndGaussian(); @@ -164,10 +163,7 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr fft_tilde_h_kt_dx; std::unique_ptr fft_tilde_h_kt_dy; std::unique_ptr fft_tilde_h_kt_dz; - std::unique_ptr fft_displacement_y; - std::unique_ptr fft_displacement_x; - std::unique_ptr fft_displacement_z; - + std::unique_ptr fft_displacement; } fft_buffers; struct @@ -198,10 +194,7 @@ class SubgroupsOperations : public ApiVulkanSample struct { VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; - VkDescriptorSet descriptor_set_asix_y = {VK_NULL_HANDLE}; - VkDescriptorSet descriptor_set_asix_x = {VK_NULL_HANDLE}; - VkDescriptorSet descriptor_set_asix_z = {VK_NULL_HANDLE}; - + VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; Pipeline pipeline; } fft_inversion; @@ -237,6 +230,7 @@ class SubgroupsOperations : public ApiVulkanSample private: uint32_t reverse(uint32_t i); + VkDescriptorImageInfo create_fb_descriptor(FBAttachment &attachment); }; std::unique_ptr create_subgroups_operations(); diff --git a/shaders/subgroups_operations/fft_invert.comp b/shaders/subgroups_operations/fft_invert.comp index d6a14b17c..0538c8fdc 100644 --- a/shaders/subgroups_operations/fft_invert.comp +++ b/shaders/subgroups_operations/fft_invert.comp @@ -25,13 +25,13 @@ layout (binding = 0, rgba32f) writeonly uniform image2D u_displacement; layout (binding = 1, rgba32f) readonly uniform image2D u_pingpong0_y; layout (binding = 2, rgba32f) readonly uniform image2D u_pingpong1_y; -//layout (binding = 3, rgba32f) readonly uniform image2D u_pingpong0_x; -//layout (binding = 4, rgba32f) readonly uniform image2D u_pingpong1_x; -// -//layout (binding = 5, rgba32f) readonly uniform image2D u_pingpong0_z; -//layout (binding = 6, rgba32f) readonly uniform image2D u_pingpong1_z; +layout (binding = 3, rgba32f) readonly uniform image2D u_pingpong0_x; +layout (binding = 4, rgba32f) readonly uniform image2D u_pingpong1_x; -layout (binding = 3) uniform InvertFft +layout (binding = 5, rgba32f) readonly uniform image2D u_pingpong0_z; +layout (binding = 6, rgba32f) readonly uniform image2D u_pingpong1_z; + +layout (binding = 7) uniform InvertFft { int pong_idx; uint grid_size; @@ -51,14 +51,15 @@ void main() if (pingpong_index == 0) { float h_y = imageLoad(u_pingpong0_y, pixel_pos).r; - // float h_x = imageLoad(u_pingpong0_x, pixel_pos).r; - // float h_z = imageLoad(u_pingpong0_z, pixel_pos).r; - // imageStore(u_displacement, pixel_pos, vec4(perm * (h_y / float(N * N)), perm * (h_x / float(N * N)), perm * (h_z / float(N * N)), 1.0f)); - imageStore(u_displacement, pixel_pos, vec4(perm * (h_y / float(N * N)), 0.0f, 0.0f, 1.0f)); + float h_x = imageLoad(u_pingpong0_x, pixel_pos).r; + float h_z = imageLoad(u_pingpong0_z, pixel_pos).r; + imageStore(u_displacement, pixel_pos, vec4(perm * (h_y / float(N * N)), perm * (h_x / float(N * N)), perm * (h_z / float(N * N)), 1.0f)); } else if (pingpong_index == 1) { - float h = imageLoad(u_pingpong1_y, pixel_pos).r; - imageStore(u_displacement, pixel_pos, vec4(perm * (h / float(N * N)), 0, 0, 1)); + float h_y = imageLoad(u_pingpong1_y, pixel_pos).r; + float h_x = imageLoad(u_pingpong1_x, pixel_pos).r; + float h_z = imageLoad(u_pingpong1_z, pixel_pos).r; + imageStore(u_displacement, pixel_pos, vec4(perm * (h_y / float(N * N)), perm * (h_x / float(N * N)), perm * (h_z / float(N * N)), 1)); } } diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 0f49159d4..d10105a04 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -19,11 +19,8 @@ layout (vertices = 4) out; - void main() { - gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position; - if (gl_InvocationID == 0) { gl_TessLevelOuter[0] = 1.0f; @@ -31,7 +28,10 @@ gl_TessLevelOuter[2] = 1.0f; gl_TessLevelOuter[3] = 1.0f; - gl_TessLevelInner[0] = 1.0f; - gl_TessLevelInner[1] = 1.0f; + + gl_TessLevelInner[0] = 10.0f; + gl_TessLevelInner[1] = 10.0f; } + + gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position; } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 05a4a46c7..094bb540a 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -16,10 +16,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - layout(quads, equal_spacing, ccw) in; +layout(quads, equal_spacing, ccw) in; - - layout (binding = 0) uniform Ubo +layout (binding = 0) uniform Ubo { mat4 projection; mat4 view; @@ -27,8 +26,8 @@ } ubo; - void main() - { +void main() +{ vec4 p00 = gl_in[0].gl_Position; vec4 p01 = gl_in[1].gl_Position; vec4 p10 = gl_in[2].gl_Position; @@ -40,4 +39,4 @@ vec4 p = (p1 - p0) * gl_TessCoord.y + p0; gl_Position = ubo.projection * ubo.view * ubo.model * p; - } \ No newline at end of file +} \ No newline at end of file From c28f3225607aef93810901f86fc91b8d541f202b Mon Sep 17 00:00:00 2001 From: Krzysztof Dmitruk Date: Wed, 4 Oct 2023 17:36:09 +0200 Subject: [PATCH 22/53] Fix tilde_h_0 indexation --- .../subgroups_operations/subgroups_operations.cpp | 13 ++++++++----- .../subgroups_operations/subgroups_operations.h | 6 +++--- shaders/subgroups_operations/fft.comp | 2 +- shaders/subgroups_operations/fft_tilde_h.comp | 5 ++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index d3b64601a..41092ef07 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -373,8 +373,8 @@ void SubgroupsOperations::load_assets() { for (uint32_t n = 0; n < grid_size; ++n) { - h_tilde_0.push_back(hTilde_0(n, m)); - h_tilde_0_conj.push_back(std::conj(hTilde_0(-n, -m))); + h_tilde_0.push_back(hTilde_0(n, m, 1)); + h_tilde_0_conj.push_back(std::conj(hTilde_0(n, m, -1))); } } @@ -409,10 +409,12 @@ void SubgroupsOperations::load_assets() fft_buffers.fft_input_weight->update(weights.data(), fft_input_weights_size); } -float SubgroupsOperations::phillips_spectrum(int32_t n, int32_t m) +float SubgroupsOperations::phillips_spectrum(int32_t n, int32_t m, float sign) { glm::vec2 k(glm::pi() * (2.0f * n - grid_size) / ui.length, glm::pi() * (2.0f * m - grid_size) / ui.length); + k = k * sign; + float k_len = glm::length(k); if (k_len < 0.000001f) return 0.000001f; @@ -459,10 +461,11 @@ std::complex SubgroupsOperations::rndGaussian() return {x1 * w, x2 * w}; } -std::complex SubgroupsOperations::hTilde_0(uint32_t n, uint32_t m) +std::complex SubgroupsOperations::hTilde_0(uint32_t n, uint32_t m, float sign) { std::complex rnd = rndGaussian(); - return rnd * glm::sqrt(phillips_spectrum(n, m) / 2.0f); + + return rnd * glm::sqrt(phillips_spectrum(n, m, sign) / 2.0f); } void SubgroupsOperations::prepare_uniform_buffers() diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index cedc66bf5..5eaa0cb97 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -65,8 +65,8 @@ class SubgroupsOperations : public ApiVulkanSample void update_uniform_buffers(); - float phillips_spectrum(int32_t n, int32_t m); - std::complex hTilde_0(uint32_t n, uint32_t m); + float phillips_spectrum(int32_t n, int32_t m, float sign); + std::complex hTilde_0(uint32_t n, uint32_t m, float sign); std::complex rndGaussian(); std::complex calculate_weight(uint32_t x, uint32_t n); @@ -121,7 +121,7 @@ class SubgroupsOperations : public ApiVulkanSample bool wireframe = {false}; float amplitude = {5.0f}; float length = {1000.0f}; - glm::vec2 wind = {100.0f, 0.0f}; + glm::vec2 wind = {100.0f, -100.0f}; } ui; uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; diff --git a/shaders/subgroups_operations/fft.comp b/shaders/subgroups_operations/fft.comp index 5e88a5fd3..1bf14c70e 100644 --- a/shaders/subgroups_operations/fft.comp +++ b/shaders/subgroups_operations/fft.comp @@ -139,7 +139,7 @@ void VerticalButterfiles(in ivec2 pixel_pos) void main() { - ivec2 uv = ivec2(gl_GlobalInvocationID.xy * 0.5f); + ivec2 uv = ivec2(gl_GlobalInvocationID.xy); if (direction == 0) HorizontalButterflies(uv); else if (direction == 1) VerticalButterfiles(uv); } diff --git a/shaders/subgroups_operations/fft_tilde_h.comp b/shaders/subgroups_operations/fft_tilde_h.comp index 826c4bf20..b2a16a370 100644 --- a/shaders/subgroups_operations/fft_tilde_h.comp +++ b/shaders/subgroups_operations/fft_tilde_h.comp @@ -96,9 +96,9 @@ void main() float w = sqrt(GRAVITY * k_mag); ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - uvec2 uv_idx = uvec2(gl_GlobalInvocationID.xy * 0.5f); + uvec2 uv_idx = uvec2(gl_GlobalInvocationID.xy * 0.5); - uint idx = uv_idx.x * uv_idx.y; + uint idx = uv_idx.x + uv_idx.y * fftUbo.grid_size; vec2 h0_k = tilde_h0_k.data[idx]; Complex amp = Complex(h0_k.x, h0_k.y); @@ -109,7 +109,6 @@ void main() Complex amp_conj = Complex(h0_minus_k.x, h0_minus_k.y); Complex exp_minus_iwt = Complex(cos(w * t.time), -sin(w * t.time)); - Complex h_k_t_dy = complex_add( complex_multiply(amp, exp_iwt), complex_multiply(amp_conj, exp_minus_iwt)); imageStore(tilde_h_kt_dy, pixel_pos, vec4(h_k_t_dy.real, h_k_t_dy.imag, 0.0f, 1.0f)); From dc95647782fe9c13650064309234f1b97dbec5c8 Mon Sep 17 00:00:00 2001 From: Krzysztof-Dmitruk-Mobica Date: Fri, 6 Oct 2023 09:14:34 +0200 Subject: [PATCH 23/53] Fix log_2_N rounding --- .../extensions/subgroups_operations/subgroups_operations.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 41092ef07..792c642d4 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -364,7 +364,7 @@ void SubgroupsOperations::create_tildas() void SubgroupsOperations::load_assets() { generate_plane(); - log_2_N = glm::log(grid_size) / glm::log(2.0f); + log_2_N = glm::log2(static_cast(grid_size)); // generate fft inputs h_tilde_0.clear(); @@ -1102,4 +1102,4 @@ VkDescriptorImageInfo SubgroupsOperations::create_fb_descriptor(FBAttachment &at image_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; image_descriptor.sampler = nullptr; return image_descriptor; -} \ No newline at end of file +} From d2a5b521d5c32224c4e0c603c58b27695502bada Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 11 Oct 2023 09:23:59 +0200 Subject: [PATCH 24/53] Some fixes, add tessellation --- .../subgroups_operations.cpp | 27 +++--- .../subgroups_operations.h | 50 +++++----- .../butterfly_precomp.comp | 96 +++++++++++-------- shaders/subgroups_operations/fft.comp | 1 - shaders/subgroups_operations/fft_invert.comp | 7 +- shaders/subgroups_operations/fft_tilde_h.comp | 7 +- shaders/subgroups_operations/ocean.frag | 3 +- shaders/subgroups_operations/ocean.tesc | 29 +++--- shaders/subgroups_operations/ocean.tese | 10 +- shaders/subgroups_operations/ocean.vert | 8 -- 10 files changed, 121 insertions(+), 117 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 792c642d4..2042ddf3b 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -200,7 +200,7 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline_layout, 0u, 1u, &precompute.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, 8u, DISPLACEMENT_MAP_DIM, 1u); + vkCmdDispatch(compute.command_buffer, 1u, DISPLACEMENT_MAP_DIM, 1u); } // tildas textures @@ -319,9 +319,8 @@ void SubgroupsOperations::create_tildas() vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 3u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 4u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 5u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u) + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u)}; - }; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &tildas.descriptor_set_layout)); @@ -492,13 +491,16 @@ void SubgroupsOperations::generate_plane() v.pos = {10.0f, -10.0f, 0.0f}; plane_vertices.push_back(v); - std::vector indices = {0, 1, 2, 2, 3, 0}; + std::vector indices = {0, 1, 3, 1, 2, 3}; auto vertex_buffer_size = vkb::to_u32(plane_vertices.size() * sizeof(Vertex)); auto index_buffer_size = vkb::to_u32(indices.size() * sizeof(uint32_t)); ocean.grid.index_count = vkb::to_u32(indices.size()); - ocean.grid.vertex = std::make_unique(get_device(), vertex_buffer_size, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + ocean.grid.vertex = std::make_unique(get_device(), + vertex_buffer_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); ocean.grid.index = std::make_unique(get_device(), index_buffer_size, @@ -519,11 +521,11 @@ void SubgroupsOperations::create_semaphore() void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 20u), // FFTParametersUbo, CameraUbo, ComputeUbo - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 20u), - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 20u)}; + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 5u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 7u)}; VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 10u); + vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 7u); VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } @@ -593,7 +595,7 @@ void SubgroupsOperations::create_pipelines() dynamic_state_enables.data(), static_cast(dynamic_state_enables.size()), 0u); - VkPipelineTessellationStateCreateInfo tessellation_state = vkb::initializers::pipeline_tessellation_state_create_info(4); + VkPipelineTessellationStateCreateInfo tessellation_state = vkb::initializers::pipeline_tessellation_state_create_info(3u); std::array shader_stages; shader_stages[0] = load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT); @@ -936,11 +938,10 @@ void SubgroupsOperations::create_butterfly_texture() bit_reverse_arr[i] = reverse(i); bit_reverse_buffer = std::make_unique(get_device(), sizeof(uint32_t) * bit_reverse_arr.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + bit_reverse_buffer->update(bit_reverse_arr.data(), sizeof(uint32_t) * bit_reverse_arr.size()); VkDescriptorBufferInfo bit_reverse_descriptor = create_descriptor(*bit_reverse_buffer); - bit_reverse_buffer->convert_and_update(bit_reverse_arr.data(), sizeof(uint32_t) * bit_reverse_arr.size()); - - VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); + VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor), diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 5eaa0cb97..d48e9cc0b 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -80,8 +80,8 @@ class SubgroupsOperations : public ApiVulkanSample struct GridBuffers { - std::unique_ptr vertex; - std::unique_ptr index; + std::unique_ptr vertex = {VK_NULL_HANDLE}; + std::unique_ptr index = {VK_NULL_HANDLE}; uint32_t index_count = {0u}; }; @@ -124,12 +124,12 @@ class SubgroupsOperations : public ApiVulkanSample glm::vec2 wind = {100.0f, -100.0f}; } ui; - uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; - std::unique_ptr camera_ubo; - std::unique_ptr fft_params_ubo; - std::unique_ptr fft_time_ubo; - std::unique_ptr fft_page_ubo; - std::unique_ptr invert_fft_ubo; + uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; + std::unique_ptr camera_ubo = {VK_NULL_HANDLE}; + std::unique_ptr fft_params_ubo = {VK_NULL_HANDLE}; + std::unique_ptr fft_time_ubo = {VK_NULL_HANDLE}; + std::unique_ptr fft_page_ubo = {VK_NULL_HANDLE}; + std::unique_ptr invert_fft_ubo = {VK_NULL_HANDLE}; std::vector> h_tilde_0; std::vector> h_tilde_0_conj; @@ -151,19 +151,17 @@ class SubgroupsOperations : public ApiVulkanSample uint32_t log_2_N; - std::unique_ptr bit_reverse_buffer; - - void createFBAttachement(VkFormat format, uint32_t width, uint32_t height, FBAttachment &result); + std::unique_ptr bit_reverse_buffer = {VK_NULL_HANDLE}; struct { - std::unique_ptr fft_input_htilde0; - std::unique_ptr fft_input_htilde0_conj; - std::unique_ptr fft_input_weight; - std::unique_ptr fft_tilde_h_kt_dx; - std::unique_ptr fft_tilde_h_kt_dy; - std::unique_ptr fft_tilde_h_kt_dz; - std::unique_ptr fft_displacement; + std::unique_ptr fft_input_htilde0 = {VK_NULL_HANDLE}; + std::unique_ptr fft_input_htilde0_conj = {VK_NULL_HANDLE}; + std::unique_ptr fft_input_weight = {VK_NULL_HANDLE}; + std::unique_ptr fft_tilde_h_kt_dx = {VK_NULL_HANDLE}; + std::unique_ptr fft_tilde_h_kt_dy = {VK_NULL_HANDLE}; + std::unique_ptr fft_tilde_h_kt_dz = {VK_NULL_HANDLE}; + std::unique_ptr fft_displacement = {VK_NULL_HANDLE}; } fft_buffers; struct @@ -181,21 +179,23 @@ class SubgroupsOperations : public ApiVulkanSample VkDescriptorSet descriptor_set_axis_y = {VK_NULL_HANDLE}; VkDescriptorSet descriptor_set_axis_x = {VK_NULL_HANDLE}; VkDescriptorSet descriptor_set_axis_z = {VK_NULL_HANDLE}; + struct { Pipeline horizontal; Pipeline vertical; } pipelines; - std::unique_ptr tilde_axis_y; - std::unique_ptr tilde_axis_x; - std::unique_ptr tilde_axis_z; + + std::unique_ptr tilde_axis_y = {VK_NULL_HANDLE}; + std::unique_ptr tilde_axis_x = {VK_NULL_HANDLE}; + std::unique_ptr tilde_axis_z = {VK_NULL_HANDLE}; } fft; struct { VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; - VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; - Pipeline pipeline; + VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; + Pipeline pipeline; } fft_inversion; struct @@ -219,6 +219,7 @@ class SubgroupsOperations : public ApiVulkanSample VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; VkSemaphore semaphore = {VK_NULL_HANDLE}; + struct { Pipeline _default; @@ -229,8 +230,9 @@ class SubgroupsOperations : public ApiVulkanSample VkPhysicalDeviceSubgroupProperties subgroups_properties; private: - uint32_t reverse(uint32_t i); + uint32_t reverse(uint32_t i); VkDescriptorImageInfo create_fb_descriptor(FBAttachment &attachment); + void createFBAttachement(VkFormat format, uint32_t width, uint32_t height, FBAttachment &result); }; std::unique_ptr create_subgroups_operations(); diff --git a/shaders/subgroups_operations/butterfly_precomp.comp b/shaders/subgroups_operations/butterfly_precomp.comp index e8debeb90..aff4f1979 100644 --- a/shaders/subgroups_operations/butterfly_precomp.comp +++ b/shaders/subgroups_operations/butterfly_precomp.comp @@ -1,68 +1,84 @@ -#version 460 core +#version 450 +#extension GL_KHR_shader_subgroup_basic : enable +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #define PI_32F 3.14159265358979f #define DISPLACEMENT_MAP_DIM 256 -layout (local_size_x = 8, local_size_y = 32) in; +layout (local_size_x = 8, local_size_y = 1) in; layout (binding = 0, rgba32f) writeonly uniform image2D u_butterfly_precomp; layout (std430, binding = 1) buffer indices { - int data[]; -} -bit_reversed_indices; + int data[]; +} bit_reversed_indices; void main() { - vec2 pos = gl_GlobalInvocationID.xy; + vec2 pos = gl_GlobalInvocationID.xy; - int N = DISPLACEMENT_MAP_DIM; + int N = DISPLACEMENT_MAP_DIM; - // Twiddle factor exponent, Thesis, Section 4.2.6, Eq 4.6 - float k = mod(pos.y * (float(N) / pow(2, pos.x + 1)), N); + // Twiddle factor exponent, Thesis, Section 4.2.6, Eq 4.6 + float k = mod(pos.y * (float(N) / pow(2, pos.x + 1)), N); - // Thesis, Section 4.2.6, Eq 4.7 - int butterfly_span = int(pow(2, pos.x)); - int butterfly_wing = 0; + // Thesis, Section 4.2.6, Eq 4.7 + int butterfly_span = int(pow(2, pos.x)); + int butterfly_wing = 0; - if ((mod(pos.y, pow(2, pos.x + 1))) < butterfly_span) butterfly_wing = 1; + if ((mod(pos.y, pow(2, pos.x + 1))) < butterfly_span) butterfly_wing = 1; - ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - vec4 result; - result.x = cos(2.f * PI_32F * k / float(N)); // Twiddle factor real part - result.y = sin(2.f * PI_32F * k / float(N)); // Twiddle factor imaginary part + vec4 result; + result.x = cos(2.f * PI_32F * k / float(N)); // Twiddle factor real part + result.y = sin(2.f * PI_32F * k / float(N)); // Twiddle factor imaginary part - // Store the sample indices for the next stage - if (pos.x == 0) + // Store the sample indices for the next stage + if (pos.x == 0) + { + if (butterfly_wing == 1) + { + result.z = bit_reversed_indices.data[int(pos.y)]; + result.w = bit_reversed_indices.data[int(pos.y + 1)]; + } + else + { + result.z = bit_reversed_indices.data[int(pos.y - 1)]; + result.w = bit_reversed_indices.data[int(pos.y)]; + } + } + else + { + if (butterfly_wing == 1) { - if (butterfly_wing == 1) - { - result.z = bit_reversed_indices.data[int(pos.y)]; - result.w = bit_reversed_indices.data[int(pos.y + 1)]; - } - else - { - result.z = bit_reversed_indices.data[int(pos.y - 1)]; - result.w = bit_reversed_indices.data[int(pos.y)]; - } + result.z = pos.y; + result.w = pos.y + butterfly_span; } else { - if (butterfly_wing == 1) - { - result.z = pos.y; - result.w = pos.y + butterfly_span; - } - else - { - result.z = pos.y - butterfly_span; - result.w = pos.y; - } + result.z = pos.y - butterfly_span; + result.w = pos.y; } + } - imageStore(u_butterfly_precomp, pixel_pos, result); + imageStore(u_butterfly_precomp, pixel_pos, result); } diff --git a/shaders/subgroups_operations/fft.comp b/shaders/subgroups_operations/fft.comp index 1bf14c70e..9d906f519 100644 --- a/shaders/subgroups_operations/fft.comp +++ b/shaders/subgroups_operations/fft.comp @@ -1,6 +1,5 @@ #version 450 #extension GL_KHR_shader_subgroup_basic : enable - /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 diff --git a/shaders/subgroups_operations/fft_invert.comp b/shaders/subgroups_operations/fft_invert.comp index 0538c8fdc..7d0a1a5ad 100644 --- a/shaders/subgroups_operations/fft_invert.comp +++ b/shaders/subgroups_operations/fft_invert.comp @@ -1,6 +1,5 @@ #version 450 #extension GL_KHR_shader_subgroup_basic : enable - /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 @@ -42,7 +41,7 @@ void main() uint N = fftUbo.grid_size; - ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy * 0.5f); + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); float perms[2] = { 1.0, -1.0 }; int index = int(mod(pixel_pos.x + pixel_pos.y, 2)); @@ -53,13 +52,13 @@ void main() float h_y = imageLoad(u_pingpong0_y, pixel_pos).r; float h_x = imageLoad(u_pingpong0_x, pixel_pos).r; float h_z = imageLoad(u_pingpong0_z, pixel_pos).r; - imageStore(u_displacement, pixel_pos, vec4(perm * (h_y / float(N * N)), perm * (h_x / float(N * N)), perm * (h_z / float(N * N)), 1.0f)); + imageStore(u_displacement, pixel_pos, vec4(perm * (h_x / float(N * N)), perm * (h_y / float(N * N)), perm * (h_z / float(N * N)), 1.0f)); } else if (pingpong_index == 1) { float h_y = imageLoad(u_pingpong1_y, pixel_pos).r; float h_x = imageLoad(u_pingpong1_x, pixel_pos).r; float h_z = imageLoad(u_pingpong1_z, pixel_pos).r; - imageStore(u_displacement, pixel_pos, vec4(perm * (h_y / float(N * N)), perm * (h_x / float(N * N)), perm * (h_z / float(N * N)), 1)); + imageStore(u_displacement, pixel_pos, vec4(perm * (h_x / float(N * N)), perm * (h_y / float(N * N)), perm * (h_z / float(N * N)), 1)); } } diff --git a/shaders/subgroups_operations/fft_tilde_h.comp b/shaders/subgroups_operations/fft_tilde_h.comp index b2a16a370..dd5dc0bbd 100644 --- a/shaders/subgroups_operations/fft_tilde_h.comp +++ b/shaders/subgroups_operations/fft_tilde_h.comp @@ -1,9 +1,5 @@ #version 450 #extension GL_KHR_shader_subgroup_basic : enable - -#define PI 3.141592f -#define GRAVITY 9.81f - /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 @@ -21,6 +17,9 @@ * limitations under the License. */ +#define PI 3.141592f +#define GRAVITY 9.81f + layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; layout (std140, binding = 0) readonly buffer TildeH0K diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index 58416b076..b44fda738 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -16,8 +16,7 @@ * limitations under the License. */ - - layout (location = 0) out vec4 outFragColor; +layout (location = 0) out vec4 outFragColor; void main() { diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index d10105a04..34f45b402 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -1,5 +1,4 @@ #version 450 - /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 @@ -17,21 +16,19 @@ * limitations under the License. */ - layout (vertices = 4) out; - - void main() - { - if (gl_InvocationID == 0) - { - gl_TessLevelOuter[0] = 1.0f; - gl_TessLevelOuter[1] = 1.0f; - gl_TessLevelOuter[2] = 1.0f; - gl_TessLevelOuter[3] = 1.0f; +layout (vertices = 3) out; +void main() +{ + if (gl_InvocationID == 0) + { + gl_TessLevelOuter[0] = 1.0f; + gl_TessLevelOuter[1] = 1.0f; + gl_TessLevelOuter[2] = 1.0f; - gl_TessLevelInner[0] = 10.0f; - gl_TessLevelInner[1] = 10.0f; - } + gl_TessLevelInner[0] = 2.0f; + gl_TessLevelInner[1] = 1.0f; + } - gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position; - } \ No newline at end of file + gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position; +} \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 094bb540a..3705bff02 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -1,5 +1,4 @@ #version 450 - /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 @@ -16,7 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -layout(quads, equal_spacing, ccw) in; + +layout(triangles) in; layout (binding = 0) uniform Ubo { @@ -33,10 +33,10 @@ void main() vec4 p10 = gl_in[2].gl_Position; vec4 p11 = gl_in[3].gl_Position; - vec4 p0 = (p01 - p00) * gl_TessCoord.x + p00; - vec4 p1 = (p11 - p10) * gl_TessCoord.x + p10; + vec4 p0 = mix(p00, p01, gl_TessCoord.x); + vec4 p1 = mix(p10, p11, gl_TessCoord.x); - vec4 p = (p1 - p0) * gl_TessCoord.y + p0; + vec4 p = mix(p0, p1, gl_TessCoord.y); gl_Position = ubo.projection * ubo.view * ubo.model * p; } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.vert b/shaders/subgroups_operations/ocean.vert index a65d54dd6..a18289a59 100644 --- a/shaders/subgroups_operations/ocean.vert +++ b/shaders/subgroups_operations/ocean.vert @@ -18,14 +18,6 @@ layout (location = 0) in vec3 inPos; -layout (binding = 0) uniform Ubo -{ - mat4 projection; - mat4 view; - mat4 model; -} ubo; - - void main() { gl_Position = vec4(inPos.xyz, 1.0f); From 09dc3a9d984e0ece94e8742ec22ab4742f51ae77 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 11 Oct 2023 13:04:26 +0200 Subject: [PATCH 25/53] Fix a buttlefly pipeline --- .../subgroups_operations/subgroups_operations.cpp | 8 ++++++-- .../subgroups_operations/butterfly_precomp.comp | 14 +++++++++++--- shaders/subgroups_operations/ocean.tesc | 3 +-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 2042ddf3b..c7bf3ca5e 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -909,8 +909,9 @@ uint32_t SubgroupsOperations::reverse(uint32_t i) void SubgroupsOperations::create_butterfly_texture() { std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u)}; + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u)}; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &precompute.descriptor_set_layout)); @@ -942,10 +943,13 @@ void SubgroupsOperations::create_butterfly_texture() VkDescriptorBufferInfo bit_reverse_descriptor = create_descriptor(*bit_reverse_buffer); VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); + VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor), - vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &bit_reverse_descriptor)}; + vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &bit_reverse_descriptor), + vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &fft_params_ubo_buffer), + }; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } diff --git a/shaders/subgroups_operations/butterfly_precomp.comp b/shaders/subgroups_operations/butterfly_precomp.comp index aff4f1979..03ca176f3 100644 --- a/shaders/subgroups_operations/butterfly_precomp.comp +++ b/shaders/subgroups_operations/butterfly_precomp.comp @@ -18,7 +18,6 @@ */ #define PI_32F 3.14159265358979f -#define DISPLACEMENT_MAP_DIM 256 layout (local_size_x = 8, local_size_y = 1) in; @@ -29,12 +28,21 @@ layout (std430, binding = 1) buffer indices int data[]; } bit_reversed_indices; +layout (binding = 2) uniform FFTParametersUbo +{ + float amplitude; + float len; + uint grid_size; + vec2 wind; +} fftUbo; + + void main() { vec2 pos = gl_GlobalInvocationID.xy; - int N = DISPLACEMENT_MAP_DIM; + uint N = fftUbo.grid_size; // Twiddle factor exponent, Thesis, Section 4.2.6, Eq 4.6 float k = mod(pos.y * (float(N) / pow(2, pos.x + 1)), N); @@ -47,7 +55,7 @@ void main() ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - vec4 result; + vec4 result = vec4(0.0f); result.x = cos(2.f * PI_32F * k / float(N)); // Twiddle factor real part result.y = sin(2.f * PI_32F * k / float(N)); // Twiddle factor imaginary part diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 34f45b402..fa2fab57d 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -26,8 +26,7 @@ void main() gl_TessLevelOuter[1] = 1.0f; gl_TessLevelOuter[2] = 1.0f; - gl_TessLevelInner[0] = 2.0f; - gl_TessLevelInner[1] = 1.0f; + gl_TessLevelInner[0] = 15.0f; } gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position; From 5524b442e3fd75d844c2fa06128196f6751c8edf Mon Sep 17 00:00:00 2001 From: Krzysztof-Dmitruk-Mobica Date: Tue, 10 Oct 2023 16:03:25 +0200 Subject: [PATCH 26/53] Add tilde_h_0 shader --- .../subgroups_operations/CMakeLists.txt | 1 + .../subgroups_operations.cpp | 249 ++++++++++++------ .../subgroups_operations.h | 24 +- .../butterfly_precomp.comp | 5 + shaders/subgroups_operations/fft.comp | 25 +- shaders/subgroups_operations/fft_invert.comp | 7 + shaders/subgroups_operations/fft_tilde_h.comp | 143 ++++++---- .../subgroups_operations/fft_tilde_h0.comp | 52 ++++ 8 files changed, 365 insertions(+), 141 deletions(-) create mode 100644 shaders/subgroups_operations/fft_tilde_h0.comp diff --git a/samples/extensions/subgroups_operations/CMakeLists.txt b/samples/extensions/subgroups_operations/CMakeLists.txt index 0bdd3e306..68184909e 100644 --- a/samples/extensions/subgroups_operations/CMakeLists.txt +++ b/samples/extensions/subgroups_operations/CMakeLists.txt @@ -33,4 +33,5 @@ add_sample( "subgroups_operations/fft_invert.comp" "subgroups_operations/butterfly_precomp.comp" "subgroups_operations/fft_tilde_h.comp" + "subgroups_operations/fft_tilde_h0.comp" "subgroups_operations/fft.comp") diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index c7bf3ca5e..3fe3cb255 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -119,6 +119,7 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) create_descriptor_set(); create_pipelines(); + create_initial_tildas(); create_tildas(); create_butterfly_texture(); create_fft(); @@ -182,97 +183,115 @@ void SubgroupsOperations::build_compute_command_buffer() VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); + /* if (compute.queue_family_index != ocean.graphics_queue_family_index) { - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.srcAccessMask = 0u; - memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; - memory_barrier.buffer = fft_buffers.fft_input_htilde0->get_handle(); - memory_barrier.offset = 0u; - memory_barrier.size = fft_buffers.fft_input_htilde0->get_size(); - - vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); + + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.srcAccessMask = 0u; + memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; + memory_barrier.buffer = fft_buffers.fft_input_htilde0->get_handle(); + memory_barrier.offset = 0u; + memory_barrier.size = fft_buffers.fft_input_htilde0->get_size(); + + vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); } + */ + // buttle fly texture { vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline_layout, 0u, 1u, &precompute.descriptor_set, 0u, nullptr); vkCmdDispatch(compute.command_buffer, 1u, DISPLACEMENT_MAP_DIM, 1u); +<<<<<<< HEAD +======= } - // tildas textures + // initial tildas textures { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline_layout, 0u, 1u, &tildas.descriptor_set, 0u, nullptr); + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, initial_tildas.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, initial_tildas.pipeline.pipeline_layout, 0u, 1u, &initial_tildas.descriptor_set, 0u, nullptr); vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); +>>>>>>> e005a65 (Add tilde_h_0 shader) } - // fft horizontal; for Y axis + // tildas textures { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline_layout, 0u, 1u, &tildas.descriptor_set, 0u, nullptr); - // fft horizontal; for X axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 8u, DISPLACEMENT_MAP_DIM, 1u); } - // fft horizontal; for Z axis + // fft horizontal; for Y axis { vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } - - // fft vertical; for Y axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } - - // fft vertical; for X axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } - - // fft vertical; for Z axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); } - // fft inverse - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } - - if (compute.queue_family_index != ocean.graphics_queue_family_index) - { - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - memory_barrier.dstAccessMask = 0u; - memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; - memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.buffer = fft_buffers.fft_input_htilde0->get_handle(); - memory_barrier.offset = 0u; - memory_barrier.size = fft_buffers.fft_input_htilde0->get_size(); - - vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); - } + /* + // fft horizontal; for X axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + // fft horizontal; for Z axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + // fft vertical; for Y axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + // fft vertical; for X axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + // fft vertical; for Z axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + + // fft inverse + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + */ + /* + if (compute.queue_family_index != ocean.graphics_queue_family_index) + { + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + memory_barrier.dstAccessMask = 0u; + memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; + memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.buffer = fft_buffers.fft_input_htilde0->get_handle(); + memory_barrier.offset = 0u; + memory_barrier.size = fft_buffers.fft_input_htilde0->get_size(); + + vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); + } + */ VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); } @@ -302,6 +321,47 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); } +void SubgroupsOperations::create_initial_tildas() +{ + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + }; + + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &initial_tildas.descriptor_set_layout)); + + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &initial_tildas.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &initial_tildas.descriptor_set)); + + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &initial_tildas.descriptor_set_layout; + + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &initial_tildas.pipeline.pipeline_layout)); + + VkComputePipelineCreateInfo computeInfo = vkb::initializers::compute_pipeline_create_info(initial_tildas.pipeline.pipeline_layout); + computeInfo.stage = load_shader("subgroups_operations/fft_tilde_h0.comp", VK_SHADER_STAGE_COMPUTE_BIT); + + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &initial_tildas.pipeline.pipeline)); + + fft_buffers.fft_input_htilde0 = std::make_unique(); + fft_buffers.fft_input_htilde0_conj = std::make_unique(); + + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_input_htilde0); + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_input_htilde0_conj); + + VkDescriptorImageInfo htilde_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0); + VkDescriptorImageInfo htilde_conj_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0_conj); + + std::vector write_descriptor_sets = { + vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &htilde_0_descriptor), + vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &htilde_conj_0_descriptor), + }; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); +} + void SubgroupsOperations::create_tildas() { fft_buffers.fft_tilde_h_kt_dx = std::make_unique(); @@ -313,14 +373,17 @@ void SubgroupsOperations::create_tildas() createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dz); std::vector set_layout_bindngs = { - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 0u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 3u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 4u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 5u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u)}; +<<<<<<< HEAD +======= +>>>>>>> e005a65 (Add tilde_h_0 shader) VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &tildas.descriptor_set_layout)); @@ -339,8 +402,8 @@ void SubgroupsOperations::create_tildas() VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &tildas.pipeline.pipeline)); - VkDescriptorBufferInfo htilde_0_descriptor = create_descriptor(*fft_buffers.fft_input_htilde0); - VkDescriptorBufferInfo htilde_conj_0_descriptor = create_descriptor(*fft_buffers.fft_input_htilde0_conj); + VkDescriptorImageInfo htilde_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0); + VkDescriptorImageInfo htilde_conj_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0_conj); VkDescriptorImageInfo image_dx_descriptor = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dx); VkDescriptorImageInfo image_dy_descriptor = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dy); @@ -350,8 +413,8 @@ void SubgroupsOperations::create_tildas() VkDescriptorBufferInfo fft_time_ubo_buffer = create_descriptor(*fft_time_ubo); std::vector write_descriptor_sets = { - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 0u, &htilde_0_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &htilde_conj_0_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &htilde_0_descriptor), + vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &htilde_conj_0_descriptor), vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_dx_descriptor), vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 3u, &image_dy_descriptor), vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &image_dz_descriptor), @@ -365,34 +428,36 @@ void SubgroupsOperations::load_assets() generate_plane(); log_2_N = glm::log2(static_cast(grid_size)); + /* + // generate fft inputs h_tilde_0.clear(); h_tilde_0_conj.clear(); for (uint32_t m = 0; m < grid_size; ++m) { - for (uint32_t n = 0; n < grid_size; ++n) - { - h_tilde_0.push_back(hTilde_0(n, m, 1)); - h_tilde_0_conj.push_back(std::conj(hTilde_0(n, m, -1))); - } + for (uint32_t n = 0; n < grid_size; ++n) + { + h_tilde_0.push_back(hTilde_0(n, m, 1)); + h_tilde_0_conj.push_back(std::conj(hTilde_0(-n, -m, 1))); + } } // calculate weights uint32_t pow2 = 1U; for (uint32_t i = 0U; i < log_2_N; ++i) { - for (uint32_t j = 0U; j < pow2; ++j) - { - weights.push_back(calculate_weight(j, pow2 * 2)); - } - pow2 *= 2; + for (uint32_t j = 0U; j < pow2; ++j) + { + weights.push_back(calculate_weight(j, pow2 * 2)); + } + pow2 *= 2; } auto fft_input_htilde0_size = static_cast(h_tilde_0.size() * sizeof(std::complex)); fft_buffers.fft_input_htilde0 = std::make_unique(get_device(), - fft_input_htilde0_size, - VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, - VMA_MEMORY_USAGE_CPU_TO_GPU); + fft_input_htilde0_size, + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); auto fft_input_htilde0_conj_size = static_cast(h_tilde_0_conj.size() * sizeof(std::complex)); fft_buffers.fft_input_htilde0_conj = std::make_unique(get_device(), @@ -406,6 +471,8 @@ void SubgroupsOperations::load_assets() fft_buffers.fft_input_htilde0->update(h_tilde_0.data(), fft_input_htilde0_size); fft_buffers.fft_input_htilde0_conj->update(h_tilde_0_conj.data(), fft_input_htilde0_conj_size); fft_buffers.fft_input_weight->update(weights.data(), fft_input_weights_size); + + */ } float SubgroupsOperations::phillips_spectrum(int32_t n, int32_t m, float sign) @@ -462,7 +529,8 @@ std::complex SubgroupsOperations::rndGaussian() std::complex SubgroupsOperations::hTilde_0(uint32_t n, uint32_t m, float sign) { - std::complex rnd = rndGaussian(); + std::complex rnd{1.0f, 1.0f}; // rndGaussian(); + // std::complex rnd = rndGaussian(); return rnd * glm::sqrt(phillips_spectrum(n, m, sign) / 2.0f); } @@ -649,7 +717,8 @@ void SubgroupsOperations::update_uniform_buffers() fft_ubo.wind = ui.wind; FFTPage pageSize; - pageSize.page = static_cast(log_2_N); + // pageSize.page = static_cast(log_2_N); + pageSize.page = 6; // static_cast(log_2_N); FFTInvert invertFft; invertFft.grid_size = grid_size; @@ -942,8 +1011,14 @@ void SubgroupsOperations::create_butterfly_texture() bit_reverse_buffer->update(bit_reverse_arr.data(), sizeof(uint32_t) * bit_reverse_arr.size()); VkDescriptorBufferInfo bit_reverse_descriptor = create_descriptor(*bit_reverse_buffer); +<<<<<<< HEAD VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); +======= + bit_reverse_buffer->update(bit_reverse_arr.data(), sizeof(uint32_t) * bit_reverse_arr.size()); + + VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); +>>>>>>> e005a65 (Add tilde_h_0 shader) std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor), diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index d48e9cc0b..ce0807f7b 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -58,6 +58,7 @@ class SubgroupsOperations : public ApiVulkanSample void create_descriptor_set(); void create_pipelines(); + void create_initial_tildas(); void create_tildas(); void create_butterfly_texture(); void create_fft(); @@ -119,7 +120,7 @@ class SubgroupsOperations : public ApiVulkanSample struct GuiConfig { bool wireframe = {false}; - float amplitude = {5.0f}; + float amplitude = {2.0f}; float length = {1000.0f}; glm::vec2 wind = {100.0f, -100.0f}; } ui; @@ -155,6 +156,7 @@ class SubgroupsOperations : public ApiVulkanSample struct { +<<<<<<< HEAD std::unique_ptr fft_input_htilde0 = {VK_NULL_HANDLE}; std::unique_ptr fft_input_htilde0_conj = {VK_NULL_HANDLE}; std::unique_ptr fft_input_weight = {VK_NULL_HANDLE}; @@ -162,6 +164,19 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr fft_tilde_h_kt_dy = {VK_NULL_HANDLE}; std::unique_ptr fft_tilde_h_kt_dz = {VK_NULL_HANDLE}; std::unique_ptr fft_displacement = {VK_NULL_HANDLE}; +======= + // std::unique_ptr fft_input_htilde0; + // std::unique_ptr fft_input_htilde0_conj; + // std::unique_ptr fft_input_weight; + + std::unique_ptr fft_input_htilde0; + std::unique_ptr fft_input_htilde0_conj; + + std::unique_ptr fft_tilde_h_kt_dx; + std::unique_ptr fft_tilde_h_kt_dy; + std::unique_ptr fft_tilde_h_kt_dz; + std::unique_ptr fft_displacement; +>>>>>>> e005a65 (Add tilde_h_0 shader) } fft_buffers; struct @@ -198,6 +213,13 @@ class SubgroupsOperations : public ApiVulkanSample Pipeline pipeline; } fft_inversion; + struct + { + VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; + Pipeline pipeline; + } initial_tildas; + struct { VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; diff --git a/shaders/subgroups_operations/butterfly_precomp.comp b/shaders/subgroups_operations/butterfly_precomp.comp index 03ca176f3..6ea091acd 100644 --- a/shaders/subgroups_operations/butterfly_precomp.comp +++ b/shaders/subgroups_operations/butterfly_precomp.comp @@ -53,7 +53,12 @@ void main() if ((mod(pos.y, pow(2, pos.x + 1))) < butterfly_span) butterfly_wing = 1; +<<<<<<< HEAD ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); +======= + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + //pixel_pos.y = N - pixel_pos.y - 1; +>>>>>>> e005a65 (Add tilde_h_0 shader) vec4 result = vec4(0.0f); result.x = cos(2.f * PI_32F * k / float(N)); // Twiddle factor real part diff --git a/shaders/subgroups_operations/fft.comp b/shaders/subgroups_operations/fft.comp index 9d906f519..f311e6346 100644 --- a/shaders/subgroups_operations/fft.comp +++ b/shaders/subgroups_operations/fft.comp @@ -1,5 +1,13 @@ #version 450 #extension GL_KHR_shader_subgroup_basic : enable +<<<<<<< HEAD +======= +#extension GL_EXT_debug_printf : enable + +precision highp float; + + +>>>>>>> e005a65 (Add tilde_h_0 shader) /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 @@ -52,19 +60,11 @@ Complex complex_multiply(Complex c1, Complex c2) return res; } -Complex complex_conj(Complex c) -{ - Complex res; - res.real = c.real; - res.imag = -c.imag; - return res; -} - - void HorizontalButterflies(in ivec2 pixel_pos) { for (int i = 0; i < fftUbo.pages; ++i) { + subgroupBarrier(); vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(i, pixel_pos.x)); if ((i % 2) == 0) { @@ -79,6 +79,10 @@ void HorizontalButterflies(in ivec2 pixel_pos) Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); subgroupMemoryBarrierImage(); imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + + //if(i == 6) + // debugPrintfEXT("[%i] %.5f, %.5f", i, result.real, result.imag); + } else { @@ -93,6 +97,9 @@ void HorizontalButterflies(in ivec2 pixel_pos) Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); subgroupMemoryBarrierImage(); imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + + if(i == 5) + debugPrintfEXT("[%i] %.5f, %.5f", i, result.real, result.imag); } } } diff --git a/shaders/subgroups_operations/fft_invert.comp b/shaders/subgroups_operations/fft_invert.comp index 7d0a1a5ad..b32126cdc 100644 --- a/shaders/subgroups_operations/fft_invert.comp +++ b/shaders/subgroups_operations/fft_invert.comp @@ -41,7 +41,14 @@ void main() uint N = fftUbo.grid_size; +<<<<<<< HEAD ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); +======= + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + + //pixel_pos.y = N - pixel_pos.y - 1; + +>>>>>>> e005a65 (Add tilde_h_0 shader) float perms[2] = { 1.0, -1.0 }; int index = int(mod(pixel_pos.x + pixel_pos.y, 2)); diff --git a/shaders/subgroups_operations/fft_tilde_h.comp b/shaders/subgroups_operations/fft_tilde_h.comp index dd5dc0bbd..c3e98d1ca 100644 --- a/shaders/subgroups_operations/fft_tilde_h.comp +++ b/shaders/subgroups_operations/fft_tilde_h.comp @@ -1,5 +1,12 @@ #version 450 #extension GL_KHR_shader_subgroup_basic : enable +<<<<<<< HEAD +======= + +#define PI 3.14159265358979f +#define GRAVITY 9.81f + +>>>>>>> e005a65 (Add tilde_h_0 shader) /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 @@ -22,17 +29,22 @@ layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; +/* layout (std140, binding = 0) readonly buffer TildeH0K { - vec2 data[]; + vec2 data[]; } tilde_h0_k; layout (std140, binding = 1) readonly buffer TildeH0MinusK { - vec2 data[]; + vec2 data[]; } tilde_h0_minus_k; +*/ + +layout (binding = 0, rgba32f) readonly uniform image2D u_tilde_h0_k; +layout (binding = 1, rgba32f) readonly uniform image2D u_tilde_h0_minus_k; layout (binding = 2, rgba32f) writeonly uniform image2D tilde_h_kt_dx; layout (binding = 3, rgba32f) writeonly uniform image2D tilde_h_kt_dy; @@ -40,82 +52,125 @@ layout (binding = 4, rgba32f) writeonly uniform image2D tilde_h_kt_dz; layout (binding = 5) uniform FFTParametersUbo { - float amplitude; - float len; - uint grid_size; - vec2 wind; + float amplitude; + float len; + uint grid_size; + vec2 wind; } fftUbo; layout (binding = 6) uniform Time { - float time; + float time; } t; struct Complex { - float real; - float imag; + float real; + float imag; }; Complex complex_add(Complex c1, Complex c2) { - Complex res; - res.real = c1.real + c2.real; - res.imag = c1.imag + c2.imag; - return res; + Complex res; + res.real = c1.real + c2.real; + res.imag = c1.imag + c2.imag; + return res; } Complex complex_multiply(Complex c1, Complex c2) { - Complex res; - res.real = c1.real * c2.real - c1.imag * c2.imag; - res.imag = c1.real * c2.imag + c1.imag * c2.real; - return res; + Complex res; + res.real = c1.real * c2.real - c1.imag * c2.imag; + res.imag = c1.real * c2.imag + c1.imag * c2.real; + return res; } Complex complex_conj(Complex c) { - Complex res; - res.real = c.real; - res.imag = -c.imag; - return res; + Complex res; + res.real = c.real; + res.imag = -c.imag; + return res; } void main() { - float L = fftUbo.len; - vec2 uv = vec2(gl_GlobalInvocationID.xy) - (fftUbo.grid_size / 2.0f); + int N = 256; + int L = 1000; + + vec2 pos = vec2(gl_GlobalInvocationID.xy) - (N / 2.0); + + // Wavevector, Section 4.3, Eq 36 + vec2 k = vec2((2.0 * PI * pos.x) / L, (2.0 * PI * pos.y) / L); + + float k_magnitude = length(k); + if (k_magnitude < 0.00001) k_magnitude = 0.00001; + + // Dispersion relation, Eq 31, Section 4.2 + float w = sqrt(GRAVITY * k_magnitude); + + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + + // First term of Eq 43, Section 4.4 + vec2 h0_k = imageLoad(u_tilde_h0_k, pixel_pos).xy; + Complex amp = Complex(h0_k.x, h0_k.y); + Complex exp_iwt = Complex(cos(w * t.time), sin(w * t.time)); + + // Second term of Eq 43, Section 4.4 + vec2 h0_minus_k = imageLoad(u_tilde_h0_minus_k, pixel_pos).xy; + Complex amp_conj = complex_conj(Complex(h0_minus_k.x, h0_minus_k.y)); + Complex exp_minus_iwt = Complex(cos(w * t.time), -sin(w * t.time)); + + // Section 4.4, Eq 43 + Complex h_k_t_dy = complex_add(complex_multiply(amp, exp_iwt), complex_multiply(amp_conj, exp_minus_iwt)); + imageStore(tilde_h_kt_dy, pixel_pos, vec4(h_k_t_dy.real, h_k_t_dy.imag, 0.0, 1.0)); + + // Section 4.6, Eq 44 + Complex dx = Complex(0.0, -k.x / k_magnitude); + Complex h_k_t_dx = complex_multiply(dx, h_k_t_dy); + imageStore(tilde_h_kt_dx, pixel_pos, vec4(h_k_t_dx.real, h_k_t_dx.imag, 0.0, 1.0)); + + // Section 4.6, Eq 44 + Complex dz = Complex(0.0, -k.y / k_magnitude); + Complex h_k_t_dz = complex_multiply(dz, h_k_t_dy); + imageStore(tilde_h_kt_dz, pixel_pos, vec4(h_k_t_dz.real, h_k_t_dz.imag, 0.0, 1.0)); + +/* + float L = fftUbo.len; + + vec2 uv = vec2(gl_GlobalInvocationID.xy) - (fftUbo.grid_size / 2.0f); - vec2 k = vec2( (2.0f * PI * uv.x) / L, (2.0f * PI * uv.y) / L); + vec2 k = vec2( (2.0f * PI * uv.x) / L, (2.0f * PI * uv.y) / L); - float k_mag = length(k); - if (k_mag < 0.00001f) k_mag = 0.00001f; + float k_mag = length(k); + if (k_mag < 0.00001f) k_mag = 0.00001f; - float w = sqrt(GRAVITY * k_mag); + float w = sqrt(GRAVITY * k_mag); - ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - uvec2 uv_idx = uvec2(gl_GlobalInvocationID.xy * 0.5); + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + //uvec2 uv_idx = uvec2(gl_GlobalInvocationID.xy * 0.5); - uint idx = uv_idx.x + uv_idx.y * fftUbo.grid_size; - vec2 h0_k = tilde_h0_k.data[idx]; + //uint idx = uv_idx.x + uv_idx.y * fftUbo.grid_size; + vec2 h0_k = imageLoad(u_tilde_h0_k, pixel_pos).xy; - Complex amp = Complex(h0_k.x, h0_k.y); - Complex exp_iwt = Complex(cos(w * t.time), sin(w * t.time)); + Complex amp = Complex(h0_k.x, h0_k.y); + Complex exp_iwt = Complex(cos(w * t.time), sin(w * t.time)); - vec2 h0_minus_k = tilde_h0_minus_k.data[idx]; + vec2 h0_minus_k = imageLoad(u_tilde_h0_minus_k, pixel_pos).xy; - Complex amp_conj = Complex(h0_minus_k.x, h0_minus_k.y); - Complex exp_minus_iwt = Complex(cos(w * t.time), -sin(w * t.time)); + Complex amp_conj = complex_conj(Complex(h0_minus_k.x, h0_minus_k.y)); + Complex exp_minus_iwt = Complex(cos(w * t.time), -sin(w * t.time)); - Complex h_k_t_dy = complex_add( complex_multiply(amp, exp_iwt), complex_multiply(amp_conj, exp_minus_iwt)); - imageStore(tilde_h_kt_dy, pixel_pos, vec4(h_k_t_dy.real, h_k_t_dy.imag, 0.0f, 1.0f)); + Complex h_k_t_dy = complex_add( complex_multiply(amp, exp_iwt), complex_multiply(amp_conj, exp_minus_iwt)); + imageStore(tilde_h_kt_dy, pixel_pos, vec4(h_k_t_dy.real, h_k_t_dy.imag, 0.0f, 1.0f)); - Complex dx = Complex(0.0f, -k.x / k_mag); - Complex h_k_t_dx = complex_multiply(dx, h_k_t_dy); - imageStore(tilde_h_kt_dx, pixel_pos, vec4(h_k_t_dx.real, h_k_t_dx.imag, 0.0f, 1.0f)); + Complex dx = Complex(0.0f, -k.x / k_mag); + Complex h_k_t_dx = complex_multiply(dx, h_k_t_dy); + imageStore(tilde_h_kt_dx, pixel_pos, vec4(h_k_t_dx.real, h_k_t_dx.imag, 0.0f, 1.0f)); - Complex dz = Complex(0.0f, -k.y / k_mag); - Complex h_k_t_dz = complex_multiply(dz, h_k_t_dy); - imageStore(tilde_h_kt_dz, pixel_pos, vec4(h_k_t_dz.real, h_k_t_dz.imag, 0.0f, 1.0f)); + Complex dz = Complex(0.0f, -k.y / k_mag); + Complex h_k_t_dz = complex_multiply(dz, h_k_t_dy); + imageStore(tilde_h_kt_dz, pixel_pos, vec4(h_k_t_dz.real, h_k_t_dz.imag, 0.0f, 1.0f)); +*/ } diff --git a/shaders/subgroups_operations/fft_tilde_h0.comp b/shaders/subgroups_operations/fft_tilde_h0.comp new file mode 100644 index 000000000..6fe55115f --- /dev/null +++ b/shaders/subgroups_operations/fft_tilde_h0.comp @@ -0,0 +1,52 @@ +#version 460 core + +#define PI 3.14159265358979f +#define g 9.81f + +layout (local_size_x = 32, local_size_y = 32) in; + +layout (binding = 0, rgba32f) uniform image2D tilde_h0_k; +layout (binding = 1, rgba32f) uniform image2D tilde_h0_minus_k; + +float SuppressionFactor(float k_magnitude_sq) +{ + float u_SuppressLength = 0.1; + return exp(-k_magnitude_sq * u_SuppressLength * u_SuppressLength); +} + +// Phillips Spectrum, Section 4.3, Eq 40 +float PhillipsSpectrum(vec2 k, float k_magnitude_sq, float L_phillips) +{ + float u_Amplitude = 2.0; + vec2 u_WindDirection = normalize(vec2(1.0, 1.0)); + return u_Amplitude * ((exp(-1.0 / (k_magnitude_sq * L_phillips * L_phillips)) * pow(dot(normalize(k), u_WindDirection), 2)) + * SuppressionFactor(k_magnitude_sq)) / (k_magnitude_sq * k_magnitude_sq); +} + +void main() +{ + int u_N = 256; + int u_L = 1000; + float u_WindSpeed = 80.0; + + vec2 pos = vec2(gl_GlobalInvocationID.xy) - (u_N / 2.0); + + // Wavevector, Section 4.3, Eq 36 + vec2 k = vec2((2.0 * PI * pos.x) / u_L, (2.0 * PI * pos.y) / u_L); + + float k_magnitude = length(k); + if (k_magnitude < 0.00001) k_magnitude = 0.00001; + + // Largest possible waves arising from a continuous wind of speed wind_speed, Section 4.3, after Eq 40 + float L_phillips = (u_WindSpeed * u_WindSpeed) / g; + + // Section 4.4, Eq 42 + float h0_k = clamp(sqrt(PhillipsSpectrum(k, k_magnitude * k_magnitude, L_phillips) / 2.0), -4000.0, 4000.0); + float h0_minus_k = clamp(sqrt(PhillipsSpectrum(-k, k_magnitude * k_magnitude, L_phillips) / 2.0), -4000.0, 4000.0); + + vec4 rnd = {1.0f, 1.0f, 1.0f, 1.0f}; //GaussianRandom(); + + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + imageStore(tilde_h0_k, pixel_pos, vec4(rnd.xy * h0_k, 0.0, 1.0)); + imageStore(tilde_h0_minus_k, pixel_pos, vec4(rnd.zw * h0_minus_k, 0.0, 1.0)); +} From 3aa8449f55f79664b254d07e15ab888d2d43559e Mon Sep 17 00:00:00 2001 From: Krzysztof-Dmitruk-Mobica Date: Fri, 13 Oct 2023 10:28:16 +0200 Subject: [PATCH 27/53] Split fft steps into separate calls --- .../subgroups_operations.cpp | 14 +- shaders/subgroups_operations/fft.comp | 131 ++++++++---------- .../subgroups_operations/fft_tilde_h0.comp | 24 +++- 3 files changed, 86 insertions(+), 83 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 3fe3cb255..d33759f87 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -206,8 +206,6 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline_layout, 0u, 1u, &precompute.descriptor_set, 0u, nullptr); vkCmdDispatch(compute.command_buffer, 1u, DISPLACEMENT_MAP_DIM, 1u); -<<<<<<< HEAD -======= } // initial tildas textures @@ -216,7 +214,6 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, initial_tildas.pipeline.pipeline_layout, 0u, 1u, &initial_tildas.descriptor_set, 0u, nullptr); vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); ->>>>>>> e005a65 (Add tilde_h_0 shader) } // tildas textures @@ -380,10 +377,8 @@ void SubgroupsOperations::create_tildas() vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 4u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 5u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u)}; -<<<<<<< HEAD + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u)}; -======= ->>>>>>> e005a65 (Add tilde_h_0 shader) VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &tildas.descriptor_set_layout)); @@ -1011,14 +1006,11 @@ void SubgroupsOperations::create_butterfly_texture() bit_reverse_buffer->update(bit_reverse_arr.data(), sizeof(uint32_t) * bit_reverse_arr.size()); VkDescriptorBufferInfo bit_reverse_descriptor = create_descriptor(*bit_reverse_buffer); -<<<<<<< HEAD - VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); - VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); -======= bit_reverse_buffer->update(bit_reverse_arr.data(), sizeof(uint32_t) * bit_reverse_arr.size()); VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); ->>>>>>> e005a65 (Add tilde_h_0 shader) + VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); + VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor), diff --git a/shaders/subgroups_operations/fft.comp b/shaders/subgroups_operations/fft.comp index f311e6346..bc3b9aec4 100644 --- a/shaders/subgroups_operations/fft.comp +++ b/shaders/subgroups_operations/fft.comp @@ -38,6 +38,10 @@ layout (binding = 3) uniform FftPage int pages; } fftUbo; +layout( push_constant ) uniform Push_Constants{ + uint i; +} step; + struct Complex { float real; @@ -62,84 +66,69 @@ Complex complex_multiply(Complex c1, Complex c2) void HorizontalButterflies(in ivec2 pixel_pos) { - for (int i = 0; i < fftUbo.pages; ++i) + vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(step.i, pixel_pos.x)); + if ((step.i % 2) == 0) + { + vec2 a_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; + vec2 b_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + subgroupMemoryBarrierImage(); + imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + } + else { - subgroupBarrier(); - vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(i, pixel_pos.x)); - if ((i % 2) == 0) - { - vec2 a_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; - vec2 b_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - // Buttefly operation - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); - - //if(i == 6) - // debugPrintfEXT("[%i] %.5f, %.5f", i, result.real, result.imag); - - } - else - { - vec2 a_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; - vec2 b_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - // Buttefly operation - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); - - if(i == 5) - debugPrintfEXT("[%i] %.5f, %.5f", i, result.real, result.imag); - } + vec2 a_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; + vec2 b_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + subgroupMemoryBarrierImage(); + imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); } } void VerticalButterfiles(in ivec2 pixel_pos) { - for (int i = 0; i < fftUbo.pages; ++i) + vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(step.i, pixel_pos.y)); + if ((step.i % 2) == 0) { - vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(i, pixel_pos.y)); - - if ((i % 2) == 0) - { - vec2 a_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; - vec2 b_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - // Buttefly operation - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - - imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); - } - else - { - vec2 a_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; - vec2 b_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - // Buttefly operation - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - - imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); - } + vec2 a_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; + vec2 b_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + subgroupMemoryBarrierImage(); + + imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + } + else + { + vec2 a_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; + vec2 b_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + subgroupMemoryBarrierImage(); + + imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); } } diff --git a/shaders/subgroups_operations/fft_tilde_h0.comp b/shaders/subgroups_operations/fft_tilde_h0.comp index 6fe55115f..ca213f3b1 100644 --- a/shaders/subgroups_operations/fft_tilde_h0.comp +++ b/shaders/subgroups_operations/fft_tilde_h0.comp @@ -8,6 +8,27 @@ layout (local_size_x = 32, local_size_y = 32) in; layout (binding = 0, rgba32f) uniform image2D tilde_h0_k; layout (binding = 1, rgba32f) uniform image2D tilde_h0_minus_k; +float nrand( vec2 n ) +{ + return fract(sin(dot(n.xy, vec2(12.9898, 78.233)))* 43758.5453); +} + +float n8rand( vec2 n ) +{ + float t = 1; // fract( iTime ); + float nrnd0 = nrand( n + 0.07*t ); + float nrnd1 = nrand( n + 0.11*t ); + float nrnd2 = nrand( n + 0.13*t ); + float nrnd3 = nrand( n + 0.17*t ); + + float nrnd4 = nrand( n + 0.19*t ); + float nrnd5 = nrand( n + 0.23*t ); + float nrnd6 = nrand( n + 0.29*t ); + float nrnd7 = nrand( n + 0.31*t ); + + return (nrnd0+nrnd1+nrnd2+nrnd3+nrnd4+nrnd5+nrnd6+nrnd7) / 8.0; +} + float SuppressionFactor(float k_magnitude_sq) { float u_SuppressLength = 0.1; @@ -44,7 +65,8 @@ void main() float h0_k = clamp(sqrt(PhillipsSpectrum(k, k_magnitude * k_magnitude, L_phillips) / 2.0), -4000.0, 4000.0); float h0_minus_k = clamp(sqrt(PhillipsSpectrum(-k, k_magnitude * k_magnitude, L_phillips) / 2.0), -4000.0, 4000.0); - vec4 rnd = {1.0f, 1.0f, 1.0f, 1.0f}; //GaussianRandom(); + //vec4 rnd = {1.0f, 1.0f, 1.0f, 1.0f}; //GaussianRandom(); + vec4 rnd = {n8rand(pos), n8rand(pos), n8rand(pos), n8rand(pos)}; ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); imageStore(tilde_h0_k, pixel_pos, vec4(rnd.xy * h0_k, 0.0, 1.0)); From cfdcef2249f671fe85ae8a56e73c85137c080a97 Mon Sep 17 00:00:00 2001 From: Krzysztof-Dmitruk-Mobica Date: Fri, 13 Oct 2023 12:37:09 +0200 Subject: [PATCH 28/53] Fix input data randomization --- .../subgroups_operations.cpp | 36 +++++++++---------- .../subgroups_operations.h | 14 -------- .../subgroups_operations/fft_tilde_h0.comp | 16 +++++++-- 3 files changed, 32 insertions(+), 34 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index d33759f87..a2e0e5e8b 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -183,22 +183,19 @@ void SubgroupsOperations::build_compute_command_buffer() VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); - /* if (compute.queue_family_index != ocean.graphics_queue_family_index) { - - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.srcAccessMask = 0u; - memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; - memory_barrier.buffer = fft_buffers.fft_input_htilde0->get_handle(); - memory_barrier.offset = 0u; - memory_barrier.size = fft_buffers.fft_input_htilde0->get_size(); - - vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); + VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); + memory_barrier.srcAccessMask = 0u; + memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; + memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; + memory_barrier.buffer = fft_buffers.fft_input_random->get_handle(); + memory_barrier.offset = 0u; + memory_barrier.size = fft_buffers.fft_input_random->get_size(); + + vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); } - */ // buttle fly texture { @@ -323,6 +320,7 @@ void SubgroupsOperations::create_initial_tildas() std::vector set_layout_bindngs = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u), }; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); @@ -349,12 +347,14 @@ void SubgroupsOperations::create_initial_tildas() createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_input_htilde0); createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_input_htilde0_conj); - VkDescriptorImageInfo htilde_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0); - VkDescriptorImageInfo htilde_conj_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0_conj); + VkDescriptorImageInfo htilde_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0); + VkDescriptorImageInfo htilde_conj_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0_conj); + VkDescriptorBufferInfo input_random_descriptor = create_descriptor(*fft_buffers.fft_input_random); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &htilde_0_descriptor), vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &htilde_conj_0_descriptor), + vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &input_random_descriptor), }; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } @@ -503,7 +503,7 @@ std::complex SubgroupsOperations::calculate_weight(uint32_t x, uint32_t n glm::cos(pi2 * x / n), glm::sin(pi2 * x / n)}; } -std::complex SubgroupsOperations::rndGaussian() +glm::vec2 SubgroupsOperations::rndGaussian() { float x1, x2, w; auto rndVal = []() -> float { @@ -519,7 +519,7 @@ std::complex SubgroupsOperations::rndGaussian() w = x1 * x1 + x2 * x2; } while (w >= 1.0f); w = glm::sqrt((-2.0f * glm::log(w)) / w); - return {x1 * w, x2 * w}; + return glm::vec2{x1 * w, x2 * w}; } std::complex SubgroupsOperations::hTilde_0(uint32_t n, uint32_t m, float sign) @@ -527,7 +527,7 @@ std::complex SubgroupsOperations::hTilde_0(uint32_t n, uint32_t m, float std::complex rnd{1.0f, 1.0f}; // rndGaussian(); // std::complex rnd = rndGaussian(); - return rnd * glm::sqrt(phillips_spectrum(n, m, sign) / 2.0f); + // return rnd * glm::sqrt(phillips_spectrum(n, m, sign) / 2.0f); } void SubgroupsOperations::prepare_uniform_buffers() diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index ce0807f7b..6e27954cb 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -156,7 +156,6 @@ class SubgroupsOperations : public ApiVulkanSample struct { -<<<<<<< HEAD std::unique_ptr fft_input_htilde0 = {VK_NULL_HANDLE}; std::unique_ptr fft_input_htilde0_conj = {VK_NULL_HANDLE}; std::unique_ptr fft_input_weight = {VK_NULL_HANDLE}; @@ -164,19 +163,6 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr fft_tilde_h_kt_dy = {VK_NULL_HANDLE}; std::unique_ptr fft_tilde_h_kt_dz = {VK_NULL_HANDLE}; std::unique_ptr fft_displacement = {VK_NULL_HANDLE}; -======= - // std::unique_ptr fft_input_htilde0; - // std::unique_ptr fft_input_htilde0_conj; - // std::unique_ptr fft_input_weight; - - std::unique_ptr fft_input_htilde0; - std::unique_ptr fft_input_htilde0_conj; - - std::unique_ptr fft_tilde_h_kt_dx; - std::unique_ptr fft_tilde_h_kt_dy; - std::unique_ptr fft_tilde_h_kt_dz; - std::unique_ptr fft_displacement; ->>>>>>> e005a65 (Add tilde_h_0 shader) } fft_buffers; struct diff --git a/shaders/subgroups_operations/fft_tilde_h0.comp b/shaders/subgroups_operations/fft_tilde_h0.comp index ca213f3b1..52bd3f2d5 100644 --- a/shaders/subgroups_operations/fft_tilde_h0.comp +++ b/shaders/subgroups_operations/fft_tilde_h0.comp @@ -7,6 +7,11 @@ layout (local_size_x = 32, local_size_y = 32) in; layout (binding = 0, rgba32f) uniform image2D tilde_h0_k; layout (binding = 1, rgba32f) uniform image2D tilde_h0_minus_k; +layout (std140, binding = 2) readonly buffer InputRandom +{ + vec4 data[]; +} +input_random; float nrand( vec2 n ) { @@ -51,6 +56,8 @@ void main() float u_WindSpeed = 80.0; vec2 pos = vec2(gl_GlobalInvocationID.xy) - (u_N / 2.0); + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + // Wavevector, Section 4.3, Eq 36 vec2 k = vec2((2.0 * PI * pos.x) / u_L, (2.0 * PI * pos.y) / u_L); @@ -61,14 +68,19 @@ void main() // Largest possible waves arising from a continuous wind of speed wind_speed, Section 4.3, after Eq 40 float L_phillips = (u_WindSpeed * u_WindSpeed) / g; + //vec4 rnd = imageLoad(gaussian_random, pixel_pos); + uint idx = pixel_pos.x + pixel_pos.y * u_N; + vec4 rnd = input_random.data[idx]; + + // Section 4.4, Eq 42 float h0_k = clamp(sqrt(PhillipsSpectrum(k, k_magnitude * k_magnitude, L_phillips) / 2.0), -4000.0, 4000.0); float h0_minus_k = clamp(sqrt(PhillipsSpectrum(-k, k_magnitude * k_magnitude, L_phillips) / 2.0), -4000.0, 4000.0); //vec4 rnd = {1.0f, 1.0f, 1.0f, 1.0f}; //GaussianRandom(); - vec4 rnd = {n8rand(pos), n8rand(pos), n8rand(pos), n8rand(pos)}; + //vec4 rnd = {n8rand(pos), n8rand(pos), n8rand(pos), n8rand(pos)}; + - ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); imageStore(tilde_h0_k, pixel_pos, vec4(rnd.xy * h0_k, 0.0, 1.0)); imageStore(tilde_h0_minus_k, pixel_pos, vec4(rnd.zw * h0_minus_k, 0.0, 1.0)); } From fc762c78f26bf98422250799938126d6beec08ce Mon Sep 17 00:00:00 2001 From: Krzysztof-Dmitruk-Mobica Date: Fri, 13 Oct 2023 15:41:52 +0200 Subject: [PATCH 29/53] Code cleanup --- .../subgroups_operations.cpp | 62 ++------------ .../subgroups_operations.h | 43 ++++------ shaders/subgroups_operations/fft.comp | 31 +++---- shaders/subgroups_operations/fft_invert.comp | 7 -- shaders/subgroups_operations/fft_tilde_h.comp | 7 -- .../subgroups_operations/fft_tilde_h0.comp | 84 +++++++------------ 6 files changed, 64 insertions(+), 170 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index a2e0e5e8b..3e09dae2e 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -321,7 +321,7 @@ void SubgroupsOperations::create_initial_tildas() vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u), - }; + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 3u)}; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &initial_tildas.descriptor_set_layout)); @@ -350,12 +350,13 @@ void SubgroupsOperations::create_initial_tildas() VkDescriptorImageInfo htilde_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0); VkDescriptorImageInfo htilde_conj_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0_conj); VkDescriptorBufferInfo input_random_descriptor = create_descriptor(*fft_buffers.fft_input_random); + VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &htilde_0_descriptor), vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &htilde_conj_0_descriptor), vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &input_random_descriptor), - }; + vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_params_ubo_buffer)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } @@ -470,39 +471,6 @@ void SubgroupsOperations::load_assets() */ } -float SubgroupsOperations::phillips_spectrum(int32_t n, int32_t m, float sign) -{ - glm::vec2 k(glm::pi() * (2.0f * n - grid_size) / ui.length, glm::pi() * (2.0f * m - grid_size) / ui.length); - - k = k * sign; - - float k_len = glm::length(k); - if (k_len < 0.000001f) - return 0.000001f; - - float k_len2 = k_len * k_len; - float k_len4 = k_len2 * k_len2; - - float k_dot_w = glm::dot(glm::normalize(k), glm::normalize(ui.wind)); - float k_dot_w2 = k_dot_w * k_dot_w; - - float w_len = glm::length(ui.wind); - float L = w_len * w_len / GRAVITY; - float L2 = L * L; - - float damping = 0.001f; - float l2 = L2 * damping * damping; - - return ui.amplitude * glm::exp(-1.0f / (k_len2 * L2)) / k_len4 * k_dot_w2 * glm::exp(-k_len2 * l2); -} - -std::complex SubgroupsOperations::calculate_weight(uint32_t x, uint32_t n) -{ - const auto pi2 = glm::pi() * 2.0f; - return { - glm::cos(pi2 * x / n), glm::sin(pi2 * x / n)}; -} - glm::vec2 SubgroupsOperations::rndGaussian() { float x1, x2, w; @@ -522,20 +490,11 @@ glm::vec2 SubgroupsOperations::rndGaussian() return glm::vec2{x1 * w, x2 * w}; } -std::complex SubgroupsOperations::hTilde_0(uint32_t n, uint32_t m, float sign) -{ - std::complex rnd{1.0f, 1.0f}; // rndGaussian(); - // std::complex rnd = rndGaussian(); - - // return rnd * glm::sqrt(phillips_spectrum(n, m, sign) / 2.0f); -} - void SubgroupsOperations::prepare_uniform_buffers() { camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - fft_page_ubo = std::make_unique(get_device(), sizeof(FFTPage), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); invert_fft_ubo = std::make_unique(get_device(), sizeof(FFTInvert), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); update_uniform_buffers(); } @@ -711,10 +670,6 @@ void SubgroupsOperations::update_uniform_buffers() fft_ubo.length = ui.length; fft_ubo.wind = ui.wind; - FFTPage pageSize; - // pageSize.page = static_cast(log_2_N); - pageSize.page = 6; // static_cast(log_2_N); - FFTInvert invertFft; invertFft.grid_size = grid_size; invertFft.page_idx = log_2_N % 2; @@ -722,7 +677,6 @@ void SubgroupsOperations::update_uniform_buffers() camera_ubo->convert_and_update(ubo); fft_params_ubo->convert_and_update(fft_ubo); fft_time_ubo->convert_and_update(fftTime); - fft_page_ubo->convert_and_update(pageSize); invert_fft_ubo->convert_and_update(invertFft); } @@ -1026,7 +980,7 @@ void SubgroupsOperations::create_fft() vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 3u)}; + }; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft.descriptor_set_layout)); @@ -1076,13 +1030,11 @@ void SubgroupsOperations::create_fft() VkDescriptorImageInfo image_descriptor_tilda_y = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dy); VkDescriptorImageInfo image_descriptor_tilde_axis_y = create_fb_descriptor(*fft.tilde_axis_y); - auto fft_page_descriptor = create_descriptor(*fft_page_ubo); - std::vector write_descriptor_sets_asix_y = { vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_y), vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_y), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + }; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_y.size()), write_descriptor_sets_asix_y.data(), 0u, nullptr); @@ -1093,7 +1045,7 @@ void SubgroupsOperations::create_fft() vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_x), vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_x), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + }; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_x.size()), write_descriptor_sets_asix_x.data(), 0u, nullptr); @@ -1104,7 +1056,7 @@ void SubgroupsOperations::create_fft() vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_z), vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_z), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_page_descriptor)}; + }; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_z.size()), write_descriptor_sets_asix_z.data(), 0u, nullptr); } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 6e27954cb..e332388f9 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -66,10 +66,7 @@ class SubgroupsOperations : public ApiVulkanSample void update_uniform_buffers(); - float phillips_spectrum(int32_t n, int32_t m, float sign); - std::complex hTilde_0(uint32_t n, uint32_t m, float sign); - std::complex rndGaussian(); - std::complex calculate_weight(uint32_t x, uint32_t n); + glm::vec2 rndGaussian(); struct Pipeline { @@ -101,11 +98,6 @@ class SubgroupsOperations : public ApiVulkanSample alignas(8) glm::vec2 wind; }; - struct FFTPage - { - alignas(4) int32_t page = {0u}; - }; - struct FFTInvert { alignas(4) int32_t page_idx = {-1}; @@ -125,16 +117,14 @@ class SubgroupsOperations : public ApiVulkanSample glm::vec2 wind = {100.0f, -100.0f}; } ui; - uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; - std::unique_ptr camera_ubo = {VK_NULL_HANDLE}; - std::unique_ptr fft_params_ubo = {VK_NULL_HANDLE}; - std::unique_ptr fft_time_ubo = {VK_NULL_HANDLE}; - std::unique_ptr fft_page_ubo = {VK_NULL_HANDLE}; - std::unique_ptr invert_fft_ubo = {VK_NULL_HANDLE}; + uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; + std::unique_ptr camera_ubo = {VK_NULL_HANDLE}; + std::unique_ptr fft_params_ubo = {VK_NULL_HANDLE}; + std::unique_ptr fft_time_ubo = {VK_NULL_HANDLE}; + std::unique_ptr invert_fft_ubo = {VK_NULL_HANDLE}; + std::unique_ptr bit_reverse_buffer = {VK_NULL_HANDLE}; - std::vector> h_tilde_0; - std::vector> h_tilde_0_conj; - std::vector> weights; + std::vector input_random; struct FBAttachment { @@ -152,17 +142,16 @@ class SubgroupsOperations : public ApiVulkanSample uint32_t log_2_N; - std::unique_ptr bit_reverse_buffer = {VK_NULL_HANDLE}; - struct { - std::unique_ptr fft_input_htilde0 = {VK_NULL_HANDLE}; - std::unique_ptr fft_input_htilde0_conj = {VK_NULL_HANDLE}; - std::unique_ptr fft_input_weight = {VK_NULL_HANDLE}; - std::unique_ptr fft_tilde_h_kt_dx = {VK_NULL_HANDLE}; - std::unique_ptr fft_tilde_h_kt_dy = {VK_NULL_HANDLE}; - std::unique_ptr fft_tilde_h_kt_dz = {VK_NULL_HANDLE}; - std::unique_ptr fft_displacement = {VK_NULL_HANDLE}; + std::unique_ptr fft_input_random; + std::unique_ptr fft_input_htilde0; + std::unique_ptr fft_input_htilde0_conj; + + std::unique_ptr fft_tilde_h_kt_dx; + std::unique_ptr fft_tilde_h_kt_dy; + std::unique_ptr fft_tilde_h_kt_dz; + std::unique_ptr fft_displacement; } fft_buffers; struct diff --git a/shaders/subgroups_operations/fft.comp b/shaders/subgroups_operations/fft.comp index bc3b9aec4..366c4634b 100644 --- a/shaders/subgroups_operations/fft.comp +++ b/shaders/subgroups_operations/fft.comp @@ -33,35 +33,30 @@ layout (binding = 0, rgba32f) readonly uniform image2D u_butterfly_precomp; layout (binding = 1, rgba32f) uniform image2D u_pingpong0; layout (binding = 2, rgba32f) uniform image2D u_pingpong1; -layout (binding = 3) uniform FftPage -{ - int pages; -} fftUbo; - layout( push_constant ) uniform Push_Constants{ uint i; } step; struct Complex { - float real; - float imag; + float real; + float imag; }; Complex complex_add(Complex c1, Complex c2) { - Complex res; - res.real = c1.real + c2.real; - res.imag = c1.imag + c2.imag; - return res; + Complex res; + res.real = c1.real + c2.real; + res.imag = c1.imag + c2.imag; + return res; } Complex complex_multiply(Complex c1, Complex c2) { - Complex res; - res.real = c1.real * c2.real - c1.imag * c2.imag; - res.imag = c1.real * c2.imag + c1.imag * c2.real; - return res; + Complex res; + res.real = c1.real * c2.real - c1.imag * c2.imag; + res.imag = c1.real * c2.imag + c1.imag * c2.real; + return res; } void HorizontalButterflies(in ivec2 pixel_pos) @@ -76,7 +71,6 @@ void HorizontalButterflies(in ivec2 pixel_pos) Complex b = Complex(b_.x, b_.y); Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - // Buttefly operation Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); subgroupMemoryBarrierImage(); imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); @@ -90,7 +84,6 @@ void HorizontalButterflies(in ivec2 pixel_pos) Complex b = Complex(b_.x, b_.y); Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - // Buttefly operation Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); subgroupMemoryBarrierImage(); imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); @@ -109,10 +102,8 @@ void VerticalButterfiles(in ivec2 pixel_pos) Complex b = Complex(b_.x, b_.y); Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - // Buttefly operation Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); subgroupMemoryBarrierImage(); - imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); } else @@ -124,10 +115,8 @@ void VerticalButterfiles(in ivec2 pixel_pos) Complex b = Complex(b_.x, b_.y); Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - // Buttefly operation Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); subgroupMemoryBarrierImage(); - imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); } } diff --git a/shaders/subgroups_operations/fft_invert.comp b/shaders/subgroups_operations/fft_invert.comp index b32126cdc..7d0a1a5ad 100644 --- a/shaders/subgroups_operations/fft_invert.comp +++ b/shaders/subgroups_operations/fft_invert.comp @@ -41,14 +41,7 @@ void main() uint N = fftUbo.grid_size; -<<<<<<< HEAD ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); -======= - ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - - //pixel_pos.y = N - pixel_pos.y - 1; - ->>>>>>> e005a65 (Add tilde_h_0 shader) float perms[2] = { 1.0, -1.0 }; int index = int(mod(pixel_pos.x + pixel_pos.y, 2)); diff --git a/shaders/subgroups_operations/fft_tilde_h.comp b/shaders/subgroups_operations/fft_tilde_h.comp index c3e98d1ca..acae11ca4 100644 --- a/shaders/subgroups_operations/fft_tilde_h.comp +++ b/shaders/subgroups_operations/fft_tilde_h.comp @@ -1,12 +1,5 @@ #version 450 #extension GL_KHR_shader_subgroup_basic : enable -<<<<<<< HEAD -======= - -#define PI 3.14159265358979f -#define GRAVITY 9.81f - ->>>>>>> e005a65 (Add tilde_h_0 shader) /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 diff --git a/shaders/subgroups_operations/fft_tilde_h0.comp b/shaders/subgroups_operations/fft_tilde_h0.comp index 52bd3f2d5..c6e35fd42 100644 --- a/shaders/subgroups_operations/fft_tilde_h0.comp +++ b/shaders/subgroups_operations/fft_tilde_h0.comp @@ -1,7 +1,7 @@ #version 460 core #define PI 3.14159265358979f -#define g 9.81f +#define GRAVITY 9.81f layout (local_size_x = 32, local_size_y = 32) in; @@ -9,78 +9,56 @@ layout (binding = 0, rgba32f) uniform image2D tilde_h0_k; layout (binding = 1, rgba32f) uniform image2D tilde_h0_minus_k; layout (std140, binding = 2) readonly buffer InputRandom { - vec4 data[]; + vec4 data[]; } input_random; -float nrand( vec2 n ) +layout (std140, binding = 3) uniform FFTParametersUbo { - return fract(sin(dot(n.xy, vec2(12.9898, 78.233)))* 43758.5453); -} - -float n8rand( vec2 n ) -{ - float t = 1; // fract( iTime ); - float nrnd0 = nrand( n + 0.07*t ); - float nrnd1 = nrand( n + 0.11*t ); - float nrnd2 = nrand( n + 0.13*t ); - float nrnd3 = nrand( n + 0.17*t ); - - float nrnd4 = nrand( n + 0.19*t ); - float nrnd5 = nrand( n + 0.23*t ); - float nrnd6 = nrand( n + 0.29*t ); - float nrnd7 = nrand( n + 0.31*t ); - - return (nrnd0+nrnd1+nrnd2+nrnd3+nrnd4+nrnd5+nrnd6+nrnd7) / 8.0; + float amplitude; + float len; + uint grid_size; + vec2 wind; } +input_params; float SuppressionFactor(float k_magnitude_sq) { - float u_SuppressLength = 0.1; - return exp(-k_magnitude_sq * u_SuppressLength * u_SuppressLength); + float suppress_length = 0.1; + return exp(-k_magnitude_sq * suppress_length * suppress_length); } -// Phillips Spectrum, Section 4.3, Eq 40 -float PhillipsSpectrum(vec2 k, float k_magnitude_sq, float L_phillips) +float PhillipsSpectrum(vec2 k, float k_magnitude_sq, float l_phillips, vec2 wind_direction) { - float u_Amplitude = 2.0; - vec2 u_WindDirection = normalize(vec2(1.0, 1.0)); - return u_Amplitude * ((exp(-1.0 / (k_magnitude_sq * L_phillips * L_phillips)) * pow(dot(normalize(k), u_WindDirection), 2)) - * SuppressionFactor(k_magnitude_sq)) / (k_magnitude_sq * k_magnitude_sq); + return input_params.amplitude + * ((exp(-1.0 / (k_magnitude_sq * l_phillips * l_phillips)) + * pow(dot(normalize(k), wind_direction), 2)) + * SuppressionFactor(k_magnitude_sq)) + / (k_magnitude_sq * k_magnitude_sq); } void main() { - int u_N = 256; - int u_L = 1000; - float u_WindSpeed = 80.0; - - vec2 pos = vec2(gl_GlobalInvocationID.xy) - (u_N / 2.0); - ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - - - // Wavevector, Section 4.3, Eq 36 - vec2 k = vec2((2.0 * PI * pos.x) / u_L, (2.0 * PI * pos.y) / u_L); - - float k_magnitude = length(k); - if (k_magnitude < 0.00001) k_magnitude = 0.00001; + vec2 wind_direction = normalize(input_params.wind); + float wind_speed = length(input_params.wind); - // Largest possible waves arising from a continuous wind of speed wind_speed, Section 4.3, after Eq 40 - float L_phillips = (u_WindSpeed * u_WindSpeed) / g; + vec2 pos = vec2(gl_GlobalInvocationID.xy) - (input_params.grid_size / 2.0); + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - //vec4 rnd = imageLoad(gaussian_random, pixel_pos); - uint idx = pixel_pos.x + pixel_pos.y * u_N; - vec4 rnd = input_random.data[idx]; + vec2 k = vec2((2.0 * PI * pos.x) / input_params.len, (2.0 * PI * pos.y) / input_params.len); + float k_magnitude = length(k); + if (k_magnitude < 0.00001) + k_magnitude = 0.00001; - // Section 4.4, Eq 42 - float h0_k = clamp(sqrt(PhillipsSpectrum(k, k_magnitude * k_magnitude, L_phillips) / 2.0), -4000.0, 4000.0); - float h0_minus_k = clamp(sqrt(PhillipsSpectrum(-k, k_magnitude * k_magnitude, L_phillips) / 2.0), -4000.0, 4000.0); + float l_phillips = (wind_speed * wind_speed) / GRAVITY; - //vec4 rnd = {1.0f, 1.0f, 1.0f, 1.0f}; //GaussianRandom(); - //vec4 rnd = {n8rand(pos), n8rand(pos), n8rand(pos), n8rand(pos)}; + uint idx = pixel_pos.x + pixel_pos.y * input_params.grid_size; + vec4 rnd = input_random.data[idx]; + float h0_k = clamp(sqrt(PhillipsSpectrum(k, k_magnitude * k_magnitude, l_phillips, wind_direction) / 2.0), -4000.0, 4000.0); + float h0_minus_k = clamp(sqrt(PhillipsSpectrum(-k, k_magnitude * k_magnitude, l_phillips, wind_direction) / 2.0), -4000.0, 4000.0); - imageStore(tilde_h0_k, pixel_pos, vec4(rnd.xy * h0_k, 0.0, 1.0)); - imageStore(tilde_h0_minus_k, pixel_pos, vec4(rnd.zw * h0_minus_k, 0.0, 1.0)); + imageStore(tilde_h0_k, pixel_pos, vec4(rnd.xy * h0_k, 0.0, 1.0)); + imageStore(tilde_h0_minus_k, pixel_pos, vec4(rnd.zw * h0_minus_k, 0.0, 1.0)); } From dddff030a6c0e507450cf98d709df549842520c5 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Sat, 14 Oct 2023 15:19:45 +0200 Subject: [PATCH 30/53] rewrite the grid generation --- .../subgroups_operations.cpp | 47 ++++++++++++++----- .../subgroups_operations.h | 5 -- shaders/subgroups_operations/ocean.tesc | 44 +++++++++++++---- shaders/subgroups_operations/ocean.tese | 32 +++++++++---- shaders/subgroups_operations/ocean.vert | 7 ++- 5 files changed, 101 insertions(+), 34 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 3e09dae2e..438b920d4 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -501,19 +501,41 @@ void SubgroupsOperations::prepare_uniform_buffers() void SubgroupsOperations::generate_plane() { - std::vector plane_vertices; + uint32_t vertex_count = grid_size + 1u; + std::vector plane_vertices; + const float tex_coord_scale = 64.0f; + std::vector indices; + int32_t half_grid_size = static_cast(grid_size / 2); - Vertex v; - v.pos = {10.0f, 10.0f, 0.0f}; - plane_vertices.push_back(v); - v.pos = {-10.0f, 10.0f, 0.0f}; - plane_vertices.push_back(v); - v.pos = {-10.0f, -10.0f, 0.0f}; - plane_vertices.push_back(v); - v.pos = {10.0f, -10.0f, 0.0f}; - plane_vertices.push_back(v); + for (int32_t z = -half_grid_size; z <= half_grid_size; ++z) + { + for (int32_t x = -half_grid_size; x <= half_grid_size; ++x) + { + float u = static_cast(x) / static_cast(grid_size) + 0.5f; + float v = static_cast(z) / static_cast(grid_size) + 0.5f; + Vertex vert; + vert.pos = glm::vec3(float(x), 0.0f, float(z)); + vert.uv = glm::vec2(u, v) * tex_coord_scale; + + plane_vertices.push_back(vert); + } + } - std::vector indices = {0, 1, 3, 1, 2, 3}; + for (uint32_t y = 0u; y < grid_size; ++y) + { + for (uint32_t x = 0u; x < grid_size; ++x) + { + // tris 1 + indices.push_back((vertex_count * y) + x); + indices.push_back((vertex_count * (y + 1u)) + x); + indices.push_back((vertex_count * y) + x + 1u); + + // tris 2 + indices.push_back((vertex_count * y) + x + 1u); + indices.push_back((vertex_count * (y + 1u)) + x); + indices.push_back((vertex_count * (y + 1u)) + x + 1u); + } + } auto vertex_buffer_size = vkb::to_u32(plane_vertices.size() * sizeof(Vertex)); auto index_buffer_size = vkb::to_u32(indices.size() * sizeof(uint32_t)); @@ -628,7 +650,8 @@ void SubgroupsOperations::create_pipelines() const std::vector vertex_input_bindings = { vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX)}; const std::vector vertex_input_attributes = { - vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos))}; + vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), + vkb::initializers::vertex_input_attribute_description(0u, 1u, VK_FORMAT_R32G32_SFLOAT, offsetof(Vertex, uv))}; VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index e332388f9..09a87cc62 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -24,11 +24,6 @@ class SubgroupsOperations : public ApiVulkanSample { - struct OceanVertex - { - glm::vec3 position; - }; - public: SubgroupsOperations(); ~SubgroupsOperations(); diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index fa2fab57d..133b977ad 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -18,16 +18,44 @@ layout (vertices = 3) out; +layout(location = 0) in vec3 inPostion[]; +layout(location = 1) in vec2 inUv[]; + +layout(location = 0) out vec3 outPos[]; +layout(location = 1) out vec2 outUv[]; + +layout (binding = 0) uniform CameraUbo +{ + vec3 position; +} cam; + +float get_tesselllation_level(float dist0, float dist1) +{ + float avg_dist = (dist0 + dist1) / 2.0f; + + if (avg_dist <= 10.0f) + return 20.0f; + if (avg_dist <= 20.0f) + return 30.0f; + if (avg_dist <= 30.0f) + return 5.0f; + + return 1.0f; +} + void main() { - if (gl_InvocationID == 0) - { - gl_TessLevelOuter[0] = 1.0f; - gl_TessLevelOuter[1] = 1.0f; - gl_TessLevelOuter[2] = 1.0f; + outPos[gl_InvocationID] = inPostion[gl_InvocationID]; + outUv[gl_InvocationID] = inUv[gl_InvocationID]; + + + float dist_cam_v0 = distance(cam.position, outPos[0]); + float dist_cam_v1 = distance(cam.position, outPos[1]); + float dist_cam_v2 = distance(cam.position, outPos[2]); - gl_TessLevelInner[0] = 15.0f; - } + gl_TessLevelOuter[0] = get_tesselllation_level(dist_cam_v1, dist_cam_v2); + gl_TessLevelOuter[1] = get_tesselllation_level(dist_cam_v0, dist_cam_v2); + gl_TessLevelOuter[2] = get_tesselllation_level(dist_cam_v0, dist_cam_v1); - gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position; + gl_TessLevelInner[0] = gl_TessLevelOuter[2]; } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 3705bff02..69beba426 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -18,6 +18,9 @@ layout(triangles) in; +layout(location = 0) in vec3 inPostion[]; +layout(location = 1) in vec2 inUv[]; + layout (binding = 0) uniform Ubo { mat4 projection; @@ -25,18 +28,31 @@ layout (binding = 0) uniform Ubo mat4 model; } ubo; +layout (binding = 1) uniform sampler2D fftDisplacementMap; + + +vec2 interpolate_2d(vec2 v0, vec2 v1, vec2 v2) +{ + return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2; +} + +vec3 interpolate_3d(vec3 v0, vec3 v1, vec3 v2) +{ + return vec3(gl_TessCoord.x) * v0 + vec3(gl_TessCoord.y) * v1 + vec3(gl_TessCoord.z) * v2; +} void main() { - vec4 p00 = gl_in[0].gl_Position; - vec4 p01 = gl_in[1].gl_Position; - vec4 p10 = gl_in[2].gl_Position; - vec4 p11 = gl_in[3].gl_Position; - vec4 p0 = mix(p00, p01, gl_TessCoord.x); - vec4 p1 = mix(p10, p11, gl_TessCoord.x); + vec3 world_pos = interpolate_3d(inPostion[0], inPostion[1], inPostion[2]); + + vec2 tex_coord = interpolate_2d(inUv[0], inUv[1], inUv[2]); + vec4 fft_texel = vec4(1.0f); // texture2D(fftDisplacementMap, tex_coord); - vec4 p = mix(p0, p1, gl_TessCoord.y); + world_pos.y += fft_texel.y; + world_pos.x -= fft_texel.x; + world_pos.z -= fft_texel.z; - gl_Position = ubo.projection * ubo.view * ubo.model * p; + + gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.vert b/shaders/subgroups_operations/ocean.vert index a18289a59..dbbc007f0 100644 --- a/shaders/subgroups_operations/ocean.vert +++ b/shaders/subgroups_operations/ocean.vert @@ -17,8 +17,13 @@ */ layout (location = 0) in vec3 inPos; +layout (location = 1) in vec2 inUv; + +layout (location = 0) out vec3 outPos; +layout (location = 1) out vec2 outUv; void main() { - gl_Position = vec4(inPos.xyz, 1.0f); + outPos = inPos; + outUv = inUv; } From e52c7729d5c832ec20594de3312bf2f8b7c3bd59 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Mon, 16 Oct 2023 13:52:38 +0200 Subject: [PATCH 31/53] fixed validation layers issues and add changes to ocean pipeline --- .../subgroups_operations.cpp | 157 +++++++++--------- .../subgroups_operations.h | 28 +++- shaders/subgroups_operations/fft_tilde_h.comp | 2 +- shaders/subgroups_operations/ocean.tesc | 14 +- shaders/subgroups_operations/ocean.tese | 16 +- 5 files changed, 117 insertions(+), 100 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 438b920d4..d087223b2 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -56,11 +56,10 @@ SubgroupsOperations::SubgroupsOperations() vkb::GLSLCompiler::set_target_environment(glslang::EShTargetSpv, glslang::EShTargetSpv_1_4); title = "Subgroups operations"; - camera.type = vkb::CameraType::LookAt; + camera.type = vkb::CameraType::FirstPerson; camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); - camera.set_position({-2.0f, -2.25f, -29.0f}); - camera.set_rotation({-24.0f, -7.0f, 0.0f}); + camera.set_position({0.0f, 5.0f, 0.0f}); } SubgroupsOperations::~SubgroupsOperations() @@ -71,6 +70,8 @@ SubgroupsOperations::~SubgroupsOperations() fft_buffers.fft_tilde_h_kt_dy->destroy(get_device().get_handle()); fft_buffers.fft_tilde_h_kt_dz->destroy(get_device().get_handle()); fft_buffers.fft_displacement->destroy(get_device().get_handle()); + fft_buffers.fft_input_htilde0->destroy(get_device().get_handle()); + fft_buffers.fft_input_htilde0_conj->destroy(get_device().get_handle()); butterfly_precomp.destroy(get_device().get_handle()); precompute.pipeline.destroy(get_device().get_handle()); @@ -79,6 +80,9 @@ SubgroupsOperations::~SubgroupsOperations() tildas.pipeline.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), tildas.descriptor_set_layout, nullptr); + initial_tildas.pipeline.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), initial_tildas.descriptor_set_layout, nullptr); + fft_inversion.pipeline.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), fft_inversion.descriptor_set_layout, nullptr); @@ -116,8 +120,6 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) // prepare grpahics pipeline create_semaphore(); create_descriptor_set_layout(); - create_descriptor_set(); - create_pipelines(); create_initial_tildas(); create_tildas(); @@ -125,6 +127,9 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) create_fft(); create_fft_inversion(); + create_descriptor_set(); + create_pipelines(); + build_compute_command_buffer(); build_command_buffers(); @@ -179,24 +184,10 @@ void SubgroupsOperations::create_compute_command_buffer() void SubgroupsOperations::build_compute_command_buffer() { - // record command + // record compute command VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); - if (compute.queue_family_index != ocean.graphics_queue_family_index) - { - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.srcAccessMask = 0u; - memory_barrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; - memory_barrier.buffer = fft_buffers.fft_input_random->get_handle(); - memory_barrier.offset = 0u; - memory_barrier.size = fft_buffers.fft_input_random->get_size(); - - vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); - } - // buttle fly texture { vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline); @@ -264,28 +255,12 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); } - // fft inverse - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } - */ - /* - if (compute.queue_family_index != ocean.graphics_queue_family_index) - { - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; - memory_barrier.dstAccessMask = 0u; - memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; - memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.buffer = fft_buffers.fft_input_htilde0->get_handle(); - memory_barrier.offset = 0u; - memory_barrier.size = fft_buffers.fft_input_htilde0->get_size(); - - vkCmdPipelineBarrier(compute.command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0u, 0u, nullptr, 1u, &memory_barrier, 0u, nullptr); - } - */ + // fft inverse + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); } @@ -297,6 +272,11 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) gpu.get_mutable_requested_features().fillModeNonSolid = VK_TRUE; } + if (gpu.get_features().vertexPipelineStoresAndAtomics) + { + gpu.get_mutable_requested_features().vertexPipelineStoresAndAtomics = VK_TRUE; + } + if (gpu.get_features().tessellationShader) { gpu.get_mutable_requested_features().tessellationShader = VK_TRUE; @@ -492,10 +472,13 @@ glm::vec2 SubgroupsOperations::rndGaussian() void SubgroupsOperations::prepare_uniform_buffers() { - camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - invert_fft_ubo = std::make_unique(get_device(), sizeof(FFTInvert), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + camera_postion_ubo = std::make_unique(get_device(), sizeof(CameraPosition), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + invert_fft_ubo = std::make_unique(get_device(), sizeof(FFTInvert), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + tessellation_params_ubo = std::make_unique(get_device(), sizeof(TessellationParams), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + update_uniform_buffers(); } @@ -565,7 +548,7 @@ void SubgroupsOperations::create_semaphore() void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 7u), vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 5u), vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 7u)}; VkDescriptorPoolCreateInfo descriptor_pool_create_info = @@ -578,7 +561,16 @@ void SubgroupsOperations::create_descriptor_set_layout() std::vector set_layout_bindings = { vkb::initializers::descriptor_set_layout_binding( VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, - 0u)}; + 0u), + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, + 1u), + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, + 2u), + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, + 3u)}; VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); @@ -592,9 +584,16 @@ void SubgroupsOperations::create_descriptor_set() VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &ocean.descriptor_set_layout, 1u); VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &ocean.descriptor_set)); - VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*camera_ubo); + VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*camera_ubo); + VkDescriptorImageInfo desplacement_descriptor = create_fb_descriptor(*fft_buffers.fft_displacement); + VkDescriptorBufferInfo tessellation_params_descriptor = create_descriptor(*tessellation_params_ubo); + VkDescriptorBufferInfo camera_pos_buffer_descriptor = create_descriptor(*camera_postion_ubo); + std::vector write_descriptor_sets = { - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor)}; + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &desplacement_descriptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &tessellation_params_descriptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &camera_pos_buffer_descriptor)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } @@ -687,6 +686,9 @@ void SubgroupsOperations::update_uniform_buffers() ubo.view = camera.matrices.view; ubo.projection = camera.matrices.perspective; + CameraPosition cam_pos; + cam_pos.position = glm::vec4(camera.position, 0.0f); + FFTParametersUbo fft_ubo; fft_ubo.amplitude = ui.amplitude; fft_ubo.grid_size = grid_size; @@ -697,10 +699,20 @@ void SubgroupsOperations::update_uniform_buffers() invertFft.grid_size = grid_size; invertFft.page_idx = log_2_N % 2; + TessellationParams tess_params; + tess_params.displacement_scale = 0.5f; + tess_params.choppines = 0.75f; + + TimeUbo t; + t.time = float(timer.elapsed()); + + fft_time_ubo->convert_and_update(t); camera_ubo->convert_and_update(ubo); fft_params_ubo->convert_and_update(fft_ubo); fft_time_ubo->convert_and_update(fftTime); invert_fft_ubo->convert_and_update(invertFft); + tessellation_params_ubo->convert_and_update(tess_params); + camera_postion_ubo->convert_and_update(cam_pos); } void SubgroupsOperations::build_command_buffers() @@ -725,19 +737,6 @@ void SubgroupsOperations::build_command_buffers() VK_CHECK(vkBeginCommandBuffer(cmd_buff, &command_buffer_begin_info)); - if (compute.queue_family_index != ocean.graphics_queue_family_index) - { - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT; - memory_barrier.srcQueueFamilyIndex = compute.queue_family_index; - memory_barrier.dstQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.buffer = ocean.grid.vertex->get_handle(); - memory_barrier.offset = 0u; - memory_barrier.size = ocean.grid.vertex->get_size(); - - vkCmdPipelineBarrier(cmd_buff, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 0, 0, nullptr, 1, &memory_barrier, 0, nullptr); - } - vkCmdBeginRenderPass(cmd_buff, &render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); VkViewport viewport = vkb::initializers::viewport(static_cast(width), static_cast(height), 0.0f, 1.0f); @@ -762,18 +761,6 @@ void SubgroupsOperations::build_command_buffers() vkCmdEndRenderPass(cmd_buff); - if (compute.queue_family_index != ocean.graphics_queue_family_index) - { - VkBufferMemoryBarrier memory_barrier = vkb::initializers::buffer_memory_barrier(); - memory_barrier.dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT; - memory_barrier.srcQueueFamilyIndex = ocean.graphics_queue_family_index; - memory_barrier.dstQueueFamilyIndex = compute.queue_family_index; - memory_barrier.buffer = ocean.grid.vertex->get_handle(); - memory_barrier.offset = 0u; - memory_barrier.size = ocean.grid.vertex->get_size(); - vkCmdPipelineBarrier(cmd_buff, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 0, nullptr, 1, &memory_barrier, 0, nullptr); - } - VK_CHECK(vkEndCommandBuffer(cmd_buff)); } } @@ -826,21 +813,21 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) if (drawer.header("Ocean settings")) { - if (drawer.input_float("Amplitude", &ui.amplitude, 0.001f, 3u)) + if (drawer.input_float("Amplitude", &ui.amplitude, 0.1f, 3u)) { // update input for fft } - if (drawer.input_float("Length", &ui.length, 0.1f, 1u)) + if (drawer.input_float("Length", &ui.length, 10.f, 1u)) { // update input for fft } if (drawer.header("Wind")) { - if (drawer.input_float("X", &ui.wind.x, 0.5f, 2u)) + if (drawer.input_float("X", &ui.wind.x, 10.f, 2u)) { // update input for fft } - if (drawer.input_float("Y", &ui.wind.y, 0.5f, 2u)) + if (drawer.input_float("Y", &ui.wind.y, 10.f, 2u)) { // update input for fft } @@ -864,9 +851,15 @@ void SubgroupsOperations::render(float delta_time) { return; } - fftTime.time = delta_time; + if (!timer.is_running()) + timer.start(); + update_uniform_buffers(); draw(); + + auto elapsed_time = static_cast(timer.elapsed()); + if (elapsed_time >= 1.0f) + timer.lap(); } std::unique_ptr create_subgroups_operations() @@ -1012,10 +1005,14 @@ void SubgroupsOperations::create_fft() VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_x)); VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft.descriptor_set_axis_z)); + VkPushConstantRange push_constant_range = vkb::initializers::push_constant_range(VK_SHADER_STAGE_COMPUTE_BIT, sizeof(int32_t), 0); + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; compute_pipeline_layout_info.setLayoutCount = 1u; compute_pipeline_layout_info.pSetLayouts = &fft.descriptor_set_layout; + compute_pipeline_layout_info.pushConstantRangeCount = 1u; + compute_pipeline_layout_info.pPushConstantRanges = &push_constant_range; VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &fft.pipelines.horizontal.pipeline_layout)); VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &fft.pipelines.vertical.pipeline_layout)); diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 09a87cc62..5d56330cd 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -85,6 +85,17 @@ class SubgroupsOperations : public ApiVulkanSample alignas(16) glm::mat4 model; }; + struct CameraPosition + { + alignas(16) glm::vec4 position; + }; + + struct TessellationParams + { + alignas(4) float choppines; + alignas(4) float displacement_scale; + }; + struct FFTParametersUbo { alignas(4) float amplitude; @@ -112,12 +123,14 @@ class SubgroupsOperations : public ApiVulkanSample glm::vec2 wind = {100.0f, -100.0f}; } ui; - uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; - std::unique_ptr camera_ubo = {VK_NULL_HANDLE}; - std::unique_ptr fft_params_ubo = {VK_NULL_HANDLE}; - std::unique_ptr fft_time_ubo = {VK_NULL_HANDLE}; - std::unique_ptr invert_fft_ubo = {VK_NULL_HANDLE}; - std::unique_ptr bit_reverse_buffer = {VK_NULL_HANDLE}; + uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; + std::unique_ptr camera_postion_ubo = {VK_NULL_HANDLE}; + std::unique_ptr camera_ubo = {VK_NULL_HANDLE}; + std::unique_ptr tessellation_params_ubo = {VK_NULL_HANDLE}; + std::unique_ptr fft_params_ubo = {VK_NULL_HANDLE}; + std::unique_ptr fft_time_ubo = {VK_NULL_HANDLE}; + std::unique_ptr invert_fft_ubo = {VK_NULL_HANDLE}; + std::unique_ptr bit_reverse_buffer = {VK_NULL_HANDLE}; std::vector input_random; @@ -135,7 +148,8 @@ class SubgroupsOperations : public ApiVulkanSample }; } butterfly_precomp; - uint32_t log_2_N; + uint32_t log_2_N; + vkb::Timer timer; struct { diff --git a/shaders/subgroups_operations/fft_tilde_h.comp b/shaders/subgroups_operations/fft_tilde_h.comp index acae11ca4..6816c2b56 100644 --- a/shaders/subgroups_operations/fft_tilde_h.comp +++ b/shaders/subgroups_operations/fft_tilde_h.comp @@ -98,7 +98,7 @@ void main() vec2 k = vec2((2.0 * PI * pos.x) / L, (2.0 * PI * pos.y) / L); float k_magnitude = length(k); - if (k_magnitude < 0.00001) k_magnitude = 0.00001; + if (k_magnitude < 0.00001f) k_magnitude = 0.00001f; // Dispersion relation, Eq 31, Section 4.2 float w = sqrt(GRAVITY * k_magnitude); diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 133b977ad..04668b511 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -24,9 +24,9 @@ layout(location = 1) in vec2 inUv[]; layout(location = 0) out vec3 outPos[]; layout(location = 1) out vec2 outUv[]; -layout (binding = 0) uniform CameraUbo +layout (binding = 3) uniform CameraPos { - vec3 position; + vec4 position; } cam; float get_tesselllation_level(float dist0, float dist1) @@ -35,9 +35,9 @@ float get_tesselllation_level(float dist0, float dist1) if (avg_dist <= 10.0f) return 20.0f; - if (avg_dist <= 20.0f) + else if (avg_dist <= 20.0f) return 30.0f; - if (avg_dist <= 30.0f) + else if (avg_dist <= 30.0f) return 5.0f; return 1.0f; @@ -49,9 +49,9 @@ void main() outUv[gl_InvocationID] = inUv[gl_InvocationID]; - float dist_cam_v0 = distance(cam.position, outPos[0]); - float dist_cam_v1 = distance(cam.position, outPos[1]); - float dist_cam_v2 = distance(cam.position, outPos[2]); + float dist_cam_v0 = distance(cam.position.xyz, outPos[0]); + float dist_cam_v1 = distance(cam.position.xyz, outPos[1]); + float dist_cam_v2 = distance(cam.position.xyz, outPos[2]); gl_TessLevelOuter[0] = get_tesselllation_level(dist_cam_v1, dist_cam_v2); gl_TessLevelOuter[1] = get_tesselllation_level(dist_cam_v0, dist_cam_v2); diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 69beba426..7f665d598 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -28,7 +28,13 @@ layout (binding = 0) uniform Ubo mat4 model; } ubo; -layout (binding = 1) uniform sampler2D fftDisplacementMap; +layout (binding = 1, rgba32f) uniform image2D fftDisplacementMap; + +layout (binding = 2) uniform TessellationParams +{ + float choppines; + float displacement_scale; +} tessParams; vec2 interpolate_2d(vec2 v0, vec2 v1, vec2 v2) @@ -47,11 +53,11 @@ void main() vec3 world_pos = interpolate_3d(inPostion[0], inPostion[1], inPostion[2]); vec2 tex_coord = interpolate_2d(inUv[0], inUv[1], inUv[2]); - vec4 fft_texel = vec4(1.0f); // texture2D(fftDisplacementMap, tex_coord); + vec4 fft_texel = imageLoad(fftDisplacementMap, ivec2(tex_coord)); - world_pos.y += fft_texel.y; - world_pos.x -= fft_texel.x; - world_pos.z -= fft_texel.z; + world_pos.y += fft_texel.y * tessParams.displacement_scale; + world_pos.x -= fft_texel.x * tessParams.choppines; + world_pos.z -= fft_texel.z * tessParams.choppines; gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); From 37feee2b310151c1d9e778d44e4dee75b687ae2e Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Mon, 16 Oct 2023 14:30:48 +0200 Subject: [PATCH 32/53] animation fix --- .../extensions/subgroups_operations/subgroups_operations.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index d087223b2..f873e9473 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -709,7 +709,6 @@ void SubgroupsOperations::update_uniform_buffers() fft_time_ubo->convert_and_update(t); camera_ubo->convert_and_update(ubo); fft_params_ubo->convert_and_update(fft_ubo); - fft_time_ubo->convert_and_update(fftTime); invert_fft_ubo->convert_and_update(invertFft); tessellation_params_ubo->convert_and_update(tess_params); camera_postion_ubo->convert_and_update(cam_pos); @@ -856,10 +855,6 @@ void SubgroupsOperations::render(float delta_time) update_uniform_buffers(); draw(); - - auto elapsed_time = static_cast(timer.elapsed()); - if (elapsed_time >= 1.0f) - timer.lap(); } std::unique_ptr create_subgroups_operations() From fb35f7d757bd1ce71cb7167bc05ff7fc6d560406 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Tue, 17 Oct 2023 11:11:31 +0200 Subject: [PATCH 33/53] add normal map --- .../subgroups_operations.cpp | 92 ++++++++++++++++--- .../subgroups_operations.h | 9 ++ shaders/subgroups_operations/fft_invert.comp | 8 +- .../subgroups_operations/fft_normal_map.comp | 57 ++++++++++++ shaders/subgroups_operations/ocean.frag | 26 +++++- shaders/subgroups_operations/ocean.tesc | 6 +- shaders/subgroups_operations/ocean.tese | 8 +- 7 files changed, 180 insertions(+), 26 deletions(-) create mode 100644 shaders/subgroups_operations/fft_normal_map.comp diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index f873e9473..1e28d6b36 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -70,6 +70,8 @@ SubgroupsOperations::~SubgroupsOperations() fft_buffers.fft_tilde_h_kt_dy->destroy(get_device().get_handle()); fft_buffers.fft_tilde_h_kt_dz->destroy(get_device().get_handle()); fft_buffers.fft_displacement->destroy(get_device().get_handle()); + fft_buffers.fft_normal_map->destroy(get_device().get_handle()); + fft_buffers.fft_input_htilde0->destroy(get_device().get_handle()); fft_buffers.fft_input_htilde0_conj->destroy(get_device().get_handle()); butterfly_precomp.destroy(get_device().get_handle()); @@ -86,6 +88,9 @@ SubgroupsOperations::~SubgroupsOperations() fft_inversion.pipeline.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), fft_inversion.descriptor_set_layout, nullptr); + fft_normal_map.pipeline.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), fft_normal_map.descriptor_set_layout, nullptr); + fft.tilde_axis_x->destroy(get_device().get_handle()); fft.tilde_axis_y->destroy(get_device().get_handle()); fft.tilde_axis_z->destroy(get_device().get_handle()); @@ -126,6 +131,7 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) create_butterfly_texture(); create_fft(); create_fft_inversion(); + create_fft_normal_map(); create_descriptor_set(); create_pipelines(); @@ -262,6 +268,13 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); } + // fft normal map + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_normal_map.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_normal_map.pipeline.pipeline_layout, 0u, 1u, &fft_normal_map.descriptor_set, 0u, nullptr); + vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + } + VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); } @@ -484,18 +497,19 @@ void SubgroupsOperations::prepare_uniform_buffers() void SubgroupsOperations::generate_plane() { - uint32_t vertex_count = grid_size + 1u; + uint32_t dim_gird = grid_size; + uint32_t vertex_count = dim_gird + 1u; std::vector plane_vertices; - const float tex_coord_scale = 64.0f; + const float tex_coord_scale = 256.0f; std::vector indices; - int32_t half_grid_size = static_cast(grid_size / 2); + int32_t half_grid_size = static_cast(dim_gird / 2); for (int32_t z = -half_grid_size; z <= half_grid_size; ++z) { for (int32_t x = -half_grid_size; x <= half_grid_size; ++x) { - float u = static_cast(x) / static_cast(grid_size) + 0.5f; - float v = static_cast(z) / static_cast(grid_size) + 0.5f; + float u = static_cast(x) / static_cast(dim_gird) + 0.5f; + float v = static_cast(z) / static_cast(dim_gird) + 0.5f; Vertex vert; vert.pos = glm::vec3(float(x), 0.0f, float(z)); vert.uv = glm::vec2(u, v) * tex_coord_scale; @@ -504,9 +518,9 @@ void SubgroupsOperations::generate_plane() } } - for (uint32_t y = 0u; y < grid_size; ++y) + for (uint32_t y = 0u; y < dim_gird; ++y) { - for (uint32_t x = 0u; x < grid_size; ++x) + for (uint32_t x = 0u; x < dim_gird; ++x) { // tris 1 indices.push_back((vertex_count * y) + x); @@ -550,9 +564,9 @@ void SubgroupsOperations::setup_descriptor_pool() std::vector pool_sizes = { vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 7u), vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 5u), - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 7u)}; + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 9u)}; VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 7u); + vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 9u); VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } @@ -560,17 +574,20 @@ void SubgroupsOperations::create_descriptor_set_layout() { std::vector set_layout_bindings = { vkb::initializers::descriptor_set_layout_binding( - VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0u), vkb::initializers::descriptor_set_layout_binding( - VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, + VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 1u), vkb::initializers::descriptor_set_layout_binding( - VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 2u), vkb::initializers::descriptor_set_layout_binding( - VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, - 3u)}; + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + 3u), + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + 4u)}; VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); @@ -588,12 +605,14 @@ void SubgroupsOperations::create_descriptor_set() VkDescriptorImageInfo desplacement_descriptor = create_fb_descriptor(*fft_buffers.fft_displacement); VkDescriptorBufferInfo tessellation_params_descriptor = create_descriptor(*tessellation_params_ubo); VkDescriptorBufferInfo camera_pos_buffer_descriptor = create_descriptor(*camera_postion_ubo); + VkDescriptorImageInfo normal_map_descirptor = create_fb_descriptor(*fft_buffers.fft_normal_map); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &desplacement_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &tessellation_params_descriptor), - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &camera_pos_buffer_descriptor)}; + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &camera_pos_buffer_descriptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &normal_map_descirptor)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } @@ -1134,6 +1153,49 @@ void SubgroupsOperations::create_fft_inversion() vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } +void SubgroupsOperations::create_fft_normal_map() +{ + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) + }; + + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft_normal_map.descriptor_set_layout)); + + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft_normal_map.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft_normal_map.descriptor_set)); + + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; + compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + compute_pipeline_layout_info.setLayoutCount = 1u; + compute_pipeline_layout_info.pSetLayouts = &fft_normal_map.descriptor_set_layout; + + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &fft_normal_map.pipeline.pipeline_layout)); + + VkComputePipelineCreateInfo computeInfo = {}; + computeInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + computeInfo.layout = fft_normal_map.pipeline.pipeline_layout; + computeInfo.stage = load_shader("subgroups_operations/fft_normal_map.comp", VK_SHADER_STAGE_COMPUTE_BIT); + + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft_normal_map.pipeline.pipeline)); + + fft_buffers.fft_normal_map = std::make_unique(); + + createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_normal_map); + + VkDescriptorImageInfo image_descriptor_normal_map = create_fb_descriptor(*fft_buffers.fft_normal_map); + VkDescriptorImageInfo image_descriptor_displacment_axis = create_fb_descriptor(*fft_buffers.fft_displacement); + auto fft_page_descriptor = create_descriptor(*invert_fft_ubo); + + std::vector write_descriptor_sets = { + vkb::initializers::write_descriptor_set(fft_normal_map.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_normal_map), + vkb::initializers::write_descriptor_set(fft_normal_map.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_displacment_axis), + vkb::initializers::write_descriptor_set(fft_normal_map.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &fft_page_descriptor)}; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); +} + VkDescriptorImageInfo SubgroupsOperations::create_fb_descriptor(FBAttachment &attachment) { VkDescriptorImageInfo image_descriptor{}; diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 5d56330cd..87eb78c74 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -58,6 +58,7 @@ class SubgroupsOperations : public ApiVulkanSample void create_butterfly_texture(); void create_fft(); void create_fft_inversion(); + void create_fft_normal_map(); void update_uniform_buffers(); @@ -161,6 +162,7 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr fft_tilde_h_kt_dy; std::unique_ptr fft_tilde_h_kt_dz; std::unique_ptr fft_displacement; + std::unique_ptr fft_normal_map; } fft_buffers; struct @@ -197,6 +199,13 @@ class SubgroupsOperations : public ApiVulkanSample Pipeline pipeline; } fft_inversion; + struct + { + VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; + VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; + Pipeline pipeline; + } fft_normal_map; + struct { VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; diff --git a/shaders/subgroups_operations/fft_invert.comp b/shaders/subgroups_operations/fft_invert.comp index 7d0a1a5ad..c3e544222 100644 --- a/shaders/subgroups_operations/fft_invert.comp +++ b/shaders/subgroups_operations/fft_invert.comp @@ -43,9 +43,9 @@ void main() ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - float perms[2] = { 1.0, -1.0 }; - int index = int(mod(pixel_pos.x + pixel_pos.y, 2)); - float perm = perms[index]; + float perms[2] = { 1.0f, -1.0f }; + int index = int(mod(pixel_pos.x + pixel_pos.y, 2)); + float perm = perms[index]; uint pingpong_index = fftUbo.pong_idx; if (pingpong_index == 0) { @@ -59,6 +59,6 @@ void main() float h_y = imageLoad(u_pingpong1_y, pixel_pos).r; float h_x = imageLoad(u_pingpong1_x, pixel_pos).r; float h_z = imageLoad(u_pingpong1_z, pixel_pos).r; - imageStore(u_displacement, pixel_pos, vec4(perm * (h_x / float(N * N)), perm * (h_y / float(N * N)), perm * (h_z / float(N * N)), 1)); + imageStore(u_displacement, pixel_pos, vec4(perm * (h_x / float(N * N)), perm * (h_y / float(N * N)), perm * (h_z / float(N * N)), 1.0f)); } } diff --git a/shaders/subgroups_operations/fft_normal_map.comp b/shaders/subgroups_operations/fft_normal_map.comp new file mode 100644 index 000000000..7d3dae901 --- /dev/null +++ b/shaders/subgroups_operations/fft_normal_map.comp @@ -0,0 +1,57 @@ +#version 450 +#extension GL_KHR_shader_subgroup_basic : enable + +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; + + + layout (binding = 0, rgba32f) writeonly uniform image2D fft_normal_map; + // xyz + layout (binding = 1, rgba32f) readonly uniform image2D fft_displacement_map; + + + layout (binding = 2) uniform InvertFft +{ + int pong_idx; + uint grid_size; +} fftUbo; + +void main() +{ + uint N = fftUbo.grid_size; + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + + vec2 tex_coord = gl_GlobalInvocationID.xy / float(N); + float tex_dim = 1.0f / N; + + float z0 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(-tex_dim, -tex_dim))).g; + float z1 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(0.0f, -tex_dim))).g; + float z2 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(tex_dim, -tex_dim))).g; + float z3 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(-tex_dim, 0.0f))).g; + float z4 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(tex_dim, 0.0f))).g; + float z5 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(-tex_dim, tex_dim))).g; + float z6 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(0.0f, tex_dim))).g; + float z7 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(tex_dim, tex_dim))).g; + + vec3 result = vec3(0.0f); + result.z = z0 + 2.0f * z1 + z2 - z5 - 2.0f * z6 - z7; + result.x = z0 + 2.0f * z3 + z5 - z2 - 2.0f * z4 - z7; + result.y = 1.0f; + + imageStore(fft_normal_map, pixel_pos, vec4(normalize(result), 1.0f)); +} \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index b44fda738..e9feaa32a 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -16,10 +16,32 @@ * limitations under the License. */ +layout (location = 0) in vec3 in_pos; +layout (location = 1) in vec3 in_normal; + layout (location = 0) out vec4 outFragColor; + +layout (binding = 3) uniform CameraPos +{ + vec4 position; +} cam; + void main() { - vec3 ocean_color = vec3(0.0f, 0.5f, 1.0f); - outFragColor = vec4(ocean_color, 1.0f); + vec3 light_pos = vec3(0.0f, 0.0f, -1.0f); + vec3 ocean_color = vec3(0.0f, 0.2423423f, 0.434335435f); + vec3 view_dir = normalize(vec3(cam.position.xyz - in_pos)); + vec3 ref_dir = reflect(-view_dir, in_normal); + + vec3 light_dir = normalize(light_pos - in_pos); + + float dot_l = max(dot(in_normal, light_dir), 0.0f); + + vec3 result = dot_l * ocean_color; + result = result / (result + vec3(1.0f)); + // gamma correction + result = pow(result, vec3(1.0f / 2.2f)); + + outFragColor = vec4(result, 1.0f); } diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 04668b511..18edb6d94 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -53,9 +53,9 @@ void main() float dist_cam_v1 = distance(cam.position.xyz, outPos[1]); float dist_cam_v2 = distance(cam.position.xyz, outPos[2]); - gl_TessLevelOuter[0] = get_tesselllation_level(dist_cam_v1, dist_cam_v2); - gl_TessLevelOuter[1] = get_tesselllation_level(dist_cam_v0, dist_cam_v2); - gl_TessLevelOuter[2] = get_tesselllation_level(dist_cam_v0, dist_cam_v1); + gl_TessLevelOuter[0] = 1.0f; //get_tesselllation_level(dist_cam_v1, dist_cam_v2); + gl_TessLevelOuter[1] = 1.0f; //get_tesselllation_level(dist_cam_v0, dist_cam_v2); + gl_TessLevelOuter[2] = 1.0f; //get_tesselllation_level(dist_cam_v0, dist_cam_v1); gl_TessLevelInner[0] = gl_TessLevelOuter[2]; } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 7f665d598..f559f945f 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -21,6 +21,9 @@ layout(triangles) in; layout(location = 0) in vec3 inPostion[]; layout(location = 1) in vec2 inUv[]; +layout (location = 0) out vec3 outPos; +layout (location = 1) out vec3 outNormal; + layout (binding = 0) uniform Ubo { mat4 projection; @@ -36,6 +39,7 @@ layout (binding = 2) uniform TessellationParams float displacement_scale; } tessParams; +layout (binding = 4, rgba32f) uniform image2D fft_height_map; vec2 interpolate_2d(vec2 v0, vec2 v1, vec2 v2) { @@ -49,7 +53,6 @@ vec3 interpolate_3d(vec3 v0, vec3 v1, vec3 v2) void main() { - vec3 world_pos = interpolate_3d(inPostion[0], inPostion[1], inPostion[2]); vec2 tex_coord = interpolate_2d(inUv[0], inUv[1], inUv[2]); @@ -59,6 +62,7 @@ void main() world_pos.x -= fft_texel.x * tessParams.choppines; world_pos.z -= fft_texel.z * tessParams.choppines; - + outNormal = imageLoad(fft_height_map, ivec2(tex_coord)).xyz; + outPos = world_pos; gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); } \ No newline at end of file From 91d3edfe37dc4d640e66a1106131436444665b20 Mon Sep 17 00:00:00 2001 From: Krzysztof-Dmitruk-Mobica Date: Tue, 17 Oct 2023 15:09:58 +0200 Subject: [PATCH 34/53] Fix interpolation of a tessellated grid --- .../subgroups_operations.cpp | 3 +-- .../subgroups_operations.h | 2 +- shaders/subgroups_operations/ocean.tesc | 14 +++++------ shaders/subgroups_operations/ocean.tese | 25 +++++++++++++++---- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 1e28d6b36..34bc0ca5f 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -1158,8 +1158,7 @@ void SubgroupsOperations::create_fft_normal_map() std::vector set_layout_bindngs = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u) - }; + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u)}; VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft_normal_map.descriptor_set_layout)); diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 87eb78c74..d10ece0ad 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -119,7 +119,7 @@ class SubgroupsOperations : public ApiVulkanSample struct GuiConfig { bool wireframe = {false}; - float amplitude = {2.0f}; + float amplitude = {1000.0f}; float length = {1000.0f}; glm::vec2 wind = {100.0f, -100.0f}; } ui; diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 18edb6d94..374905867 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -34,11 +34,11 @@ float get_tesselllation_level(float dist0, float dist1) float avg_dist = (dist0 + dist1) / 2.0f; if (avg_dist <= 10.0f) - return 20.0f; + return 2.0f; else if (avg_dist <= 20.0f) - return 30.0f; + return 3.0f; else if (avg_dist <= 30.0f) - return 5.0f; + return 1.0f; return 1.0f; } @@ -53,9 +53,9 @@ void main() float dist_cam_v1 = distance(cam.position.xyz, outPos[1]); float dist_cam_v2 = distance(cam.position.xyz, outPos[2]); - gl_TessLevelOuter[0] = 1.0f; //get_tesselllation_level(dist_cam_v1, dist_cam_v2); - gl_TessLevelOuter[1] = 1.0f; //get_tesselllation_level(dist_cam_v0, dist_cam_v2); - gl_TessLevelOuter[2] = 1.0f; //get_tesselllation_level(dist_cam_v0, dist_cam_v1); + gl_TessLevelOuter[0] = get_tesselllation_level(dist_cam_v1, dist_cam_v2); + gl_TessLevelOuter[1] = get_tesselllation_level(dist_cam_v0, dist_cam_v2); + gl_TessLevelOuter[2] = get_tesselllation_level(dist_cam_v0, dist_cam_v1); gl_TessLevelInner[0] = gl_TessLevelOuter[2]; -} \ No newline at end of file +} diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index f559f945f..b59a298f4 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -43,7 +43,7 @@ layout (binding = 4, rgba32f) uniform image2D fft_height_map; vec2 interpolate_2d(vec2 v0, vec2 v1, vec2 v2) { - return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2; + return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2; } vec3 interpolate_3d(vec3 v0, vec3 v1, vec3 v2) @@ -51,18 +51,33 @@ vec3 interpolate_3d(vec3 v0, vec3 v1, vec3 v2) return vec3(gl_TessCoord.x) * v0 + vec3(gl_TessCoord.y) * v1 + vec3(gl_TessCoord.z) * v2; } +vec4 interpolate_4d(vec4 v0, vec4 v1, vec4 v2) +{ + return vec4(gl_TessCoord.x) * v0 + vec4(gl_TessCoord.y) * v1 + vec4(gl_TessCoord.z) * v2; +} + void main() { vec3 world_pos = interpolate_3d(inPostion[0], inPostion[1], inPostion[2]); - vec2 tex_coord = interpolate_2d(inUv[0], inUv[1], inUv[2]); - vec4 fft_texel = imageLoad(fftDisplacementMap, ivec2(tex_coord)); + vec4 fft_texel_at_vertex[4]; + fft_texel_at_vertex[0] = imageLoad(fftDisplacementMap, ivec2(inUv[0])); + fft_texel_at_vertex[1] = imageLoad(fftDisplacementMap, ivec2(inUv[1])); + fft_texel_at_vertex[2] = imageLoad(fftDisplacementMap, ivec2(inUv[2])); + + vec4 height_texel_at_vertex[4]; + height_texel_at_vertex[0] = imageLoad(fft_height_map, ivec2(inUv[0])); + height_texel_at_vertex[1] = imageLoad(fft_height_map, ivec2(inUv[1])); + height_texel_at_vertex[2] = imageLoad(fft_height_map, ivec2(inUv[2])); + + vec4 fft_texel = interpolate_4d(fft_texel_at_vertex[0], fft_texel_at_vertex[1], fft_texel_at_vertex[2]); + vec4 height_texel = interpolate_4d(height_texel_at_vertex[0], height_texel_at_vertex[1], height_texel_at_vertex[2]); world_pos.y += fft_texel.y * tessParams.displacement_scale; world_pos.x -= fft_texel.x * tessParams.choppines; world_pos.z -= fft_texel.z * tessParams.choppines; - outNormal = imageLoad(fft_height_map, ivec2(tex_coord)).xyz; + outNormal = height_texel.xyz; outPos = world_pos; gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); -} \ No newline at end of file +} From dedd5734b5088af407c955a9df85e9e813c90bde Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Tue, 17 Oct 2023 15:36:04 +0200 Subject: [PATCH 35/53] some fixes and new ui params --- .../subgroups_operations.cpp | 61 ++++++++++++------- .../subgroups_operations.h | 16 ++++- .../subgroups_operations/fft_normal_map.comp | 21 +++---- shaders/subgroups_operations/ocean.frag | 30 +++++++-- shaders/subgroups_operations/ocean.tese | 29 ++++----- 5 files changed, 100 insertions(+), 57 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 34bc0ca5f..2063e1abe 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -491,6 +491,7 @@ void SubgroupsOperations::prepare_uniform_buffers() fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); invert_fft_ubo = std::make_unique(get_device(), sizeof(FFTInvert), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); tessellation_params_ubo = std::make_unique(get_device(), sizeof(TessellationParams), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + ocean_params_ubo = std::make_unique(get_device(), sizeof(OceanParamsUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); update_uniform_buffers(); } @@ -500,7 +501,7 @@ void SubgroupsOperations::generate_plane() uint32_t dim_gird = grid_size; uint32_t vertex_count = dim_gird + 1u; std::vector plane_vertices; - const float tex_coord_scale = 256.0f; + const float tex_coord_scale = float(grid_size); std::vector indices; int32_t half_grid_size = static_cast(dim_gird / 2); @@ -562,11 +563,11 @@ void SubgroupsOperations::create_semaphore() void SubgroupsOperations::setup_descriptor_pool() { std::vector pool_sizes = { - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 7u), - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 5u), - vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 9u)}; + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 20u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 20u), + vkb::initializers::descriptor_pool_size(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 20u)}; VkDescriptorPoolCreateInfo descriptor_pool_create_info = - vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 9u); + vkb::initializers::descriptor_pool_create_info(static_cast(pool_sizes.size()), pool_sizes.data(), 15u); VK_CHECK(vkCreateDescriptorPool(get_device().get_handle(), &descriptor_pool_create_info, nullptr, &descriptor_pool)); } @@ -587,7 +588,10 @@ void SubgroupsOperations::create_descriptor_set_layout() 3u), vkb::initializers::descriptor_set_layout_binding( VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, - 4u)}; + 4u), + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + 5u)}; VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); @@ -606,13 +610,15 @@ void SubgroupsOperations::create_descriptor_set() VkDescriptorBufferInfo tessellation_params_descriptor = create_descriptor(*tessellation_params_ubo); VkDescriptorBufferInfo camera_pos_buffer_descriptor = create_descriptor(*camera_postion_ubo); VkDescriptorImageInfo normal_map_descirptor = create_fb_descriptor(*fft_buffers.fft_normal_map); + VkDescriptorBufferInfo ocean_params_buffer_descriptor = create_descriptor(*ocean_params_ubo); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &desplacement_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &tessellation_params_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &camera_pos_buffer_descriptor), - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &normal_map_descirptor)}; + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &normal_map_descirptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &ocean_params_buffer_descriptor)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } @@ -725,6 +731,12 @@ void SubgroupsOperations::update_uniform_buffers() TimeUbo t; t.time = float(timer.elapsed()); + OceanParamsUbo ocean_params; + ocean_params.light_color = ui.light_color; + ocean_params.light_position = ui.light_pos; + ocean_params.ocean_color = ui.ocean_color; + + ocean_params_ubo->convert_and_update(ocean_params); fft_time_ubo->convert_and_update(t); camera_ubo->convert_and_update(ubo); fft_params_ubo->convert_and_update(fft_ubo); @@ -826,29 +838,34 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) build_command_buffers(); } } + + if (drawer.header("Light")) + { + drawer.slider_float("Position x", &ui.light_pos.x, -1000.0f, 1000.0f); + drawer.slider_float("Position y", &ui.light_pos.y, -1000.0f, 1000.0f); + drawer.slider_float("Position z", &ui.light_pos.z, -1000.0f, 1000.0f); + + drawer.slider_float("Color Red", &ui.light_color.r, 0.0f, 1.0f); + drawer.slider_float("Color Green", &ui.light_color.g, 0.0f, 1.0f); + drawer.slider_float("Color Blue", &ui.light_color.b, 0.0f, 1.0f); + } } if (drawer.header("Ocean settings")) - { - if (drawer.input_float("Amplitude", &ui.amplitude, 0.1f, 3u)) + drawer.input_float("Amplitude", &ui.amplitude, 0.1f, 3u); + drawer.input_float("Length", &ui.length, 10.f, 1u); + if (drawer.header("Color")) { - // update input for fft - } - if (drawer.input_float("Length", &ui.length, 10.f, 1u)) - { - // update input for fft + drawer.slider_float("Red", &ui.ocean_color.r, 0.0f, 1.0f); + drawer.slider_float("Green", &ui.ocean_color.g, 0.0f, 1.0f); + drawer.slider_float("Blue", &ui.ocean_color.b, 0.0f, 1.0f); } + if (drawer.header("Wind")) { - if (drawer.input_float("X", &ui.wind.x, 10.f, 2u)) - { - // update input for fft - } - if (drawer.input_float("Y", &ui.wind.y, 10.f, 2u)) - { - // update input for fft - } + drawer.input_float("X", &ui.wind.x, 10.f, 2u); + drawer.input_float("Y", &ui.wind.y, 10.f, 2u); } } } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index d10ece0ad..badb3ed4e 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -86,6 +86,13 @@ class SubgroupsOperations : public ApiVulkanSample alignas(16) glm::mat4 model; }; + struct OceanParamsUbo + { + alignas(16) glm::vec3 light_color; + alignas(16) glm::vec3 light_position; + alignas(16) glm::vec3 ocean_color; + }; + struct CameraPosition { alignas(16) glm::vec4 position; @@ -118,13 +125,18 @@ class SubgroupsOperations : public ApiVulkanSample struct GuiConfig { - bool wireframe = {false}; - float amplitude = {1000.0f}; + bool wireframe = {true}; + float amplitude = {20.0f}; float length = {1000.0f}; glm::vec2 wind = {100.0f, -100.0f}; + + glm::vec3 light_pos = {100.0f, 15.0f, -1.0f}; + glm::vec3 light_color = {1.0f, 1.0f, 1.0f}; + glm::vec3 ocean_color = {0.0f, 0.2423423f, 0.434335435f}; } ui; uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; + std::unique_ptr ocean_params_ubo = {VK_NULL_HANDLE}; std::unique_ptr camera_postion_ubo = {VK_NULL_HANDLE}; std::unique_ptr camera_ubo = {VK_NULL_HANDLE}; std::unique_ptr tessellation_params_ubo = {VK_NULL_HANDLE}; diff --git a/shaders/subgroups_operations/fft_normal_map.comp b/shaders/subgroups_operations/fft_normal_map.comp index 7d3dae901..a6f9b93a3 100644 --- a/shaders/subgroups_operations/fft_normal_map.comp +++ b/shaders/subgroups_operations/fft_normal_map.comp @@ -19,9 +19,8 @@ */ layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; - layout (binding = 0, rgba32f) writeonly uniform image2D fft_normal_map; - // xyz + layout (binding = 1, rgba32f) readonly uniform image2D fft_displacement_map; @@ -39,19 +38,15 @@ void main() vec2 tex_coord = gl_GlobalInvocationID.xy / float(N); float tex_dim = 1.0f / N; - float z0 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(-tex_dim, -tex_dim))).g; - float z1 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(0.0f, -tex_dim))).g; - float z2 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(tex_dim, -tex_dim))).g; - float z3 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(-tex_dim, 0.0f))).g; - float z4 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(tex_dim, 0.0f))).g; - float z5 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(-tex_dim, tex_dim))).g; - float z6 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(0.0f, tex_dim))).g; - float z7 = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(tex_dim, tex_dim))).g; + float left_y = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(-tex_dim, 0.0f))).g; + float right_y = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(tex_dim, 0.0f))).g; + float bottom_y = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(0.0f, tex_dim))).g; + float top_y = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(0.0f, tex_dim))).g; vec3 result = vec3(0.0f); - result.z = z0 + 2.0f * z1 + z2 - z5 - 2.0f * z6 - z7; - result.x = z0 + 2.0f * z3 + z5 - z2 - 2.0f * z4 - z7; - result.y = 1.0f; + result.z = bottom_y - top_y; + result.x = left_y - right_y; + result.y = 1.0f / float(N); imageStore(fft_normal_map, pixel_pos, vec4(normalize(result), 1.0f)); } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index e9feaa32a..7dfbdbcd4 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -27,19 +27,37 @@ layout (binding = 3) uniform CameraPos vec4 position; } cam; +layout (binding = 5) uniform OceanParamsUbo +{ + vec3 light_color; + vec3 light_position; + vec3 ocean_color; +} ocean_ubo; + void main() { - vec3 light_pos = vec3(0.0f, 0.0f, -1.0f); - vec3 ocean_color = vec3(0.0f, 0.2423423f, 0.434335435f); - vec3 view_dir = normalize(vec3(cam.position.xyz - in_pos)); - vec3 ref_dir = reflect(-view_dir, in_normal); + vec3 normal = in_normal; + + vec3 light_pos = ocean_ubo.light_position; + vec3 light_color = ocean_ubo.light_color; + vec3 ocean_color = ocean_ubo.ocean_color; + float ambient_strength = 0.91f; + + vec3 ambient = ambient_strength * light_color; vec3 light_dir = normalize(light_pos - in_pos); + float diff = max(dot(normal, light_dir), 0.0f); + vec3 diffuse = diff * light_color; - float dot_l = max(dot(in_normal, light_dir), 0.0f); + float specular_strength = 0.5f; + vec3 view_dir = normalize(vec3(cam.position.xyz - in_pos)); + vec3 ref_dir = reflect(-light_dir, normal); + float spec = pow(max(dot(view_dir, ref_dir), 0.0f), 32); + vec3 specular = specular_strength * spec * light_color; - vec3 result = dot_l * ocean_color; + vec3 result = (ambient + diffuse + specular) * ocean_color; result = result / (result + vec3(1.0f)); + // gamma correction result = pow(result, vec3(1.0f / 2.2f)); diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index b59a298f4..14c09ca2d 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -31,7 +31,7 @@ layout (binding = 0) uniform Ubo mat4 model; } ubo; -layout (binding = 1, rgba32f) uniform image2D fftDisplacementMap; +layout (binding = 1, rgba32f) uniform image2D fft_displacement_map; layout (binding = 2) uniform TessellationParams { @@ -43,7 +43,7 @@ layout (binding = 4, rgba32f) uniform image2D fft_height_map; vec2 interpolate_2d(vec2 v0, vec2 v1, vec2 v2) { - return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2; + return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2; } vec3 interpolate_3d(vec3 v0, vec3 v1, vec3 v2) @@ -53,31 +53,32 @@ vec3 interpolate_3d(vec3 v0, vec3 v1, vec3 v2) vec4 interpolate_4d(vec4 v0, vec4 v1, vec4 v2) { - return vec4(gl_TessCoord.x) * v0 + vec4(gl_TessCoord.y) * v1 + vec4(gl_TessCoord.z) * v2; + return vec4(gl_TessCoord.x) * v0 + vec4(gl_TessCoord.y) * v1 + vec4(gl_TessCoord.z) * v2; } void main() { vec3 world_pos = interpolate_3d(inPostion[0], inPostion[1], inPostion[2]); - vec4 fft_texel_at_vertex[4]; - fft_texel_at_vertex[0] = imageLoad(fftDisplacementMap, ivec2(inUv[0])); - fft_texel_at_vertex[1] = imageLoad(fftDisplacementMap, ivec2(inUv[1])); - fft_texel_at_vertex[2] = imageLoad(fftDisplacementMap, ivec2(inUv[2])); + vec4 fft_texel_at_vertex[4]; + fft_texel_at_vertex[0] = imageLoad(fft_displacement_map, ivec2(inUv[0])); + fft_texel_at_vertex[1] = imageLoad(fft_displacement_map, ivec2(inUv[1])); + fft_texel_at_vertex[2] = imageLoad(fft_displacement_map, ivec2(inUv[2])); - vec4 height_texel_at_vertex[4]; - height_texel_at_vertex[0] = imageLoad(fft_height_map, ivec2(inUv[0])); - height_texel_at_vertex[1] = imageLoad(fft_height_map, ivec2(inUv[1])); - height_texel_at_vertex[2] = imageLoad(fft_height_map, ivec2(inUv[2])); + vec4 height_texel_at_vertex[4]; + height_texel_at_vertex[0] = imageLoad(fft_height_map, ivec2(inUv[0])); + height_texel_at_vertex[1] = imageLoad(fft_height_map, ivec2(inUv[1])); + height_texel_at_vertex[2] = imageLoad(fft_height_map, ivec2(inUv[2])); + + vec4 fft_texel = interpolate_4d(fft_texel_at_vertex[0], fft_texel_at_vertex[1], fft_texel_at_vertex[2]); + vec4 height_texel = interpolate_4d(height_texel_at_vertex[0], height_texel_at_vertex[1], height_texel_at_vertex[2]); - vec4 fft_texel = interpolate_4d(fft_texel_at_vertex[0], fft_texel_at_vertex[1], fft_texel_at_vertex[2]); - vec4 height_texel = interpolate_4d(height_texel_at_vertex[0], height_texel_at_vertex[1], height_texel_at_vertex[2]); world_pos.y += fft_texel.y * tessParams.displacement_scale; world_pos.x -= fft_texel.x * tessParams.choppines; world_pos.z -= fft_texel.z * tessParams.choppines; - outNormal = height_texel.xyz; + outNormal = height_texel.xyz; outPos = world_pos; gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); } From 9a8d989b1564d71a25f3683c0b8b1cf50ee39ba1 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Tue, 17 Oct 2023 15:44:38 +0200 Subject: [PATCH 36/53] ui update --- .../subgroups_operations/subgroups_operations.cpp | 6 +++--- .../extensions/subgroups_operations/subgroups_operations.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 2063e1abe..70d20f608 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -853,7 +853,7 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) if (drawer.header("Ocean settings")) { - drawer.input_float("Amplitude", &ui.amplitude, 0.1f, 3u); + drawer.input_float("Amplitude", &ui.amplitude, 1.f, 3u); drawer.input_float("Length", &ui.length, 10.f, 1u); if (drawer.header("Color")) { @@ -864,8 +864,8 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) if (drawer.header("Wind")) { - drawer.input_float("X", &ui.wind.x, 10.f, 2u); - drawer.input_float("Y", &ui.wind.y, 10.f, 2u); + drawer.slider_float("Angel", &ui.wind.x, 0.0f, 360.0f); + drawer.input_float("Force", &ui.wind.y, 10.f, 2u); } } } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index badb3ed4e..bf5994faf 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -126,8 +126,8 @@ class SubgroupsOperations : public ApiVulkanSample struct GuiConfig { bool wireframe = {true}; - float amplitude = {20.0f}; - float length = {1000.0f}; + float amplitude = {32.0f}; + float length = {1900.0f}; glm::vec2 wind = {100.0f, -100.0f}; glm::vec3 light_pos = {100.0f, 15.0f, -1.0f}; From 8b53cd652a2a762446a8ee0b20c7783722ebbe73 Mon Sep 17 00:00:00 2001 From: Krzysztof-Dmitruk-Mobica Date: Tue, 17 Oct 2023 16:16:10 +0200 Subject: [PATCH 37/53] Add wind coordinate conversion --- .../subgroups_operations.cpp | 17 ++++++++++++++--- .../subgroups_operations/subgroups_operations.h | 16 ++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 70d20f608..272342c54 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -415,6 +415,8 @@ void SubgroupsOperations::create_tildas() void SubgroupsOperations::load_assets() { generate_plane(); + ui.wind.recalc(); + log_2_N = glm::log2(static_cast(grid_size)); /* @@ -718,7 +720,7 @@ void SubgroupsOperations::update_uniform_buffers() fft_ubo.amplitude = ui.amplitude; fft_ubo.grid_size = grid_size; fft_ubo.length = ui.length; - fft_ubo.wind = ui.wind; + fft_ubo.wind = ui.wind.vec; FFTInvert invertFft; invertFft.grid_size = grid_size; @@ -864,8 +866,10 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) if (drawer.header("Wind")) { - drawer.slider_float("Angel", &ui.wind.x, 0.0f, 360.0f); - drawer.input_float("Force", &ui.wind.y, 10.f, 2u); + drawer.slider_float("Angle", &ui.wind.angle, 0.0f, 360.0f); + drawer.slider_float("Force", &ui.wind.force, 0.0f, 50.0f); + + ui.wind.recalc(); } } } @@ -1220,3 +1224,10 @@ VkDescriptorImageInfo SubgroupsOperations::create_fb_descriptor(FBAttachment &at image_descriptor.sampler = nullptr; return image_descriptor; } + +void SubgroupsOperations::Wind::recalc() +{ + float rad = angle * glm::pi() / 180.0f; + vec.x = force * cos(rad); + vec.y = force * sin(rad); +} diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index bf5994faf..81fbb53b2 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -123,12 +123,20 @@ class SubgroupsOperations : public ApiVulkanSample alignas(4) float time = {0.0f}; } fftTime; + struct Wind + { + void recalc(); + glm::vec2 vec; + float angle = 180; + float force = 25; + }; + struct GuiConfig { - bool wireframe = {true}; - float amplitude = {32.0f}; - float length = {1900.0f}; - glm::vec2 wind = {100.0f, -100.0f}; + bool wireframe = {true}; + float amplitude = {32.0f}; + float length = {1900.0f}; + Wind wind; glm::vec3 light_pos = {100.0f, 15.0f, -1.0f}; glm::vec3 light_color = {1.0f, 1.0f, 1.0f}; From da33331d3ae5692b3774dde9f66644ce1de822ce Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 18 Oct 2023 08:54:52 +0200 Subject: [PATCH 38/53] refactor --- .../subgroups_operations/CMakeLists.txt | 1 + .../subgroups_operations.cpp | 225 ++++++++++-------- .../subgroups_operations.h | 45 ++-- 3 files changed, 148 insertions(+), 123 deletions(-) diff --git a/samples/extensions/subgroups_operations/CMakeLists.txt b/samples/extensions/subgroups_operations/CMakeLists.txt index 68184909e..5609b73e0 100644 --- a/samples/extensions/subgroups_operations/CMakeLists.txt +++ b/samples/extensions/subgroups_operations/CMakeLists.txt @@ -34,4 +34,5 @@ add_sample( "subgroups_operations/butterfly_precomp.comp" "subgroups_operations/fft_tilde_h.comp" "subgroups_operations/fft_tilde_h0.comp" + "subgroups_operations/fft_normal_map.comp" "subgroups_operations/fft.comp") diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 272342c54..fe39ec2ed 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -20,7 +20,6 @@ #include -#define GRAVITY 9.81f void SubgroupsOperations::Pipeline::destroy(VkDevice device) { @@ -199,7 +198,7 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline_layout, 0u, 1u, &precompute.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, 1u, DISPLACEMENT_MAP_DIM, 1u); + vkCmdDispatch(compute.command_buffer, 1u, grid_size, 1u); } // initial tildas textures @@ -207,7 +206,7 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, initial_tildas.pipeline.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, initial_tildas.pipeline.pipeline_layout, 0u, 1u, &initial_tildas.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + vkCmdDispatch(compute.command_buffer, grid_size / 32u, grid_size, 1u); } // tildas textures @@ -215,64 +214,93 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline_layout, 0u, 1u, &tildas.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 8u, DISPLACEMENT_MAP_DIM, 1u); + vkCmdDispatch(compute.command_buffer, grid_size / 8u, grid_size, 1u); } // fft horizontal; for Y axis { vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + + for (uint32_t i = 0; i < log_2_N; ++i) + { + vkCmdPushConstants(compute.command_buffer, fft.pipelines.horizontal.pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(uint32_t), &i); + vkCmdDispatch(compute.command_buffer, grid_size / 32u, grid_size, 1u); + } } - /* - // fft horizontal; for X axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } + // fft horizontal; for X axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); - // fft horizontal; for Z axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } + for (uint32_t i = 0; i < log_2_N; ++i) + { + vkCmdPushConstants(compute.command_buffer, fft.pipelines.horizontal.pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(uint32_t), &i); + vkCmdDispatch(compute.command_buffer, grid_size / 32u, grid_size, 1u); + } + } - // fft vertical; for Y axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } + // fft horizontal; for Z axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.horizontal.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); - // fft vertical; for X axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } + for (uint32_t i = 0; i < log_2_N; ++i) + { + vkCmdPushConstants(compute.command_buffer, fft.pipelines.horizontal.pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(uint32_t), &i); + vkCmdDispatch(compute.command_buffer, grid_size / 32u, grid_size, 1u); + } + } - // fft vertical; for Z axis - { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); - } + // fft vertical; for Y axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_y, 0u, nullptr); + + for (uint32_t i = 0; i < log_2_N; ++i) + { + vkCmdPushConstants(compute.command_buffer, fft.pipelines.vertical.pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(uint32_t), &i); + vkCmdDispatch(compute.command_buffer, grid_size / 32u, grid_size, 1u); + } + } + + // fft vertical; for X axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_x, 0u, nullptr); + + for (uint32_t i = 0; i < log_2_N; ++i) + { + vkCmdPushConstants(compute.command_buffer, fft.pipelines.vertical.pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(uint32_t), &i); + vkCmdDispatch(compute.command_buffer, grid_size / 32u, grid_size, 1u); + } + } + + // fft vertical; for Z axis + { + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft.pipelines.vertical.pipeline_layout, 0u, 1u, &fft.descriptor_set_axis_z, 0u, nullptr); + + for (uint32_t i = 0; i < log_2_N; ++i) + { + vkCmdPushConstants(compute.command_buffer, fft.pipelines.vertical.pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(uint32_t), &i); + vkCmdDispatch(compute.command_buffer, grid_size / 32u, grid_size, 1u); + } + } // fft inverse { vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_inversion.pipeline.pipeline_layout, 0u, 1u, &fft_inversion.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + vkCmdDispatch(compute.command_buffer, grid_size / 32u, grid_size, 1u); } // fft normal map { vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_normal_map.pipeline.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, fft_normal_map.pipeline.pipeline_layout, 0u, 1u, &fft_normal_map.descriptor_set, 0u, nullptr); - vkCmdDispatch(compute.command_buffer, DISPLACEMENT_MAP_DIM / 32u, DISPLACEMENT_MAP_DIM, 1u); + vkCmdDispatch(compute.command_buffer, grid_size / 32u, grid_size, 1u); } VK_CHECK(vkEndCommandBuffer(compute.command_buffer)); @@ -334,14 +362,14 @@ void SubgroupsOperations::create_initial_tildas() VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &initial_tildas.pipeline.pipeline)); - fft_buffers.fft_input_htilde0 = std::make_unique(); - fft_buffers.fft_input_htilde0_conj = std::make_unique(); + fft_buffers.fft_input_htilde0 = std::make_unique(); + fft_buffers.fft_input_htilde0_conj = std::make_unique(); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_input_htilde0); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_input_htilde0_conj); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_input_htilde0); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_input_htilde0_conj); - VkDescriptorImageInfo htilde_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0); - VkDescriptorImageInfo htilde_conj_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0_conj); + VkDescriptorImageInfo htilde_0_descriptor = create_ia_descriptor(*fft_buffers.fft_input_htilde0); + VkDescriptorImageInfo htilde_conj_0_descriptor = create_ia_descriptor(*fft_buffers.fft_input_htilde0_conj); VkDescriptorBufferInfo input_random_descriptor = create_descriptor(*fft_buffers.fft_input_random); VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); @@ -355,13 +383,13 @@ void SubgroupsOperations::create_initial_tildas() void SubgroupsOperations::create_tildas() { - fft_buffers.fft_tilde_h_kt_dx = std::make_unique(); - fft_buffers.fft_tilde_h_kt_dy = std::make_unique(); - fft_buffers.fft_tilde_h_kt_dz = std::make_unique(); + fft_buffers.fft_tilde_h_kt_dx = std::make_unique(); + fft_buffers.fft_tilde_h_kt_dy = std::make_unique(); + fft_buffers.fft_tilde_h_kt_dz = std::make_unique(); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dx); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dy); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dz); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dx); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dy); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dz); std::vector set_layout_bindngs = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), @@ -391,12 +419,12 @@ void SubgroupsOperations::create_tildas() VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &tildas.pipeline.pipeline)); - VkDescriptorImageInfo htilde_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0); - VkDescriptorImageInfo htilde_conj_0_descriptor = create_fb_descriptor(*fft_buffers.fft_input_htilde0_conj); + VkDescriptorImageInfo htilde_0_descriptor = create_ia_descriptor(*fft_buffers.fft_input_htilde0); + VkDescriptorImageInfo htilde_conj_0_descriptor = create_ia_descriptor(*fft_buffers.fft_input_htilde0_conj); - VkDescriptorImageInfo image_dx_descriptor = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dx); - VkDescriptorImageInfo image_dy_descriptor = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dy); - VkDescriptorImageInfo image_dz_descriptor = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dz); + VkDescriptorImageInfo image_dx_descriptor = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dx); + VkDescriptorImageInfo image_dy_descriptor = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dy); + VkDescriptorImageInfo image_dz_descriptor = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dz); VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); VkDescriptorBufferInfo fft_time_ubo_buffer = create_descriptor(*fft_time_ubo); @@ -488,11 +516,11 @@ glm::vec2 SubgroupsOperations::rndGaussian() void SubgroupsOperations::prepare_uniform_buffers() { camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - camera_postion_ubo = std::make_unique(get_device(), sizeof(CameraPosition), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + camera_postion_ubo = std::make_unique(get_device(), sizeof(CameraPositionUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - invert_fft_ubo = std::make_unique(get_device(), sizeof(FFTInvert), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - tessellation_params_ubo = std::make_unique(get_device(), sizeof(TessellationParams), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + invert_fft_ubo = std::make_unique(get_device(), sizeof(FFTInvertUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + tessellation_params_ubo = std::make_unique(get_device(), sizeof(TessellationParamsUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); ocean_params_ubo = std::make_unique(get_device(), sizeof(OceanParamsUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); update_uniform_buffers(); @@ -608,10 +636,10 @@ void SubgroupsOperations::create_descriptor_set() VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &ocean.descriptor_set)); VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*camera_ubo); - VkDescriptorImageInfo desplacement_descriptor = create_fb_descriptor(*fft_buffers.fft_displacement); + VkDescriptorImageInfo desplacement_descriptor = create_ia_descriptor(*fft_buffers.fft_displacement); VkDescriptorBufferInfo tessellation_params_descriptor = create_descriptor(*tessellation_params_ubo); VkDescriptorBufferInfo camera_pos_buffer_descriptor = create_descriptor(*camera_postion_ubo); - VkDescriptorImageInfo normal_map_descirptor = create_fb_descriptor(*fft_buffers.fft_normal_map); + VkDescriptorImageInfo normal_map_descirptor = create_ia_descriptor(*fft_buffers.fft_normal_map); VkDescriptorBufferInfo ocean_params_buffer_descriptor = create_descriptor(*ocean_params_ubo); std::vector write_descriptor_sets = { @@ -713,7 +741,7 @@ void SubgroupsOperations::update_uniform_buffers() ubo.view = camera.matrices.view; ubo.projection = camera.matrices.perspective; - CameraPosition cam_pos; + CameraPositionUbo cam_pos; cam_pos.position = glm::vec4(camera.position, 0.0f); FFTParametersUbo fft_ubo; @@ -722,11 +750,11 @@ void SubgroupsOperations::update_uniform_buffers() fft_ubo.length = ui.length; fft_ubo.wind = ui.wind.vec; - FFTInvert invertFft; + FFTInvertUbo invertFft; invertFft.grid_size = grid_size; invertFft.page_idx = log_2_N % 2; - TessellationParams tess_params; + TessellationParamsUbo tess_params; tess_params.displacement_scale = 0.5f; tess_params.choppines = 0.75f; @@ -867,7 +895,7 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) if (drawer.header("Wind")) { drawer.slider_float("Angle", &ui.wind.angle, 0.0f, 360.0f); - drawer.slider_float("Force", &ui.wind.force, 0.0f, 50.0f); + drawer.slider_float("Force", &ui.wind.force, 0.1f, 50.0f); ui.wind.recalc(); } @@ -903,7 +931,7 @@ std::unique_ptr create_subgroups_operations() } // TODO: move out usage to the function arguments -void SubgroupsOperations::createFBAttachement(VkFormat format, uint32_t width, uint32_t height, FBAttachment &attachment) +void SubgroupsOperations::create_image_attachement(VkFormat format, uint32_t width, uint32_t height, ImageAttachment &attachment) { attachment.format = format; @@ -1001,20 +1029,17 @@ void SubgroupsOperations::create_butterfly_texture() VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &precompute.pipeline.pipeline)); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, log_2_N, DISPLACEMENT_MAP_DIM, butterfly_precomp); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, log_2_N, grid_size, butterfly_precomp); - std::array bit_reverse_arr; - for (uint32_t i = 0; i < DISPLACEMENT_MAP_DIM; ++i) - bit_reverse_arr[i] = reverse(i); + std::vector bit_reverse_arr; + for (uint32_t i = 0; i < grid_size; ++i) + bit_reverse_arr.push_back(reverse(i)); bit_reverse_buffer = std::make_unique(get_device(), sizeof(uint32_t) * bit_reverse_arr.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); bit_reverse_buffer->update(bit_reverse_arr.data(), sizeof(uint32_t) * bit_reverse_arr.size()); VkDescriptorBufferInfo bit_reverse_descriptor = create_descriptor(*bit_reverse_buffer); - bit_reverse_buffer->update(bit_reverse_arr.data(), sizeof(uint32_t) * bit_reverse_arr.size()); - - VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); - VkDescriptorImageInfo image_descriptor = create_fb_descriptor(butterfly_precomp); + VkDescriptorImageInfo image_descriptor = create_ia_descriptor(butterfly_precomp); VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); std::vector write_descriptor_sets = { @@ -1074,16 +1099,16 @@ void SubgroupsOperations::create_fft() VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft.pipelines.vertical.pipeline)); - fft.tilde_axis_y = std::make_unique(); - fft.tilde_axis_x = std::make_unique(); - fft.tilde_axis_z = std::make_unique(); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_y); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_x); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_z); + fft.tilde_axis_y = std::make_unique(); + fft.tilde_axis_x = std::make_unique(); + fft.tilde_axis_z = std::make_unique(); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_y); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_x); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_z); - VkDescriptorImageInfo image_descriptor_battlefly = create_fb_descriptor(butterfly_precomp); - VkDescriptorImageInfo image_descriptor_tilda_y = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dy); - VkDescriptorImageInfo image_descriptor_tilde_axis_y = create_fb_descriptor(*fft.tilde_axis_y); + VkDescriptorImageInfo image_descriptor_battlefly = create_ia_descriptor(butterfly_precomp); + VkDescriptorImageInfo image_descriptor_tilda_y = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dy); + VkDescriptorImageInfo image_descriptor_tilde_axis_y = create_ia_descriptor(*fft.tilde_axis_y); std::vector write_descriptor_sets_asix_y = { vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), @@ -1093,8 +1118,8 @@ void SubgroupsOperations::create_fft() vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_y.size()), write_descriptor_sets_asix_y.data(), 0u, nullptr); - VkDescriptorImageInfo image_descriptor_tilda_x = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dx); - VkDescriptorImageInfo image_descriptor_tilde_axis_x = create_fb_descriptor(*fft.tilde_axis_x); + VkDescriptorImageInfo image_descriptor_tilda_x = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dx); + VkDescriptorImageInfo image_descriptor_tilde_axis_x = create_ia_descriptor(*fft.tilde_axis_x); std::vector write_descriptor_sets_asix_x = { vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), @@ -1104,8 +1129,8 @@ void SubgroupsOperations::create_fft() vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_x.size()), write_descriptor_sets_asix_x.data(), 0u, nullptr); - VkDescriptorImageInfo image_descriptor_tilda_z = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dz); - VkDescriptorImageInfo image_descriptor_tilde_axis_z = create_fb_descriptor(*fft.tilde_axis_z); + VkDescriptorImageInfo image_descriptor_tilda_z = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dz); + VkDescriptorImageInfo image_descriptor_tilde_axis_z = create_ia_descriptor(*fft.tilde_axis_z); std::vector write_descriptor_sets_asix_z = { vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), @@ -1148,17 +1173,17 @@ void SubgroupsOperations::create_fft_inversion() VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft_inversion.pipeline.pipeline)); - fft_buffers.fft_displacement = std::make_unique(); + fft_buffers.fft_displacement = std::make_unique(); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_displacement); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_displacement); - VkDescriptorImageInfo image_descriptor_displacment_axis = create_fb_descriptor(*fft_buffers.fft_displacement); - VkDescriptorImageInfo image_descriptor_pingpong0_axis_y = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dy); - VkDescriptorImageInfo image_descriptor_pingpong1_axis_y = create_fb_descriptor(*fft.tilde_axis_y); - VkDescriptorImageInfo image_descriptor_pingpong0_axis_x = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dx); - VkDescriptorImageInfo image_descriptor_pingpong1_axis_x = create_fb_descriptor(*fft.tilde_axis_x); - VkDescriptorImageInfo image_descriptor_pingpong0_axis_z = create_fb_descriptor(*fft_buffers.fft_tilde_h_kt_dz); - VkDescriptorImageInfo image_descriptor_pingpong1_axis_z = create_fb_descriptor(*fft.tilde_axis_z); + VkDescriptorImageInfo image_descriptor_displacment_axis = create_ia_descriptor(*fft_buffers.fft_displacement); + VkDescriptorImageInfo image_descriptor_pingpong0_axis_y = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dy); + VkDescriptorImageInfo image_descriptor_pingpong1_axis_y = create_ia_descriptor(*fft.tilde_axis_y); + VkDescriptorImageInfo image_descriptor_pingpong0_axis_x = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dx); + VkDescriptorImageInfo image_descriptor_pingpong1_axis_x = create_ia_descriptor(*fft.tilde_axis_x); + VkDescriptorImageInfo image_descriptor_pingpong0_axis_z = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dz); + VkDescriptorImageInfo image_descriptor_pingpong1_axis_z = create_ia_descriptor(*fft.tilde_axis_z); auto fft_page_descriptor = create_descriptor(*invert_fft_ubo); @@ -1201,12 +1226,12 @@ void SubgroupsOperations::create_fft_normal_map() VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &fft_normal_map.pipeline.pipeline)); - fft_buffers.fft_normal_map = std::make_unique(); + fft_buffers.fft_normal_map = std::make_unique(); - createFBAttachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_normal_map); + create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_normal_map); - VkDescriptorImageInfo image_descriptor_normal_map = create_fb_descriptor(*fft_buffers.fft_normal_map); - VkDescriptorImageInfo image_descriptor_displacment_axis = create_fb_descriptor(*fft_buffers.fft_displacement); + VkDescriptorImageInfo image_descriptor_normal_map = create_ia_descriptor(*fft_buffers.fft_normal_map); + VkDescriptorImageInfo image_descriptor_displacment_axis = create_ia_descriptor(*fft_buffers.fft_displacement); auto fft_page_descriptor = create_descriptor(*invert_fft_ubo); std::vector write_descriptor_sets = { @@ -1216,7 +1241,7 @@ void SubgroupsOperations::create_fft_normal_map() vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } -VkDescriptorImageInfo SubgroupsOperations::create_fb_descriptor(FBAttachment &attachment) +VkDescriptorImageInfo SubgroupsOperations::create_ia_descriptor(ImageAttachment &attachment) { VkDescriptorImageInfo image_descriptor{}; image_descriptor.imageView = attachment.view; diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 81fbb53b2..6fe20ec88 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -18,9 +18,6 @@ #pragma once #include "api_vulkan_sample.h" -#include - -#define DISPLACEMENT_MAP_DIM 256u class SubgroupsOperations : public ApiVulkanSample { @@ -93,12 +90,12 @@ class SubgroupsOperations : public ApiVulkanSample alignas(16) glm::vec3 ocean_color; }; - struct CameraPosition + struct CameraPositionUbo { alignas(16) glm::vec4 position; }; - struct TessellationParams + struct TessellationParamsUbo { alignas(4) float choppines; alignas(4) float displacement_scale; @@ -112,10 +109,10 @@ class SubgroupsOperations : public ApiVulkanSample alignas(8) glm::vec2 wind; }; - struct FFTInvert + struct FFTInvertUbo { alignas(4) int32_t page_idx = {-1}; - alignas(4) uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; + alignas(4) uint32_t grid_size = {grid_size}; }; struct TimeUbo @@ -143,7 +140,7 @@ class SubgroupsOperations : public ApiVulkanSample glm::vec3 ocean_color = {0.0f, 0.2423423f, 0.434335435f}; } ui; - uint32_t grid_size = {DISPLACEMENT_MAP_DIM}; + uint32_t grid_size = {256U}; std::unique_ptr ocean_params_ubo = {VK_NULL_HANDLE}; std::unique_ptr camera_postion_ubo = {VK_NULL_HANDLE}; std::unique_ptr camera_ubo = {VK_NULL_HANDLE}; @@ -155,7 +152,7 @@ class SubgroupsOperations : public ApiVulkanSample std::vector input_random; - struct FBAttachment + struct ImageAttachment { VkImage image; VkDeviceMemory memory; @@ -167,7 +164,9 @@ class SubgroupsOperations : public ApiVulkanSample vkDestroyImage(device, image, nullptr); vkFreeMemory(device, memory, nullptr); }; - } butterfly_precomp; + }; + + ImageAttachment butterfly_precomp; uint32_t log_2_N; vkb::Timer timer; @@ -175,14 +174,14 @@ class SubgroupsOperations : public ApiVulkanSample struct { std::unique_ptr fft_input_random; - std::unique_ptr fft_input_htilde0; - std::unique_ptr fft_input_htilde0_conj; - - std::unique_ptr fft_tilde_h_kt_dx; - std::unique_ptr fft_tilde_h_kt_dy; - std::unique_ptr fft_tilde_h_kt_dz; - std::unique_ptr fft_displacement; - std::unique_ptr fft_normal_map; + std::unique_ptr fft_input_htilde0; + std::unique_ptr fft_input_htilde0_conj; + + std::unique_ptr fft_tilde_h_kt_dx; + std::unique_ptr fft_tilde_h_kt_dy; + std::unique_ptr fft_tilde_h_kt_dz; + std::unique_ptr fft_displacement; + std::unique_ptr fft_normal_map; } fft_buffers; struct @@ -207,9 +206,9 @@ class SubgroupsOperations : public ApiVulkanSample Pipeline vertical; } pipelines; - std::unique_ptr tilde_axis_y = {VK_NULL_HANDLE}; - std::unique_ptr tilde_axis_x = {VK_NULL_HANDLE}; - std::unique_ptr tilde_axis_z = {VK_NULL_HANDLE}; + std::unique_ptr tilde_axis_y = {VK_NULL_HANDLE}; + std::unique_ptr tilde_axis_x = {VK_NULL_HANDLE}; + std::unique_ptr tilde_axis_z = {VK_NULL_HANDLE}; } fft; struct @@ -266,8 +265,8 @@ class SubgroupsOperations : public ApiVulkanSample private: uint32_t reverse(uint32_t i); - VkDescriptorImageInfo create_fb_descriptor(FBAttachment &attachment); - void createFBAttachement(VkFormat format, uint32_t width, uint32_t height, FBAttachment &result); + VkDescriptorImageInfo create_ia_descriptor(ImageAttachment &attachment); + void create_image_attachement(VkFormat format, uint32_t width, uint32_t height, ImageAttachment &result); }; std::unique_ptr create_subgroups_operations(); From 6345881134898f2538ab941cfcdd88953640b5e6 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 18 Oct 2023 09:14:11 +0200 Subject: [PATCH 39/53] shader clean-up --- .../subgroups_operations.cpp | 4 - .../butterfly_precomp.comp | 5 +- shaders/subgroups_operations/fft.comp | 156 ++++++++++-------- shaders/subgroups_operations/fft_invert.comp | 6 +- .../subgroups_operations/fft_normal_map.comp | 2 - .../subgroups_operations/fft_tilde_h0.comp | 27 ++- shaders/subgroups_operations/ocean.frag | 2 - shaders/subgroups_operations/ocean.tesc | 1 - shaders/subgroups_operations/ocean.tese | 1 - 9 files changed, 109 insertions(+), 95 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index fe39ec2ed..319848a91 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -20,7 +20,6 @@ #include - void SubgroupsOperations::Pipeline::destroy(VkDevice device) { if (pipeline != VK_NULL_HANDLE) @@ -48,9 +47,6 @@ SubgroupsOperations::SubgroupsOperations() // Required by VK_KHR_spirv_1_4 add_device_extension(VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME); - // TODO: remove when not needed; for #extension GL_EXT_debug_printf : enable - add_device_extension(VK_KHR_SHADER_NON_SEMANTIC_INFO_EXTENSION_NAME); - // Targeting SPIR-V version vkb::GLSLCompiler::set_target_environment(glslang::EShTargetSpv, glslang::EShTargetSpv_1_4); diff --git a/shaders/subgroups_operations/butterfly_precomp.comp b/shaders/subgroups_operations/butterfly_precomp.comp index 6ea091acd..67202fbe7 100644 --- a/shaders/subgroups_operations/butterfly_precomp.comp +++ b/shaders/subgroups_operations/butterfly_precomp.comp @@ -19,7 +19,7 @@ #define PI_32F 3.14159265358979f -layout (local_size_x = 8, local_size_y = 1) in; +layout (local_size_x = 8, local_size_y = 1, local_size_z = 1) in; layout (binding = 0, rgba32f) writeonly uniform image2D u_butterfly_precomp; @@ -36,12 +36,9 @@ layout (binding = 2) uniform FFTParametersUbo vec2 wind; } fftUbo; - - void main() { vec2 pos = gl_GlobalInvocationID.xy; - uint N = fftUbo.grid_size; // Twiddle factor exponent, Thesis, Section 4.2.6, Eq 4.6 diff --git a/shaders/subgroups_operations/fft.comp b/shaders/subgroups_operations/fft.comp index 366c4634b..e48b10c0e 100644 --- a/shaders/subgroups_operations/fft.comp +++ b/shaders/subgroups_operations/fft.comp @@ -1,13 +1,5 @@ #version 450 #extension GL_KHR_shader_subgroup_basic : enable -<<<<<<< HEAD -======= -#extension GL_EXT_debug_printf : enable - -precision highp float; - - ->>>>>>> e005a65 (Add tilde_h_0 shader) /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 @@ -33,91 +25,113 @@ layout (binding = 0, rgba32f) readonly uniform image2D u_butterfly_precomp; layout (binding = 1, rgba32f) uniform image2D u_pingpong0; layout (binding = 2, rgba32f) uniform image2D u_pingpong1; -layout( push_constant ) uniform Push_Constants{ - uint i; -} step; +layout (binding = 3) uniform FftPage +{ + int pages; +} fftUbo; struct Complex { - float real; - float imag; + float real; + float imag; }; Complex complex_add(Complex c1, Complex c2) { - Complex res; - res.real = c1.real + c2.real; - res.imag = c1.imag + c2.imag; - return res; + Complex res; + res.real = c1.real + c2.real; + res.imag = c1.imag + c2.imag; + return res; } Complex complex_multiply(Complex c1, Complex c2) { - Complex res; - res.real = c1.real * c2.real - c1.imag * c2.imag; - res.imag = c1.real * c2.imag + c1.imag * c2.real; - return res; + Complex res; + res.real = c1.real * c2.real - c1.imag * c2.imag; + res.imag = c1.real * c2.imag + c1.imag * c2.real; + return res; } void HorizontalButterflies(in ivec2 pixel_pos) { - vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(step.i, pixel_pos.x)); - if ((step.i % 2) == 0) - { - vec2 a_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; - vec2 b_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); - } - else + for (int i = 0; i < fftUbo.pages; ++i) { - vec2 a_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; - vec2 b_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + subgroupBarrier(); + vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(i, pixel_pos.x)); + if ((i % 2) == 0) + { + vec2 a_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; + vec2 b_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + subgroupMemoryBarrierImage(); + imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + + //if(i == 6) + // debugPrintfEXT("[%i] %.5f, %.5f", i, result.real, result.imag); + + } + else + { + vec2 a_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; + vec2 b_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + subgroupMemoryBarrierImage(); + imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + + if(i == 5) + debugPrintfEXT("[%i] %.5f, %.5f", i, result.real, result.imag); + } } } void VerticalButterfiles(in ivec2 pixel_pos) { - vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(step.i, pixel_pos.y)); - if ((step.i % 2) == 0) - { - vec2 a_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; - vec2 b_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); - } - else + for (int i = 0; i < fftUbo.pages; ++i) { - vec2 a_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; - vec2 b_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(i, pixel_pos.y)); + + if ((i % 2) == 0) + { + vec2 a_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; + vec2 b_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + subgroupMemoryBarrierImage(); + + imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + } + else + { + vec2 a_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; + vec2 b_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + // Buttefly operation + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + subgroupMemoryBarrierImage(); + + imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + } } } diff --git a/shaders/subgroups_operations/fft_invert.comp b/shaders/subgroups_operations/fft_invert.comp index c3e544222..0996b5521 100644 --- a/shaders/subgroups_operations/fft_invert.comp +++ b/shaders/subgroups_operations/fft_invert.comp @@ -38,10 +38,8 @@ layout (binding = 7) uniform InvertFft void main() { - uint N = fftUbo.grid_size; - - ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); + ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); float perms[2] = { 1.0f, -1.0f }; int index = int(mod(pixel_pos.x + pixel_pos.y, 2)); @@ -52,7 +50,7 @@ void main() float h_y = imageLoad(u_pingpong0_y, pixel_pos).r; float h_x = imageLoad(u_pingpong0_x, pixel_pos).r; float h_z = imageLoad(u_pingpong0_z, pixel_pos).r; - imageStore(u_displacement, pixel_pos, vec4(perm * (h_x / float(N * N)), perm * (h_y / float(N * N)), perm * (h_z / float(N * N)), 1.0f)); + imageStore(u_displacement, pixel_pos, vec4(perm * (h_x / float(N * N)), perm * (h_y / float(N * N)), perm * (h_z / float(N * N)), 1.0f)); } else if (pingpong_index == 1) { diff --git a/shaders/subgroups_operations/fft_normal_map.comp b/shaders/subgroups_operations/fft_normal_map.comp index a6f9b93a3..8af783cb2 100644 --- a/shaders/subgroups_operations/fft_normal_map.comp +++ b/shaders/subgroups_operations/fft_normal_map.comp @@ -1,6 +1,5 @@ #version 450 #extension GL_KHR_shader_subgroup_basic : enable - /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 @@ -23,7 +22,6 @@ layout (binding = 1, rgba32f) readonly uniform image2D fft_displacement_map; - layout (binding = 2) uniform InvertFft { int pong_idx; diff --git a/shaders/subgroups_operations/fft_tilde_h0.comp b/shaders/subgroups_operations/fft_tilde_h0.comp index c6e35fd42..b747e3933 100644 --- a/shaders/subgroups_operations/fft_tilde_h0.comp +++ b/shaders/subgroups_operations/fft_tilde_h0.comp @@ -1,17 +1,33 @@ -#version 460 core +#version 450 +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #define PI 3.14159265358979f #define GRAVITY 9.81f -layout (local_size_x = 32, local_size_y = 32) in; +layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; layout (binding = 0, rgba32f) uniform image2D tilde_h0_k; layout (binding = 1, rgba32f) uniform image2D tilde_h0_minus_k; + layout (std140, binding = 2) readonly buffer InputRandom { vec4 data[]; -} -input_random; +} input_random; layout (std140, binding = 3) uniform FFTParametersUbo { @@ -19,8 +35,7 @@ layout (std140, binding = 3) uniform FFTParametersUbo float len; uint grid_size; vec2 wind; -} -input_params; +} input_params; float SuppressionFactor(float k_magnitude_sq) { diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index 7dfbdbcd4..3c68f8ce4 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -21,7 +21,6 @@ layout (location = 1) in vec3 in_normal; layout (location = 0) out vec4 outFragColor; - layout (binding = 3) uniform CameraPos { vec4 position; @@ -37,7 +36,6 @@ layout (binding = 5) uniform OceanParamsUbo void main() { vec3 normal = in_normal; - vec3 light_pos = ocean_ubo.light_position; vec3 light_color = ocean_ubo.light_color; vec3 ocean_color = ocean_ubo.ocean_color; diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 374905867..a904d509e 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -48,7 +48,6 @@ void main() outPos[gl_InvocationID] = inPostion[gl_InvocationID]; outUv[gl_InvocationID] = inUv[gl_InvocationID]; - float dist_cam_v0 = distance(cam.position.xyz, outPos[0]); float dist_cam_v1 = distance(cam.position.xyz, outPos[1]); float dist_cam_v2 = distance(cam.position.xyz, outPos[2]); diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 14c09ca2d..205b03ae3 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -73,7 +73,6 @@ void main() vec4 fft_texel = interpolate_4d(fft_texel_at_vertex[0], fft_texel_at_vertex[1], fft_texel_at_vertex[2]); vec4 height_texel = interpolate_4d(height_texel_at_vertex[0], height_texel_at_vertex[1], height_texel_at_vertex[2]); - world_pos.y += fft_texel.y * tessParams.displacement_scale; world_pos.x -= fft_texel.x * tessParams.choppines; world_pos.z -= fft_texel.z * tessParams.choppines; From fe174b15535290c82b46b91ddad218e152afb51d Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 18 Oct 2023 10:06:29 +0200 Subject: [PATCH 40/53] add skybox --- .../subgroups_operations/CMakeLists.txt | 2 + .../subgroups_operations.cpp | 121 ++++++++++++++++++ .../subgroups_operations.h | 25 +++- shaders/subgroups_operations/ocean.tesc | 6 +- shaders/subgroups_operations/skybox.frag | 29 +++++ shaders/subgroups_operations/skybox.vert | 34 +++++ 6 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 shaders/subgroups_operations/skybox.frag create mode 100644 shaders/subgroups_operations/skybox.vert diff --git a/samples/extensions/subgroups_operations/CMakeLists.txt b/samples/extensions/subgroups_operations/CMakeLists.txt index 5609b73e0..441027ac0 100644 --- a/samples/extensions/subgroups_operations/CMakeLists.txt +++ b/samples/extensions/subgroups_operations/CMakeLists.txt @@ -26,6 +26,8 @@ add_sample( NAME "subgroups_operations" DESCRIPTION "Demonstrates the use of a subgroups feature" SHADER_FILES_GLSL + "subgroups_operations/skybox.vert" + "subgroups_operations/skybox.frag" "subgroups_operations/ocean.vert" "subgroups_operations/ocean.frag" "subgroups_operations/ocean.tesc" diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 319848a91..e04f8f0ea 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -96,6 +96,8 @@ SubgroupsOperations::~SubgroupsOperations() vkDestroySemaphore(get_device().get_handle(), compute.semaphore, nullptr); vkDestroyCommandPool(get_device().get_handle(), compute.command_pool, nullptr); + skybox.destroy(get_device().get_handle()); + ocean.pipelines._default.destroy(get_device().get_handle()); ocean.pipelines.wireframe.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), ocean.descriptor_set_layout, nullptr); @@ -130,6 +132,7 @@ bool SubgroupsOperations::prepare(vkb::Platform &platform) create_descriptor_set(); create_pipelines(); + create_skybox(); build_compute_command_buffer(); @@ -438,6 +441,9 @@ void SubgroupsOperations::create_tildas() void SubgroupsOperations::load_assets() { + skybox.skybox_shape = load_model("scenes/geosphere.gltf"); + skybox.skybox_texture = load_texture("textures/skysphere_rgba.ktx", vkb::sg::Image::Color); + generate_plane(); ui.wind.recalc(); @@ -511,6 +517,7 @@ glm::vec2 SubgroupsOperations::rndGaussian() void SubgroupsOperations::prepare_uniform_buffers() { + skybox_ubo = std::make_unique(get_device(), sizeof(SkyboxUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); camera_postion_ubo = std::make_unique(get_device(), sizeof(CameraPositionUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); @@ -729,6 +736,109 @@ void SubgroupsOperations::create_pipelines() } } +void SubgroupsOperations::create_skybox() +{ + // descriptors + std::vector set_layout_bindngs = { + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT, 0u), + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, VK_SHADER_STAGE_FRAGMENT_BIT, 1u)}; + + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &skybox.descriptor_set_layout)); + + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &skybox.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &skybox.descriptor_set)); + + VkPipelineLayoutCreateInfo pipeline_layout_info = {}; + pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + pipeline_layout_info.setLayoutCount = 1u; + pipeline_layout_info.pSetLayouts = &skybox.descriptor_set_layout; + + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &pipeline_layout_info, nullptr, &skybox.pipeline.pipeline_layout)); + + VkDescriptorBufferInfo skybox_uniform_descriptor = create_descriptor(*skybox_ubo); + VkDescriptorImageInfo skybox_image_descriptor = create_descriptor(skybox.skybox_texture); + + std::vector write_descriptor_sets = { + vkb::initializers::write_descriptor_set(skybox.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &skybox_uniform_descriptor), + vkb::initializers::write_descriptor_set(skybox.descriptor_set, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1u, &skybox_image_descriptor), + }; + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); + + // pipeline + VkPipelineInputAssemblyStateCreateInfo input_assembly_state = + vkb::initializers::pipeline_input_assembly_state_create_info( + VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, + 0u, + VK_FALSE); + VkPipelineRasterizationStateCreateInfo rasterization_state = + vkb::initializers::pipeline_rasterization_state_create_info( + VK_POLYGON_MODE_FILL, + VK_CULL_MODE_NONE, + VK_FRONT_FACE_COUNTER_CLOCKWISE, + 0u); + VkPipelineColorBlendAttachmentState blend_attachment_state = + vkb::initializers::pipeline_color_blend_attachment_state( + 0xf, + VK_FALSE); + VkPipelineColorBlendStateCreateInfo color_blend_state = + vkb::initializers::pipeline_color_blend_state_create_info( + 1u, + &blend_attachment_state); + VkPipelineDepthStencilStateCreateInfo depth_stencil_state = + vkb::initializers::pipeline_depth_stencil_state_create_info( + VK_TRUE, + VK_TRUE, + VK_COMPARE_OP_GREATER); + + VkPipelineViewportStateCreateInfo viewport_state = + vkb::initializers::pipeline_viewport_state_create_info(1u, 1u, 0u); + + VkPipelineMultisampleStateCreateInfo multisample_state = + vkb::initializers::pipeline_multisample_state_create_info( + VK_SAMPLE_COUNT_1_BIT, + 0u); + + std::vector dynamic_state_enables = { + VK_DYNAMIC_STATE_VIEWPORT, + VK_DYNAMIC_STATE_SCISSOR}; + + VkPipelineDynamicStateCreateInfo dynamic_state = + vkb::initializers::pipeline_dynamic_state_create_info( + dynamic_state_enables.data(), + static_cast(dynamic_state_enables.size()), + 0u); + + std::array shader_stages; + shader_stages[0] = load_shader("subgroups_operations/skybox.vert", VK_SHADER_STAGE_VERTEX_BIT); + shader_stages[1] = load_shader("subgroups_operations/skybox.frag", VK_SHADER_STAGE_FRAGMENT_BIT); + + const std::vector vertex_input_bindings = { + vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX)}; + const std::vector vertex_input_attributes = { + vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), + vkb::initializers::vertex_input_attribute_description(0u, 1u, VK_FORMAT_R32G32_SFLOAT, offsetof(Vertex, uv))}; + + VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); + vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); + vertex_input_state.pVertexBindingDescriptions = vertex_input_bindings.data(); + vertex_input_state.vertexAttributeDescriptionCount = static_cast(vertex_input_attributes.size()); + vertex_input_state.pVertexAttributeDescriptions = vertex_input_attributes.data(); + + VkGraphicsPipelineCreateInfo pipeline_create_info = vkb::initializers::pipeline_create_info(skybox.pipeline.pipeline_layout, render_pass, 0u); + pipeline_create_info.pVertexInputState = &vertex_input_state; + pipeline_create_info.pInputAssemblyState = &input_assembly_state; + pipeline_create_info.pRasterizationState = &rasterization_state; + pipeline_create_info.pColorBlendState = &color_blend_state; + pipeline_create_info.pMultisampleState = &multisample_state; + pipeline_create_info.pViewportState = &viewport_state; + pipeline_create_info.pDepthStencilState = &depth_stencil_state; + pipeline_create_info.pDynamicState = &dynamic_state; + pipeline_create_info.stageCount = static_cast(shader_stages.size()); + pipeline_create_info.pStages = shader_stages.data(); + VK_CHECK(vkCreateGraphicsPipelines(get_device().get_handle(), pipeline_cache, 1u, &pipeline_create_info, nullptr, &skybox.pipeline.pipeline)); +} + void SubgroupsOperations::update_uniform_buffers() { CameraUbo ubo; @@ -762,6 +872,10 @@ void SubgroupsOperations::update_uniform_buffers() ocean_params.light_position = ui.light_pos; ocean_params.ocean_color = ui.ocean_color; + SkyboxUbo skybox_params; + skybox_params.mvp = camera.matrices.perspective * glm::mat4(glm::mat3(camera.matrices.view)); + + skybox_ubo->convert_and_update(skybox_params); ocean_params_ubo->convert_and_update(ocean_params); fft_time_ubo->convert_and_update(t); camera_ubo->convert_and_update(ubo); @@ -801,6 +915,13 @@ void SubgroupsOperations::build_command_buffers() VkRect2D scissor = vkb::initializers::rect2D(static_cast(width), static_cast(height), 0, 0); vkCmdSetScissor(cmd_buff, 0u, 1u, &scissor); + // skybox + { + vkCmdBindPipeline(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, skybox.pipeline.pipeline); + vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, skybox.pipeline.pipeline_layout, 0u, 1u, &skybox.descriptor_set, 0u, nullptr); + draw_model(skybox.skybox_shape, cmd_buff); + } + // draw ocean { vkCmdBindDescriptorSets(cmd_buff, VK_PIPELINE_BIND_POINT_GRAPHICS, ocean.pipelines._default.pipeline_layout, 0u, 1u, &ocean.descriptor_set, 0u, nullptr); diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 6fe20ec88..8e3bf5f09 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -49,6 +49,7 @@ class SubgroupsOperations : public ApiVulkanSample void create_descriptor_set_layout(); void create_descriptor_set(); void create_pipelines(); + void create_skybox(); void create_initial_tildas(); void create_tildas(); @@ -83,6 +84,11 @@ class SubgroupsOperations : public ApiVulkanSample alignas(16) glm::mat4 model; }; + struct SkyboxUbo + { + alignas(16) glm::mat4 mvp; + }; + struct OceanParamsUbo { alignas(16) glm::vec3 light_color; @@ -141,6 +147,7 @@ class SubgroupsOperations : public ApiVulkanSample } ui; uint32_t grid_size = {256U}; + std::unique_ptr skybox_ubo = {VK_NULL_HANDLE}; std::unique_ptr ocean_params_ubo = {VK_NULL_HANDLE}; std::unique_ptr camera_postion_ubo = {VK_NULL_HANDLE}; std::unique_ptr camera_ubo = {VK_NULL_HANDLE}; @@ -165,7 +172,7 @@ class SubgroupsOperations : public ApiVulkanSample vkFreeMemory(device, memory, nullptr); }; }; - + ImageAttachment butterfly_precomp; uint32_t log_2_N; @@ -261,6 +268,22 @@ class SubgroupsOperations : public ApiVulkanSample } pipelines; } ocean; + struct Skybox + { + void destroy(VkDevice device) + { + pipeline.destroy(device); + vkDestroyDescriptorSetLayout(device, descriptor_set_layout, nullptr); + } + + Pipeline pipeline; + VkDescriptorSetLayout descriptor_set_layout; + VkDescriptorSet descriptor_set; + + Texture skybox_texture; + std::unique_ptr skybox_shape; + } skybox; + VkPhysicalDeviceSubgroupProperties subgroups_properties; private: diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index a904d509e..054c8a781 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -52,9 +52,9 @@ void main() float dist_cam_v1 = distance(cam.position.xyz, outPos[1]); float dist_cam_v2 = distance(cam.position.xyz, outPos[2]); - gl_TessLevelOuter[0] = get_tesselllation_level(dist_cam_v1, dist_cam_v2); - gl_TessLevelOuter[1] = get_tesselllation_level(dist_cam_v0, dist_cam_v2); - gl_TessLevelOuter[2] = get_tesselllation_level(dist_cam_v0, dist_cam_v1); + gl_TessLevelOuter[0] = 1.0f; // get_tesselllation_level(dist_cam_v1, dist_cam_v2); + gl_TessLevelOuter[1] = 1.0f; // get_tesselllation_level(dist_cam_v0, dist_cam_v2); + gl_TessLevelOuter[2] = 1.0f; // get_tesselllation_level(dist_cam_v0, dist_cam_v1); gl_TessLevelInner[0] = gl_TessLevelOuter[2]; } diff --git a/shaders/subgroups_operations/skybox.frag b/shaders/subgroups_operations/skybox.frag new file mode 100644 index 000000000..e284eb34d --- /dev/null +++ b/shaders/subgroups_operations/skybox.frag @@ -0,0 +1,29 @@ +#version 450 +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +layout (location = 0) out vec4 outFragColor; + +layout (location = 0) in vec2 inUV; + +layout (set = 0, binding = 1) uniform sampler2D skybox_texture_map; + +void main(void) +{ + vec4 color = texture(skybox_texture_map, inUV); + outFragColor = vec4(color.rgb, 1.0); +} diff --git a/shaders/subgroups_operations/skybox.vert b/shaders/subgroups_operations/skybox.vert new file mode 100644 index 000000000..752807347 --- /dev/null +++ b/shaders/subgroups_operations/skybox.vert @@ -0,0 +1,34 @@ +#version 450 +/* Copyright (c) 2023, Mobica Limited + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +layout (location = 0) in vec3 inPos; +layout (location = 1) in vec2 inUV; + +layout (location = 0) out vec2 outUV; + +layout (set = 0, binding = 0) uniform SkyboxUbo +{ + mat4 mvp; +} ubo; + +void main(void) +{ + gl_Position = ubo.mvp * vec4(inPos, 1.0f); + outUV = inUV; + outUV.t = 1.0 - outUV.t; +} From 3a200dc1afb00937d953f3ede2fd756238704a75 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 18 Oct 2023 10:56:29 +0200 Subject: [PATCH 41/53] rebase --- .../subgroups_operations.cpp | 69 +++----- .../subgroups_operations.h | 2 +- .../butterfly_precomp.comp | 8 +- shaders/subgroups_operations/fft.comp | 150 ++++++++---------- shaders/subgroups_operations/fft_invert.comp | 4 +- shaders/subgroups_operations/fft_tilde_h.comp | 68 +------- .../subgroups_operations/fft_tilde_h0.comp | 2 +- shaders/subgroups_operations/ocean.frag | 2 +- shaders/subgroups_operations/ocean.tesc | 2 +- shaders/subgroups_operations/ocean.tese | 2 +- shaders/subgroups_operations/ocean.vert | 2 +- shaders/subgroups_operations/skybox.frag | 2 +- shaders/subgroups_operations/skybox.vert | 2 +- 13 files changed, 102 insertions(+), 213 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index e04f8f0ea..c10e057ba 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -105,9 +105,9 @@ SubgroupsOperations::~SubgroupsOperations() } } -bool SubgroupsOperations::prepare(vkb::Platform &platform) +bool SubgroupsOperations::prepare(const vkb::ApplicationOptions &options) { - if (!ApiVulkanSample::prepare(platform)) + if (!ApiVulkanSample::prepare(options)) { return false; } @@ -307,6 +307,11 @@ void SubgroupsOperations::build_compute_command_buffer() void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) { + if (gpu.get_features().samplerAnisotropy) + { + gpu.get_mutable_requested_features().samplerAnisotropy = VK_TRUE; + } + if (gpu.get_features().fillModeNonSolid) { gpu.get_mutable_requested_features().fillModeNonSolid = VK_TRUE; @@ -398,8 +403,6 @@ void SubgroupsOperations::create_tildas() vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 4u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 5u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u)}; - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u)}; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &tildas.descriptor_set_layout)); @@ -449,51 +452,25 @@ void SubgroupsOperations::load_assets() log_2_N = glm::log2(static_cast(grid_size)); - /* + input_random.clear(); - // generate fft inputs - h_tilde_0.clear(); - h_tilde_0_conj.clear(); for (uint32_t m = 0; m < grid_size; ++m) { - for (uint32_t n = 0; n < grid_size; ++n) - { - h_tilde_0.push_back(hTilde_0(n, m, 1)); - h_tilde_0_conj.push_back(std::conj(hTilde_0(-n, -m, 1))); - } - } - - // calculate weights - uint32_t pow2 = 1U; - for (uint32_t i = 0U; i < log_2_N; ++i) - { - for (uint32_t j = 0U; j < pow2; ++j) - { - weights.push_back(calculate_weight(j, pow2 * 2)); - } - pow2 *= 2; + for (uint32_t n = 0; n < grid_size; ++n) + { + glm::vec2 rnd1 = rndGaussian(); + glm::vec2 rnd2 = rndGaussian(); + input_random.push_back(glm::vec4{rnd1.x, rnd1.y, rnd2.x, rnd2.y}); + } } - auto fft_input_htilde0_size = static_cast(h_tilde_0.size() * sizeof(std::complex)); - fft_buffers.fft_input_htilde0 = std::make_unique(get_device(), - fft_input_htilde0_size, - VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, - VMA_MEMORY_USAGE_CPU_TO_GPU); - auto fft_input_htilde0_conj_size = static_cast(h_tilde_0_conj.size() * sizeof(std::complex)); - - fft_buffers.fft_input_htilde0_conj = std::make_unique(get_device(), - fft_input_htilde0_conj_size, - VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, - VMA_MEMORY_USAGE_CPU_TO_GPU); - - auto fft_input_weights_size = static_cast(weights.size() * sizeof(std::complex)); - fft_buffers.fft_input_weight = std::make_unique(get_device(), fft_input_weights_size, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + auto input_random_size = static_cast(input_random.size() * sizeof(glm::vec4)); + fft_buffers.fft_input_random = std::make_unique(get_device(), + input_random_size, + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + VMA_MEMORY_USAGE_CPU_TO_GPU); - fft_buffers.fft_input_htilde0->update(h_tilde_0.data(), fft_input_htilde0_size); - fft_buffers.fft_input_htilde0_conj->update(h_tilde_0_conj.data(), fft_input_htilde0_conj_size); - fft_buffers.fft_input_weight->update(weights.data(), fft_input_weights_size); - - */ + fft_buffers.fft_input_random->update(input_random.data(), input_random_size); } glm::vec2 SubgroupsOperations::rndGaussian() @@ -755,9 +732,9 @@ void SubgroupsOperations::create_skybox() pipeline_layout_info.pSetLayouts = &skybox.descriptor_set_layout; VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &pipeline_layout_info, nullptr, &skybox.pipeline.pipeline_layout)); - + VkDescriptorBufferInfo skybox_uniform_descriptor = create_descriptor(*skybox_ubo); - VkDescriptorImageInfo skybox_image_descriptor = create_descriptor(skybox.skybox_texture); + VkDescriptorImageInfo skybox_image_descriptor = create_descriptor(skybox.skybox_texture); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(skybox.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &skybox_uniform_descriptor), @@ -1372,4 +1349,4 @@ void SubgroupsOperations::Wind::recalc() float rad = angle * glm::pi() / 180.0f; vec.x = force * cos(rad); vec.y = force * sin(rad); -} +} \ No newline at end of file diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 8e3bf5f09..91b7b23bd 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -25,7 +25,7 @@ class SubgroupsOperations : public ApiVulkanSample SubgroupsOperations(); ~SubgroupsOperations(); - bool prepare(vkb::Platform &platform) override; + bool prepare(const vkb::ApplicationOptions &options) override; void request_gpu_features(vkb::PhysicalDevice &gpu) override; void build_command_buffers() override; void render(float delta_time) override; diff --git a/shaders/subgroups_operations/butterfly_precomp.comp b/shaders/subgroups_operations/butterfly_precomp.comp index 67202fbe7..c7ef7f4fc 100644 --- a/shaders/subgroups_operations/butterfly_precomp.comp +++ b/shaders/subgroups_operations/butterfly_precomp.comp @@ -50,12 +50,7 @@ void main() if ((mod(pos.y, pow(2, pos.x + 1))) < butterfly_span) butterfly_wing = 1; -<<<<<<< HEAD ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); -======= - ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - //pixel_pos.y = N - pixel_pos.y - 1; ->>>>>>> e005a65 (Add tilde_h_0 shader) vec4 result = vec4(0.0f); result.x = cos(2.f * PI_32F * k / float(N)); // Twiddle factor real part @@ -90,5 +85,4 @@ void main() } imageStore(u_butterfly_precomp, pixel_pos, result); -} - +} \ No newline at end of file diff --git a/shaders/subgroups_operations/fft.comp b/shaders/subgroups_operations/fft.comp index e48b10c0e..daf390078 100644 --- a/shaders/subgroups_operations/fft.comp +++ b/shaders/subgroups_operations/fft.comp @@ -25,113 +25,91 @@ layout (binding = 0, rgba32f) readonly uniform image2D u_butterfly_precomp; layout (binding = 1, rgba32f) uniform image2D u_pingpong0; layout (binding = 2, rgba32f) uniform image2D u_pingpong1; -layout (binding = 3) uniform FftPage -{ - int pages; -} fftUbo; +layout( push_constant ) uniform Push_Constants{ + uint i; +} step; struct Complex { - float real; - float imag; + float real; + float imag; }; Complex complex_add(Complex c1, Complex c2) { - Complex res; - res.real = c1.real + c2.real; - res.imag = c1.imag + c2.imag; - return res; + Complex res; + res.real = c1.real + c2.real; + res.imag = c1.imag + c2.imag; + return res; } Complex complex_multiply(Complex c1, Complex c2) { - Complex res; - res.real = c1.real * c2.real - c1.imag * c2.imag; - res.imag = c1.real * c2.imag + c1.imag * c2.real; - return res; + Complex res; + res.real = c1.real * c2.real - c1.imag * c2.imag; + res.imag = c1.real * c2.imag + c1.imag * c2.real; + return res; } void HorizontalButterflies(in ivec2 pixel_pos) { - for (int i = 0; i < fftUbo.pages; ++i) + vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(step.i, pixel_pos.x)); + if ((step.i % 2) == 0) { - subgroupBarrier(); - vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(i, pixel_pos.x)); - if ((i % 2) == 0) - { - vec2 a_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; - vec2 b_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - // Buttefly operation - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); - - //if(i == 6) - // debugPrintfEXT("[%i] %.5f, %.5f", i, result.real, result.imag); - - } - else - { - vec2 a_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; - vec2 b_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - // Buttefly operation - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); - - if(i == 5) - debugPrintfEXT("[%i] %.5f, %.5f", i, result.real, result.imag); - } + subgroupMemoryBarrierImage(); + vec2 a_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; + vec2 b_ = imageLoad(u_pingpong0, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + } + else + { + subgroupMemoryBarrierImage(); + vec2 a_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.z, pixel_pos.y)).rg; + vec2 b_ = imageLoad(u_pingpong1, ivec2(butterfly_precomp.w, pixel_pos.y)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); } } void VerticalButterfiles(in ivec2 pixel_pos) { - for (int i = 0; i < fftUbo.pages; ++i) + vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(step.i, pixel_pos.y)); + if ((step.i % 2) == 0) { - vec4 butterfly_precomp = imageLoad(u_butterfly_precomp, ivec2(i, pixel_pos.y)); - - if ((i % 2) == 0) - { - vec2 a_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; - vec2 b_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - // Buttefly operation - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - - imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); - } - else - { - vec2 a_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; - vec2 b_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; - - Complex a = Complex(a_.x, a_.y); - Complex b = Complex(b_.x, b_.y); - Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); - - // Buttefly operation - Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); - subgroupMemoryBarrierImage(); - - imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); - } + subgroupMemoryBarrierImage(); + vec2 a_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; + vec2 b_ = imageLoad(u_pingpong0, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + imageStore(u_pingpong1, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); + } + else + { + subgroupMemoryBarrierImage(); + vec2 a_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.z)).rg; + vec2 b_ = imageLoad(u_pingpong1, ivec2(pixel_pos.x, butterfly_precomp.w)).rg; + + Complex a = Complex(a_.x, a_.y); + Complex b = Complex(b_.x, b_.y); + Complex twiddle_factor = Complex(butterfly_precomp.x, butterfly_precomp.y); + + Complex result = complex_add(a, complex_multiply(twiddle_factor, b)); + imageStore(u_pingpong0, pixel_pos, vec4(result.real, result.imag, 0.f, 1.f)); } } @@ -140,4 +118,4 @@ void main() ivec2 uv = ivec2(gl_GlobalInvocationID.xy); if (direction == 0) HorizontalButterflies(uv); else if (direction == 1) VerticalButterfiles(uv); -} +} \ No newline at end of file diff --git a/shaders/subgroups_operations/fft_invert.comp b/shaders/subgroups_operations/fft_invert.comp index 0996b5521..e4673a343 100644 --- a/shaders/subgroups_operations/fft_invert.comp +++ b/shaders/subgroups_operations/fft_invert.comp @@ -45,7 +45,7 @@ void main() int index = int(mod(pixel_pos.x + pixel_pos.y, 2)); float perm = perms[index]; uint pingpong_index = fftUbo.pong_idx; - if (pingpong_index == 0) + if (pingpong_index == 0) { float h_y = imageLoad(u_pingpong0_y, pixel_pos).r; float h_x = imageLoad(u_pingpong0_x, pixel_pos).r; @@ -59,4 +59,4 @@ void main() float h_z = imageLoad(u_pingpong1_z, pixel_pos).r; imageStore(u_displacement, pixel_pos, vec4(perm * (h_x / float(N * N)), perm * (h_y / float(N * N)), perm * (h_z / float(N * N)), 1.0f)); } -} +} \ No newline at end of file diff --git a/shaders/subgroups_operations/fft_tilde_h.comp b/shaders/subgroups_operations/fft_tilde_h.comp index 6816c2b56..91ea64b8a 100644 --- a/shaders/subgroups_operations/fft_tilde_h.comp +++ b/shaders/subgroups_operations/fft_tilde_h.comp @@ -17,25 +17,11 @@ * limitations under the License. */ -#define PI 3.141592f +#define PI 3.14159265358979f #define GRAVITY 9.81f layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; -/* -layout (std140, binding = 0) readonly buffer TildeH0K -{ - vec2 data[]; -} -tilde_h0_k; - -layout (std140, binding = 1) readonly buffer TildeH0MinusK -{ - vec2 data[]; -} -tilde_h0_minus_k; -*/ - layout (binding = 0, rgba32f) readonly uniform image2D u_tilde_h0_k; layout (binding = 1, rgba32f) readonly uniform image2D u_tilde_h0_minus_k; @@ -88,82 +74,36 @@ Complex complex_conj(Complex c) void main() { - - int N = 256; - int L = 1000; + uint N = fftUbo.grid_size; + float L = fftUbo.len; vec2 pos = vec2(gl_GlobalInvocationID.xy) - (N / 2.0); - // Wavevector, Section 4.3, Eq 36 vec2 k = vec2((2.0 * PI * pos.x) / L, (2.0 * PI * pos.y) / L); float k_magnitude = length(k); if (k_magnitude < 0.00001f) k_magnitude = 0.00001f; - // Dispersion relation, Eq 31, Section 4.2 float w = sqrt(GRAVITY * k_magnitude); ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - // First term of Eq 43, Section 4.4 vec2 h0_k = imageLoad(u_tilde_h0_k, pixel_pos).xy; Complex amp = Complex(h0_k.x, h0_k.y); Complex exp_iwt = Complex(cos(w * t.time), sin(w * t.time)); - // Second term of Eq 43, Section 4.4 vec2 h0_minus_k = imageLoad(u_tilde_h0_minus_k, pixel_pos).xy; Complex amp_conj = complex_conj(Complex(h0_minus_k.x, h0_minus_k.y)); Complex exp_minus_iwt = Complex(cos(w * t.time), -sin(w * t.time)); - // Section 4.4, Eq 43 Complex h_k_t_dy = complex_add(complex_multiply(amp, exp_iwt), complex_multiply(amp_conj, exp_minus_iwt)); imageStore(tilde_h_kt_dy, pixel_pos, vec4(h_k_t_dy.real, h_k_t_dy.imag, 0.0, 1.0)); - // Section 4.6, Eq 44 Complex dx = Complex(0.0, -k.x / k_magnitude); Complex h_k_t_dx = complex_multiply(dx, h_k_t_dy); imageStore(tilde_h_kt_dx, pixel_pos, vec4(h_k_t_dx.real, h_k_t_dx.imag, 0.0, 1.0)); - // Section 4.6, Eq 44 Complex dz = Complex(0.0, -k.y / k_magnitude); Complex h_k_t_dz = complex_multiply(dz, h_k_t_dy); imageStore(tilde_h_kt_dz, pixel_pos, vec4(h_k_t_dz.real, h_k_t_dz.imag, 0.0, 1.0)); - -/* - float L = fftUbo.len; - - vec2 uv = vec2(gl_GlobalInvocationID.xy) - (fftUbo.grid_size / 2.0f); - - vec2 k = vec2( (2.0f * PI * uv.x) / L, (2.0f * PI * uv.y) / L); - - float k_mag = length(k); - if (k_mag < 0.00001f) k_mag = 0.00001f; - - float w = sqrt(GRAVITY * k_mag); - - ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - //uvec2 uv_idx = uvec2(gl_GlobalInvocationID.xy * 0.5); - - //uint idx = uv_idx.x + uv_idx.y * fftUbo.grid_size; - vec2 h0_k = imageLoad(u_tilde_h0_k, pixel_pos).xy; - - Complex amp = Complex(h0_k.x, h0_k.y); - Complex exp_iwt = Complex(cos(w * t.time), sin(w * t.time)); - - vec2 h0_minus_k = imageLoad(u_tilde_h0_minus_k, pixel_pos).xy; - - Complex amp_conj = complex_conj(Complex(h0_minus_k.x, h0_minus_k.y)); - Complex exp_minus_iwt = Complex(cos(w * t.time), -sin(w * t.time)); - - Complex h_k_t_dy = complex_add( complex_multiply(amp, exp_iwt), complex_multiply(amp_conj, exp_minus_iwt)); - imageStore(tilde_h_kt_dy, pixel_pos, vec4(h_k_t_dy.real, h_k_t_dy.imag, 0.0f, 1.0f)); - - Complex dx = Complex(0.0f, -k.x / k_mag); - Complex h_k_t_dx = complex_multiply(dx, h_k_t_dy); - imageStore(tilde_h_kt_dx, pixel_pos, vec4(h_k_t_dx.real, h_k_t_dx.imag, 0.0f, 1.0f)); - - Complex dz = Complex(0.0f, -k.y / k_mag); - Complex h_k_t_dz = complex_multiply(dz, h_k_t_dy); - imageStore(tilde_h_kt_dz, pixel_pos, vec4(h_k_t_dz.real, h_k_t_dz.imag, 0.0f, 1.0f)); -*/ -} +} \ No newline at end of file diff --git a/shaders/subgroups_operations/fft_tilde_h0.comp b/shaders/subgroups_operations/fft_tilde_h0.comp index b747e3933..1f52f367e 100644 --- a/shaders/subgroups_operations/fft_tilde_h0.comp +++ b/shaders/subgroups_operations/fft_tilde_h0.comp @@ -76,4 +76,4 @@ void main() imageStore(tilde_h0_k, pixel_pos, vec4(rnd.xy * h0_k, 0.0, 1.0)); imageStore(tilde_h0_minus_k, pixel_pos, vec4(rnd.zw * h0_minus_k, 0.0, 1.0)); -} +} \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index 3c68f8ce4..4d45a8984 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -60,4 +60,4 @@ void main() result = pow(result, vec3(1.0f / 2.2f)); outFragColor = vec4(result, 1.0f); -} +} \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 054c8a781..723247496 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -57,4 +57,4 @@ void main() gl_TessLevelOuter[2] = 1.0f; // get_tesselllation_level(dist_cam_v0, dist_cam_v1); gl_TessLevelInner[0] = gl_TessLevelOuter[2]; -} +} \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 205b03ae3..68056b480 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -80,4 +80,4 @@ void main() outNormal = height_texel.xyz; outPos = world_pos; gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); -} +} \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.vert b/shaders/subgroups_operations/ocean.vert index dbbc007f0..d404982c4 100644 --- a/shaders/subgroups_operations/ocean.vert +++ b/shaders/subgroups_operations/ocean.vert @@ -26,4 +26,4 @@ void main() { outPos = inPos; outUv = inUv; -} +} \ No newline at end of file diff --git a/shaders/subgroups_operations/skybox.frag b/shaders/subgroups_operations/skybox.frag index e284eb34d..0dc3e6ebb 100644 --- a/shaders/subgroups_operations/skybox.frag +++ b/shaders/subgroups_operations/skybox.frag @@ -26,4 +26,4 @@ void main(void) { vec4 color = texture(skybox_texture_map, inUV); outFragColor = vec4(color.rgb, 1.0); -} +} \ No newline at end of file diff --git a/shaders/subgroups_operations/skybox.vert b/shaders/subgroups_operations/skybox.vert index 752807347..4f83571cc 100644 --- a/shaders/subgroups_operations/skybox.vert +++ b/shaders/subgroups_operations/skybox.vert @@ -31,4 +31,4 @@ void main(void) gl_Position = ubo.mvp * vec4(inPos, 1.0f); outUV = inUV; outUV.t = 1.0 - outUV.t; -} +} \ No newline at end of file From ebb8d25e60d3fcf0d078e3688b237f48f2b42144 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 18 Oct 2023 12:00:35 +0200 Subject: [PATCH 42/53] update ui --- .../subgroups_operations.cpp | 28 +++++++++++++------ .../subgroups_operations.h | 1 + shaders/subgroups_operations/ocean.tesc | 6 ++-- shaders/subgroups_operations/ocean.tese | 2 +- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index c10e057ba..19af8e9dc 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -969,9 +969,15 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) drawer.slider_float("Position y", &ui.light_pos.y, -1000.0f, 1000.0f); drawer.slider_float("Position z", &ui.light_pos.z, -1000.0f, 1000.0f); - drawer.slider_float("Color Red", &ui.light_color.r, 0.0f, 1.0f); - drawer.slider_float("Color Green", &ui.light_color.g, 0.0f, 1.0f); - drawer.slider_float("Color Blue", &ui.light_color.b, 0.0f, 1.0f); + std::array colors = {ui.light_color.r, ui.light_color.g, ui.light_color.b}; + drawer.color_op("Light color", colors, 0, + ImGuiColorEditFlags_NoSidePreview | + ImGuiColorEditFlags_NoSmallPreview | + ImGuiColorEditFlags_Float | + ImGuiColorEditFlags_RGB); + ui.light_color.r = colors[0]; + ui.light_color.g = colors[1]; + ui.light_color.b = colors[2]; } } @@ -979,12 +985,6 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) { drawer.input_float("Amplitude", &ui.amplitude, 1.f, 3u); drawer.input_float("Length", &ui.length, 10.f, 1u); - if (drawer.header("Color")) - { - drawer.slider_float("Red", &ui.ocean_color.r, 0.0f, 1.0f); - drawer.slider_float("Green", &ui.ocean_color.g, 0.0f, 1.0f); - drawer.slider_float("Blue", &ui.ocean_color.b, 0.0f, 1.0f); - } if (drawer.header("Wind")) { @@ -993,6 +993,16 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) ui.wind.recalc(); } + + std::array colors = {ui.ocean_color.r, ui.ocean_color.g, ui.ocean_color.b}; + drawer.color_op("Ocean color", colors, 0, + ImGuiColorEditFlags_NoSidePreview | + ImGuiColorEditFlags_NoSmallPreview | + ImGuiColorEditFlags_Float | + ImGuiColorEditFlags_RGB); + ui.ocean_color.r = colors[0]; + ui.ocean_color.g = colors[1]; + ui.ocean_color.b = colors[2]; } } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 91b7b23bd..28ba03ae5 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -274,6 +274,7 @@ class SubgroupsOperations : public ApiVulkanSample { pipeline.destroy(device); vkDestroyDescriptorSetLayout(device, descriptor_set_layout, nullptr); + skybox_shape.reset(); } Pipeline pipeline; diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 723247496..45c1beaa9 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -45,9 +45,6 @@ float get_tesselllation_level(float dist0, float dist1) void main() { - outPos[gl_InvocationID] = inPostion[gl_InvocationID]; - outUv[gl_InvocationID] = inUv[gl_InvocationID]; - float dist_cam_v0 = distance(cam.position.xyz, outPos[0]); float dist_cam_v1 = distance(cam.position.xyz, outPos[1]); float dist_cam_v2 = distance(cam.position.xyz, outPos[2]); @@ -57,4 +54,7 @@ void main() gl_TessLevelOuter[2] = 1.0f; // get_tesselllation_level(dist_cam_v0, dist_cam_v1); gl_TessLevelInner[0] = gl_TessLevelOuter[2]; + + outPos[gl_InvocationID] = inPostion[gl_InvocationID]; + outUv[gl_InvocationID] = inUv[gl_InvocationID]; } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 68056b480..8747dfb51 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -16,7 +16,7 @@ * limitations under the License. */ -layout(triangles) in; +layout(triangles, equal_spacing, ccw) in; layout(location = 0) in vec3 inPostion[]; layout(location = 1) in vec2 inUv[]; From 4cbe4dfef2e9cb5bf4f480d1c7582ef93a69b997 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 18 Oct 2023 12:56:53 +0200 Subject: [PATCH 43/53] fix normal map --- shaders/subgroups_operations/fft_normal_map.comp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/shaders/subgroups_operations/fft_normal_map.comp b/shaders/subgroups_operations/fft_normal_map.comp index 8af783cb2..162a03dfa 100644 --- a/shaders/subgroups_operations/fft_normal_map.comp +++ b/shaders/subgroups_operations/fft_normal_map.comp @@ -32,18 +32,16 @@ void main() { uint N = fftUbo.grid_size; ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - - vec2 tex_coord = gl_GlobalInvocationID.xy / float(N); float tex_dim = 1.0f / N; - float left_y = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(-tex_dim, 0.0f))).g; - float right_y = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(tex_dim, 0.0f))).g; - float bottom_y = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(0.0f, tex_dim))).g; - float top_y = imageLoad(fft_displacement_map, ivec2(tex_coord + vec2(0.0f, tex_dim))).g; + float left_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-tex_dim, 0.0f))).g; + float right_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, 0.0f))).g; + float bottom_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, tex_dim))).g; + float top_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, tex_dim))).g; vec3 result = vec3(0.0f); - result.z = bottom_y - top_y; - result.x = left_y - right_y; + result.z = (bottom_y - top_y) / float(N); + result.x = (left_y - right_y) / float(N); result.y = 1.0f / float(N); imageStore(fft_normal_map, pixel_pos, vec4(normalize(result), 1.0f)); From 60d4809826e733935e646e7e181f5844970fa9b7 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 18 Oct 2023 13:23:43 +0200 Subject: [PATCH 44/53] add a new ui elements --- .../subgroups_operations/subgroups_operations.cpp | 6 ++++-- .../subgroups_operations/subgroups_operations.h | 14 ++++++++------ shaders/subgroups_operations/ocean.tesc | 6 +++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 19af8e9dc..b30601b07 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -838,8 +838,8 @@ void SubgroupsOperations::update_uniform_buffers() invertFft.page_idx = log_2_N % 2; TessellationParamsUbo tess_params; - tess_params.displacement_scale = 0.5f; - tess_params.choppines = 0.75f; + tess_params.displacement_scale = ui.displacement_scale; + tess_params.choppines = ui.choppines; TimeUbo t; t.time = float(timer.elapsed()); @@ -985,6 +985,8 @@ void SubgroupsOperations::on_update_ui_overlay(vkb::Drawer &drawer) { drawer.input_float("Amplitude", &ui.amplitude, 1.f, 3u); drawer.input_float("Length", &ui.length, 10.f, 1u); + drawer.slider_float("Choppines", &ui.choppines, 0.0f, 1.0f); + drawer.slider_float("Displacement scale", &ui.displacement_scale, 0.0f, 1.0f); if (drawer.header("Wind")) { diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 28ba03ae5..618431124 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -136,9 +136,11 @@ class SubgroupsOperations : public ApiVulkanSample struct GuiConfig { - bool wireframe = {true}; - float amplitude = {32.0f}; - float length = {1900.0f}; + bool wireframe = {true}; + float choppines = {0.75f}; + float displacement_scale = {0.5f}; + float amplitude = {32.0f}; + float length = {1900.0f}; Wind wind; glm::vec3 light_pos = {100.0f, 15.0f, -1.0f}; @@ -276,13 +278,13 @@ class SubgroupsOperations : public ApiVulkanSample vkDestroyDescriptorSetLayout(device, descriptor_set_layout, nullptr); skybox_shape.reset(); } - + Pipeline pipeline; VkDescriptorSetLayout descriptor_set_layout; VkDescriptorSet descriptor_set; - Texture skybox_texture; - std::unique_ptr skybox_shape; + Texture skybox_texture; + std::unique_ptr skybox_shape; } skybox; VkPhysicalDeviceSubgroupProperties subgroups_properties; diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 45c1beaa9..1159e9de2 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -49,9 +49,9 @@ void main() float dist_cam_v1 = distance(cam.position.xyz, outPos[1]); float dist_cam_v2 = distance(cam.position.xyz, outPos[2]); - gl_TessLevelOuter[0] = 1.0f; // get_tesselllation_level(dist_cam_v1, dist_cam_v2); - gl_TessLevelOuter[1] = 1.0f; // get_tesselllation_level(dist_cam_v0, dist_cam_v2); - gl_TessLevelOuter[2] = 1.0f; // get_tesselllation_level(dist_cam_v0, dist_cam_v1); + gl_TessLevelOuter[0] = get_tesselllation_level(dist_cam_v1, dist_cam_v2); + gl_TessLevelOuter[1] = get_tesselllation_level(dist_cam_v0, dist_cam_v2); + gl_TessLevelOuter[2] = get_tesselllation_level(dist_cam_v0, dist_cam_v1); gl_TessLevelInner[0] = gl_TessLevelOuter[2]; From a6b4864b401862b1f28beb1ff76e8ee6e3215a86 Mon Sep 17 00:00:00 2001 From: Krzysztof-Dmitruk-Mobica Date: Fri, 20 Oct 2023 14:32:07 +0200 Subject: [PATCH 45/53] Add (broken) reflection --- .../subgroups_operations.cpp | 31 +++++--- .../subgroups_operations.h | 2 +- shaders/subgroups_operations/ocean.frag | 72 +++++++++++++------ shaders/subgroups_operations/ocean.tesc | 8 ++- shaders/subgroups_operations/ocean.tese | 53 ++++++++------ shaders/subgroups_operations/ocean.vert | 6 +- shaders/subgroups_operations/skybox.frag | 11 +-- shaders/subgroups_operations/skybox.vert | 11 ++- 8 files changed, 126 insertions(+), 68 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index b30601b07..3d1aed010 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -50,10 +50,11 @@ SubgroupsOperations::SubgroupsOperations() // Targeting SPIR-V version vkb::GLSLCompiler::set_target_environment(glslang::EShTargetSpv, glslang::EShTargetSpv_1_4); - title = "Subgroups operations"; - camera.type = vkb::CameraType::FirstPerson; + title = "Subgroups operations"; + camera.type = vkb::CameraType::FirstPerson; + camera.translation_speed = 15.0f; - camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); + camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.01f, 256.0f); camera.set_position({0.0f, 5.0f, 0.0f}); } @@ -444,8 +445,11 @@ void SubgroupsOperations::create_tildas() void SubgroupsOperations::load_assets() { - skybox.skybox_shape = load_model("scenes/geosphere.gltf"); - skybox.skybox_texture = load_texture("textures/skysphere_rgba.ktx", vkb::sg::Image::Color); + // skybox.skybox_shape = load_model("scenes/geosphere.gltf"); + // skybox.skybox_texture = load_texture("textures/skysphere_rgba.ktx", vkb::sg::Image::Color); + + skybox.skybox_shape = load_model("scenes/cube.gltf"); + skybox.skybox_texture = load_texture_cubemap("textures/uffizi_rgba16f_cube.ktx", vkb::sg::Image::Color); generate_plane(); ui.wind.recalc(); @@ -601,7 +605,13 @@ void SubgroupsOperations::create_descriptor_set_layout() 4u), vkb::initializers::descriptor_set_layout_binding( VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, - 5u)}; + 5u), + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + 6u), + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, VK_SHADER_STAGE_FRAGMENT_BIT, + 7u)}; VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); @@ -621,6 +631,8 @@ void SubgroupsOperations::create_descriptor_set() VkDescriptorBufferInfo camera_pos_buffer_descriptor = create_descriptor(*camera_postion_ubo); VkDescriptorImageInfo normal_map_descirptor = create_ia_descriptor(*fft_buffers.fft_normal_map); VkDescriptorBufferInfo ocean_params_buffer_descriptor = create_descriptor(*ocean_params_ubo); + VkDescriptorBufferInfo skybox_uniform_descriptor = create_descriptor(*skybox_ubo); + VkDescriptorImageInfo skybox_image_descriptor = create_descriptor(skybox.skybox_texture); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor), @@ -628,7 +640,10 @@ void SubgroupsOperations::create_descriptor_set() vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &tessellation_params_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &camera_pos_buffer_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &normal_map_descirptor), - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &ocean_params_buffer_descriptor)}; + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &ocean_params_buffer_descriptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 6u, &skybox_uniform_descriptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 7u, &skybox_image_descriptor), + }; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } @@ -1361,4 +1376,4 @@ void SubgroupsOperations::Wind::recalc() float rad = angle * glm::pi() / 180.0f; vec.x = force * cos(rad); vec.y = force * sin(rad); -} \ No newline at end of file +} diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 618431124..856789247 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -131,7 +131,7 @@ class SubgroupsOperations : public ApiVulkanSample void recalc(); glm::vec2 vec; float angle = 180; - float force = 25; + float force = 3; }; struct GuiConfig diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index 4d45a8984..8eb68f2a6 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -16,8 +16,9 @@ * limitations under the License. */ -layout (location = 0) in vec3 in_pos; -layout (location = 1) in vec3 in_normal; +layout (location=1) in vec3 normalVector; +layout (location=2) in vec3 eyePosition; +layout (location=3) in vec3 lightVector; layout (location = 0) out vec4 outFragColor; @@ -33,31 +34,58 @@ layout (binding = 5) uniform OceanParamsUbo vec3 ocean_color; } ocean_ubo; +layout (binding = 7) uniform samplerCube skybox_texture_map; + void main() { - vec3 normal = in_normal; - vec3 light_pos = ocean_ubo.light_position; - vec3 light_color = ocean_ubo.light_color; - vec3 ocean_color = ocean_ubo.ocean_color; - float ambient_strength = 0.91f; - vec3 ambient = ambient_strength * light_color; + float material_shiness = 27.89f; + + vec3 n = normalize(normalVector); + vec3 l = normalize(lightVector); + + float i_diffuse = clamp(dot(n, l), 0, 1); + + vec3 e = normalize(-eyePosition); + vec3 r = reflect(-l, n); + + float i_specular = pow(clamp(dot(e, r), 0, 1), material_shiness); + float lightPower = 1; + + vec3 specular = vec3(0.3f, 0.3f, 0.3f); + + vec3 R = reflect(eyePosition.xyz, normalVector); + vec3 reflection = texture(skybox_texture_map, R).rgb; + + vec3 color = ocean_ubo.light_color * ocean_ubo.ocean_color * i_diffuse + specular * i_specular; + + outFragColor=vec4(mix(color, reflection, 0.1f), 1.0f); + +/* + vec3 normal = in_normal; + vec3 light_pos = ocean_ubo.light_position; + vec3 light_color = ocean_ubo.light_color; + vec3 ocean_color = ocean_ubo.ocean_color; + float ambient_strength = 0.91f; + + vec3 ambient = ambient_strength * light_color; - vec3 light_dir = normalize(light_pos - in_pos); - float diff = max(dot(normal, light_dir), 0.0f); - vec3 diffuse = diff * light_color; + vec3 light_dir = normalize(light_pos - in_pos); + float diff = max(dot(normal, light_dir), 0.0f); + vec3 diffuse = diff * light_color; - float specular_strength = 0.5f; - vec3 view_dir = normalize(vec3(cam.position.xyz - in_pos)); - vec3 ref_dir = reflect(-light_dir, normal); - float spec = pow(max(dot(view_dir, ref_dir), 0.0f), 32); - vec3 specular = specular_strength * spec * light_color; + float specular_strength = 0.5f; + vec3 view_dir = normalize(vec3(cam.position.xyz - in_pos)); + vec3 ref_dir = reflect(-light_dir, normal); + float spec = pow(max(dot(view_dir, ref_dir), 0.0f), 32); + vec3 specular = specular_strength * spec * light_color; - vec3 result = (ambient + diffuse + specular) * ocean_color; - result = result / (result + vec3(1.0f)); + vec3 result = (ambient + diffuse + specular) * ocean_color; + result = result / (result + vec3(1.0f)); - // gamma correction - result = pow(result, vec3(1.0f / 2.2f)); + // gamma correction + result = pow(result, vec3(1.0f / 2.2f)); - outFragColor = vec4(result, 1.0f); -} \ No newline at end of file + outFragColor = vec4(result, 1.0f); +*/ +} diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 1159e9de2..3f560f840 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -45,6 +45,9 @@ float get_tesselllation_level(float dist0, float dist1) void main() { + outPos[gl_InvocationID] = inPostion[gl_InvocationID]; + outUv[gl_InvocationID] = inUv[gl_InvocationID]; + float dist_cam_v0 = distance(cam.position.xyz, outPos[0]); float dist_cam_v1 = distance(cam.position.xyz, outPos[1]); float dist_cam_v2 = distance(cam.position.xyz, outPos[2]); @@ -55,6 +58,5 @@ void main() gl_TessLevelInner[0] = gl_TessLevelOuter[2]; - outPos[gl_InvocationID] = inPostion[gl_InvocationID]; - outUv[gl_InvocationID] = inUv[gl_InvocationID]; -} \ No newline at end of file + +} diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 8747dfb51..f4361193d 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -21,8 +21,9 @@ layout(triangles, equal_spacing, ccw) in; layout(location = 0) in vec3 inPostion[]; layout(location = 1) in vec2 inUv[]; -layout (location = 0) out vec3 outPos; -layout (location = 1) out vec3 outNormal; +layout (location=1) out vec3 outNormal; +layout (location=2) out vec3 outEyePosition; +layout (location=3) out vec3 outLightVec; layout (binding = 0) uniform Ubo { @@ -41,6 +42,11 @@ layout (binding = 2) uniform TessellationParams layout (binding = 4, rgba32f) uniform image2D fft_height_map; +layout (binding = 6) uniform SkyboxUbo +{ + mat4 view; +} skybox_ubo; + vec2 interpolate_2d(vec2 v0, vec2 v1, vec2 v2) { return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2; @@ -58,26 +64,33 @@ vec4 interpolate_4d(vec4 v0, vec4 v1, vec4 v2) void main() { - vec3 world_pos = interpolate_3d(inPostion[0], inPostion[1], inPostion[2]); + vec3 world_pos = interpolate_3d(inPostion[0], inPostion[1], inPostion[2]); + vec2 uv = interpolate_2d(inUv[0], inUv[1], inUv[2]); + + vec4 fft_texel_at_vertex[4]; + fft_texel_at_vertex[0] = imageLoad(fft_displacement_map, ivec2(inUv[0])); + fft_texel_at_vertex[1] = imageLoad(fft_displacement_map, ivec2(inUv[1])); + fft_texel_at_vertex[2] = imageLoad(fft_displacement_map, ivec2(inUv[2])); - vec4 fft_texel_at_vertex[4]; - fft_texel_at_vertex[0] = imageLoad(fft_displacement_map, ivec2(inUv[0])); - fft_texel_at_vertex[1] = imageLoad(fft_displacement_map, ivec2(inUv[1])); - fft_texel_at_vertex[2] = imageLoad(fft_displacement_map, ivec2(inUv[2])); + vec4 height_texel_at_vertex[4]; + height_texel_at_vertex[0] = imageLoad(fft_height_map, ivec2(inUv[0])); + height_texel_at_vertex[1] = imageLoad(fft_height_map, ivec2(inUv[1])); + height_texel_at_vertex[2] = imageLoad(fft_height_map, ivec2(inUv[2])); - vec4 height_texel_at_vertex[4]; - height_texel_at_vertex[0] = imageLoad(fft_height_map, ivec2(inUv[0])); - height_texel_at_vertex[1] = imageLoad(fft_height_map, ivec2(inUv[1])); - height_texel_at_vertex[2] = imageLoad(fft_height_map, ivec2(inUv[2])); + vec4 fft_texel = interpolate_4d(fft_texel_at_vertex[0], fft_texel_at_vertex[1], fft_texel_at_vertex[2]); + vec4 height_texel = interpolate_4d(height_texel_at_vertex[0], height_texel_at_vertex[1], height_texel_at_vertex[2]); - vec4 fft_texel = interpolate_4d(fft_texel_at_vertex[0], fft_texel_at_vertex[1], fft_texel_at_vertex[2]); - vec4 height_texel = interpolate_4d(height_texel_at_vertex[0], height_texel_at_vertex[1], height_texel_at_vertex[2]); + world_pos.y += fft_texel.y * tessParams.displacement_scale; + world_pos.x -= fft_texel.x * tessParams.choppines; + world_pos.z -= fft_texel.z * tessParams.choppines; - world_pos.y += fft_texel.y * tessParams.displacement_scale; - world_pos.x -= fft_texel.x * tessParams.choppines; - world_pos.z -= fft_texel.z * tessParams.choppines; + outNormal = height_texel.xyz; - outNormal = height_texel.xyz; - outPos = world_pos; - gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); -} \ No newline at end of file + gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); + + outEyePosition = vec4(ubo.view * ubo.model * vec4(world_pos, 1.0f)).xyz; + outNormal = vec4(ubo.view * ubo.model * vec4(height_texel.xyz, 0.0f)).xyz; + vec3 lightPos = vec3(0.0f, -5.0f, 5.0f); + outLightVec = vec4(ubo.view * vec4(lightPos, 1.0f)).xyz - outEyePosition; + +} diff --git a/shaders/subgroups_operations/ocean.vert b/shaders/subgroups_operations/ocean.vert index d404982c4..5eac8a01f 100644 --- a/shaders/subgroups_operations/ocean.vert +++ b/shaders/subgroups_operations/ocean.vert @@ -24,6 +24,6 @@ layout (location = 1) out vec2 outUv; void main() { - outPos = inPos; - outUv = inUv; -} \ No newline at end of file + outPos = inPos; + outUv = inUv; +} diff --git a/shaders/subgroups_operations/skybox.frag b/shaders/subgroups_operations/skybox.frag index 0dc3e6ebb..ce82b35b3 100644 --- a/shaders/subgroups_operations/skybox.frag +++ b/shaders/subgroups_operations/skybox.frag @@ -18,12 +18,13 @@ layout (location = 0) out vec4 outFragColor; -layout (location = 0) in vec2 inUV; +layout (location = 0) in vec3 inUV; -layout (set = 0, binding = 1) uniform sampler2D skybox_texture_map; +layout (set = 0, binding = 1) uniform samplerCube skybox_texture_map; void main(void) { - vec4 color = texture(skybox_texture_map, inUV); - outFragColor = vec4(color.rgb, 1.0); -} \ No newline at end of file + vec3 normal = normalize(inUV); + vec4 color = texture(skybox_texture_map, inUV); + outFragColor = vec4(color.rgb, 1.0); +} diff --git a/shaders/subgroups_operations/skybox.vert b/shaders/subgroups_operations/skybox.vert index 4f83571cc..a9255df18 100644 --- a/shaders/subgroups_operations/skybox.vert +++ b/shaders/subgroups_operations/skybox.vert @@ -17,9 +17,9 @@ */ layout (location = 0) in vec3 inPos; -layout (location = 1) in vec2 inUV; +layout (location = 1) in vec3 inUV; -layout (location = 0) out vec2 outUV; +layout (location = 0) out vec3 outUV; layout (set = 0, binding = 0) uniform SkyboxUbo { @@ -28,7 +28,6 @@ layout (set = 0, binding = 0) uniform SkyboxUbo void main(void) { - gl_Position = ubo.mvp * vec4(inPos, 1.0f); - outUV = inUV; - outUV.t = 1.0 - outUV.t; -} \ No newline at end of file + gl_Position = ubo.mvp * vec4(inPos, 1.0f); + outUV = inPos; +} From a0bc2731312e90055b674a76876eee7bcbf01ffc Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Tue, 24 Oct 2023 08:41:04 +0200 Subject: [PATCH 46/53] Fix skybox cliping issue --- .../subgroups_operations.cpp | 4 +-- .../subgroups_operations.h | 1 + shaders/subgroups_operations/ocean.tesc | 25 +++++++++---------- shaders/subgroups_operations/ocean.tese | 11 ++++---- shaders/subgroups_operations/ocean.vert | 5 ++-- shaders/subgroups_operations/skybox.vert | 4 +-- 6 files changed, 24 insertions(+), 26 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index b30601b07..f3ad727f0 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -55,6 +55,7 @@ SubgroupsOperations::SubgroupsOperations() camera.set_perspective(60.0f, static_cast(width) / static_cast(height), 0.1f, 256.0f); camera.set_position({0.0f, 5.0f, 0.0f}); + camera.translation_speed = {15.0f}; } SubgroupsOperations::~SubgroupsOperations() @@ -765,7 +766,7 @@ void SubgroupsOperations::create_skybox() VkPipelineDepthStencilStateCreateInfo depth_stencil_state = vkb::initializers::pipeline_depth_stencil_state_create_info( VK_TRUE, - VK_TRUE, + VK_FALSE, VK_COMPARE_OP_GREATER); VkPipelineViewportStateCreateInfo viewport_state = @@ -820,7 +821,6 @@ void SubgroupsOperations::update_uniform_buffers() { CameraUbo ubo; ubo.model = glm::mat4(1.0f); - ubo.model = glm::translate(ubo.model, glm::vec3(0.0f)); ubo.view = camera.matrices.view; ubo.projection = camera.matrices.perspective; diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 618431124..2b18c39b4 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -275,6 +275,7 @@ class SubgroupsOperations : public ApiVulkanSample void destroy(VkDevice device) { pipeline.destroy(device); + vkDestroySampler(device, skybox_texture.sampler, nullptr); vkDestroyDescriptorSetLayout(device, descriptor_set_layout, nullptr); skybox_shape.reset(); } diff --git a/shaders/subgroups_operations/ocean.tesc b/shaders/subgroups_operations/ocean.tesc index 1159e9de2..ce9d97111 100644 --- a/shaders/subgroups_operations/ocean.tesc +++ b/shaders/subgroups_operations/ocean.tesc @@ -18,11 +18,9 @@ layout (vertices = 3) out; -layout(location = 0) in vec3 inPostion[]; -layout(location = 1) in vec2 inUv[]; +layout(location = 0) in vec2 inUv[]; -layout(location = 0) out vec3 outPos[]; -layout(location = 1) out vec2 outUv[]; +layout(location = 0) out vec2 outUv[]; layout (binding = 3) uniform CameraPos { @@ -34,20 +32,21 @@ float get_tesselllation_level(float dist0, float dist1) float avg_dist = (dist0 + dist1) / 2.0f; if (avg_dist <= 10.0f) - return 2.0f; - else if (avg_dist <= 20.0f) return 3.0f; - else if (avg_dist <= 30.0f) - return 1.0f; - + if (avg_dist <= 40.0f) + return 2.0f; return 1.0f; } void main() { - float dist_cam_v0 = distance(cam.position.xyz, outPos[0]); - float dist_cam_v1 = distance(cam.position.xyz, outPos[1]); - float dist_cam_v2 = distance(cam.position.xyz, outPos[2]); + vec3 p0 = vec3(-gl_in[0].gl_Position.x, gl_in[0].gl_Position.y, -gl_in[0].gl_Position.z); + vec3 p1 = vec3(-gl_in[1].gl_Position.x, gl_in[1].gl_Position.y, -gl_in[1].gl_Position.z); + vec3 p2 = vec3(-gl_in[2].gl_Position.x, gl_in[2].gl_Position.y, -gl_in[2].gl_Position.z); + + float dist_cam_v0 = distance(cam.position.xyz, p0); + float dist_cam_v1 = distance(cam.position.xyz, p1); + float dist_cam_v2 = distance(cam.position.xyz, p2); gl_TessLevelOuter[0] = get_tesselllation_level(dist_cam_v1, dist_cam_v2); gl_TessLevelOuter[1] = get_tesselllation_level(dist_cam_v0, dist_cam_v2); @@ -55,6 +54,6 @@ void main() gl_TessLevelInner[0] = gl_TessLevelOuter[2]; - outPos[gl_InvocationID] = inPostion[gl_InvocationID]; + gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position; outUv[gl_InvocationID] = inUv[gl_InvocationID]; } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 8747dfb51..f6f93ccfa 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -18,8 +18,7 @@ layout(triangles, equal_spacing, ccw) in; -layout(location = 0) in vec3 inPostion[]; -layout(location = 1) in vec2 inUv[]; +layout(location = 0) in vec2 inUv[]; layout (location = 0) out vec3 outPos; layout (location = 1) out vec3 outNormal; @@ -31,7 +30,7 @@ layout (binding = 0) uniform Ubo mat4 model; } ubo; -layout (binding = 1, rgba32f) uniform image2D fft_displacement_map; +layout (binding = 1, rgba32f) uniform image2D fft_displacement_map; layout (binding = 2) uniform TessellationParams { @@ -58,14 +57,14 @@ vec4 interpolate_4d(vec4 v0, vec4 v1, vec4 v2) void main() { - vec3 world_pos = interpolate_3d(inPostion[0], inPostion[1], inPostion[2]); + vec3 world_pos = interpolate_3d(gl_in[0].gl_Position.xyz, gl_in[1].gl_Position.xyz, gl_in[2].gl_Position.xyz); - vec4 fft_texel_at_vertex[4]; + vec4 fft_texel_at_vertex[3]; fft_texel_at_vertex[0] = imageLoad(fft_displacement_map, ivec2(inUv[0])); fft_texel_at_vertex[1] = imageLoad(fft_displacement_map, ivec2(inUv[1])); fft_texel_at_vertex[2] = imageLoad(fft_displacement_map, ivec2(inUv[2])); - vec4 height_texel_at_vertex[4]; + vec4 height_texel_at_vertex[3]; height_texel_at_vertex[0] = imageLoad(fft_height_map, ivec2(inUv[0])); height_texel_at_vertex[1] = imageLoad(fft_height_map, ivec2(inUv[1])); height_texel_at_vertex[2] = imageLoad(fft_height_map, ivec2(inUv[2])); diff --git a/shaders/subgroups_operations/ocean.vert b/shaders/subgroups_operations/ocean.vert index d404982c4..1076591ee 100644 --- a/shaders/subgroups_operations/ocean.vert +++ b/shaders/subgroups_operations/ocean.vert @@ -19,11 +19,10 @@ layout (location = 0) in vec3 inPos; layout (location = 1) in vec2 inUv; -layout (location = 0) out vec3 outPos; -layout (location = 1) out vec2 outUv; +layout (location = 0) out vec2 outUv; void main() { - outPos = inPos; + gl_Position = vec4(inPos, 1.0f); outUv = inUv; } \ No newline at end of file diff --git a/shaders/subgroups_operations/skybox.vert b/shaders/subgroups_operations/skybox.vert index 4f83571cc..a67c45fd2 100644 --- a/shaders/subgroups_operations/skybox.vert +++ b/shaders/subgroups_operations/skybox.vert @@ -21,14 +21,14 @@ layout (location = 1) in vec2 inUV; layout (location = 0) out vec2 outUV; -layout (set = 0, binding = 0) uniform SkyboxUbo +layout (binding = 0) uniform SkyboxUbo { mat4 mvp; } ubo; void main(void) { - gl_Position = ubo.mvp * vec4(inPos, 1.0f); outUV = inUV; outUV.t = 1.0 - outUV.t; + gl_Position = ubo.mvp * vec4(inPos, 1.0f); } \ No newline at end of file From d4954d2f11b369beebde47704193f6de078ac84e Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Tue, 24 Oct 2023 08:58:46 +0200 Subject: [PATCH 47/53] A rebase fix --- .../subgroups_operations.cpp | 22 ++++-------------- .../subgroups_operations.h | 8 +++++-- shaders/subgroups_operations/ocean.frag | 18 ++++++++------- shaders/subgroups_operations/ocean.tese | 23 ++++++++----------- shaders/subgroups_operations/skybox.frag | 9 ++++---- shaders/subgroups_operations/skybox.vert | 4 ++-- 6 files changed, 36 insertions(+), 48 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 7bda5e6de..ad23c1e60 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -446,11 +446,8 @@ void SubgroupsOperations::create_tildas() void SubgroupsOperations::load_assets() { - // skybox.skybox_shape = load_model("scenes/geosphere.gltf"); - // skybox.skybox_texture = load_texture("textures/skysphere_rgba.ktx", vkb::sg::Image::Color); - - skybox.skybox_shape = load_model("scenes/cube.gltf"); - skybox.skybox_texture = load_texture_cubemap("textures/uffizi_rgba16f_cube.ktx", vkb::sg::Image::Color); + skybox.skybox_shape = load_model("scenes/geosphere.gltf"); + skybox.skybox_texture = load_texture("textures/skysphere_rgba.ktx", vkb::sg::Image::Color); generate_plane(); ui.wind.recalc(); @@ -606,13 +603,7 @@ void SubgroupsOperations::create_descriptor_set_layout() 4u), vkb::initializers::descriptor_set_layout_binding( VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, - 5u), - vkb::initializers::descriptor_set_layout_binding( - VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, - 6u), - vkb::initializers::descriptor_set_layout_binding( - VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, VK_SHADER_STAGE_FRAGMENT_BIT, - 7u)}; + 5u)}; VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); @@ -632,8 +623,6 @@ void SubgroupsOperations::create_descriptor_set() VkDescriptorBufferInfo camera_pos_buffer_descriptor = create_descriptor(*camera_postion_ubo); VkDescriptorImageInfo normal_map_descirptor = create_ia_descriptor(*fft_buffers.fft_normal_map); VkDescriptorBufferInfo ocean_params_buffer_descriptor = create_descriptor(*ocean_params_ubo); - VkDescriptorBufferInfo skybox_uniform_descriptor = create_descriptor(*skybox_ubo); - VkDescriptorImageInfo skybox_image_descriptor = create_descriptor(skybox.skybox_texture); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor), @@ -641,10 +630,7 @@ void SubgroupsOperations::create_descriptor_set() vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &tessellation_params_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &camera_pos_buffer_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &normal_map_descirptor), - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &ocean_params_buffer_descriptor), - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 6u, &skybox_uniform_descriptor), - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 7u, &skybox_image_descriptor), - }; + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &ocean_params_buffer_descriptor)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 7e450f06e..60cb03ae6 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -128,10 +128,14 @@ class SubgroupsOperations : public ApiVulkanSample struct Wind { + Wind() + { + recalc(); + } void recalc(); glm::vec2 vec; - float angle = 180; - float force = 3; + float angle = {180.0f}; + float force = {25.0f}; }; struct GuiConfig diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index 8eb68f2a6..937306998 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -16,9 +16,12 @@ * limitations under the License. */ -layout (location=1) in vec3 normalVector; -layout (location=2) in vec3 eyePosition; -layout (location=3) in vec3 lightVector; +// layout (location=1) in vec3 normalVector; +// layout (location=2) in vec3 eyePosition; +// layout (location=3) in vec3 lightVector; + +layout (location = 0) in vec3 in_pos; +layout (location = 1) in vec3 in_normal; layout (location = 0) out vec4 outFragColor; @@ -34,12 +37,12 @@ layout (binding = 5) uniform OceanParamsUbo vec3 ocean_color; } ocean_ubo; -layout (binding = 7) uniform samplerCube skybox_texture_map; +// layout (binding = 7) uniform samplerCube skybox_texture_map; void main() { - float material_shiness = 27.89f; + /* float material_shiness = 27.89f; vec3 n = normalize(normalVector); vec3 l = normalize(lightVector); @@ -60,8 +63,7 @@ void main() vec3 color = ocean_ubo.light_color * ocean_ubo.ocean_color * i_diffuse + specular * i_specular; outFragColor=vec4(mix(color, reflection, 0.1f), 1.0f); - -/* + */ vec3 normal = in_normal; vec3 light_pos = ocean_ubo.light_position; vec3 light_color = ocean_ubo.light_color; @@ -87,5 +89,5 @@ void main() result = pow(result, vec3(1.0f / 2.2f)); outFragColor = vec4(result, 1.0f); -*/ + } diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index e08d3313b..4cf8006e3 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -20,9 +20,12 @@ layout(triangles, equal_spacing, ccw) in; layout(location = 0) in vec2 inUv[]; -layout (location=1) out vec3 outNormal; -layout (location=2) out vec3 outEyePosition; -layout (location=3) out vec3 outLightVec; +// layout (location=1) out vec3 outNormal; +// layout (location=2) out vec3 outEyePosition; +// layout (location=3) out vec3 outLightVec; + +layout (location = 0) out vec3 outPos; +layout (location = 1) out vec3 outNormal; layout (binding = 0) uniform Ubo { @@ -41,11 +44,6 @@ layout (binding = 2) uniform TessellationParams layout (binding = 4, rgba32f) uniform image2D fft_height_map; -layout (binding = 6) uniform SkyboxUbo -{ - mat4 view; -} skybox_ubo; - vec2 interpolate_2d(vec2 v0, vec2 v1, vec2 v2) { return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2; @@ -86,9 +84,8 @@ void main() gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); - outEyePosition = vec4(ubo.view * ubo.model * vec4(world_pos, 1.0f)).xyz; - outNormal = vec4(ubo.view * ubo.model * vec4(height_texel.xyz, 0.0f)).xyz; - vec3 lightPos = vec3(0.0f, -5.0f, 5.0f); - outLightVec = vec4(ubo.view * vec4(lightPos, 1.0f)).xyz - outEyePosition; - + // outEyePosition = vec4(ubo.view * ubo.model * vec4(world_pos, 1.0f)).xyz; + // outNormal = vec4(ubo.view * ubo.model * vec4(height_texel.xyz, 0.0f)).xyz; +// vec3 lightPos = vec3(0.0f, -5.0f, 5.0f); + // outLightVec = vec4(ubo.view * vec4(lightPos, 1.0f)).xyz - outEyePosition; } diff --git a/shaders/subgroups_operations/skybox.frag b/shaders/subgroups_operations/skybox.frag index ce82b35b3..118451338 100644 --- a/shaders/subgroups_operations/skybox.frag +++ b/shaders/subgroups_operations/skybox.frag @@ -18,13 +18,12 @@ layout (location = 0) out vec4 outFragColor; -layout (location = 0) in vec3 inUV; +layout (location = 0) in vec2 inUV; -layout (set = 0, binding = 1) uniform samplerCube skybox_texture_map; +layout (binding = 1) uniform sampler2D skybox_texture_map; void main(void) { - vec3 normal = normalize(inUV); - vec4 color = texture(skybox_texture_map, inUV); - outFragColor = vec4(color.rgb, 1.0); + vec4 color = texture(skybox_texture_map, inUV); + outFragColor = vec4(color.rgb, 1.0); } diff --git a/shaders/subgroups_operations/skybox.vert b/shaders/subgroups_operations/skybox.vert index 7da418dc5..a67c45fd2 100644 --- a/shaders/subgroups_operations/skybox.vert +++ b/shaders/subgroups_operations/skybox.vert @@ -17,9 +17,9 @@ */ layout (location = 0) in vec3 inPos; -layout (location = 1) in vec3 inUV; +layout (location = 1) in vec2 inUV; -layout (location = 0) out vec3 outUV; +layout (location = 0) out vec2 outUV; layout (binding = 0) uniform SkyboxUbo { From d887e5f71ec90a2472093cbcb35bc53188a0d1ad Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 25 Oct 2023 09:09:22 +0200 Subject: [PATCH 48/53] use initialisation functions to initialise objects --- .../subgroups_operations.cpp | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index ad23c1e60..289e331f6 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -356,10 +356,7 @@ void SubgroupsOperations::create_initial_tildas() VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &initial_tildas.descriptor_set_layout, 1u); VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &initial_tildas.descriptor_set)); - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; - compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - compute_pipeline_layout_info.setLayoutCount = 1u; - compute_pipeline_layout_info.pSetLayouts = &initial_tildas.descriptor_set_layout; + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = vkb::initializers::pipeline_layout_create_info(&initial_tildas.descriptor_set_layout); VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &initial_tildas.pipeline.pipeline_layout)); @@ -411,10 +408,7 @@ void SubgroupsOperations::create_tildas() VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &tildas.descriptor_set_layout, 1u); VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &tildas.descriptor_set)); - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; - compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - compute_pipeline_layout_info.setLayoutCount = 1u; - compute_pipeline_layout_info.pSetLayouts = &tildas.descriptor_set_layout; + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = vkb::initializers::pipeline_layout_create_info(&tildas.descriptor_set_layout); VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &tildas.pipeline.pipeline_layout)); @@ -1122,10 +1116,7 @@ void SubgroupsOperations::create_butterfly_texture() VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &precompute.descriptor_set_layout, 1u); VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &precompute.descriptor_set)); - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; - compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - compute_pipeline_layout_info.setLayoutCount = 1u; - compute_pipeline_layout_info.pSetLayouts = &precompute.descriptor_set_layout; + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = vkb::initializers::pipeline_layout_create_info(&precompute.descriptor_set_layout); VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &precompute.pipeline.pipeline_layout)); @@ -1266,10 +1257,7 @@ void SubgroupsOperations::create_fft_inversion() VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft_inversion.descriptor_set_layout, 1u); VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft_inversion.descriptor_set)); - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; - compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - compute_pipeline_layout_info.setLayoutCount = 1u; - compute_pipeline_layout_info.pSetLayouts = &fft_inversion.descriptor_set_layout; + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = vkb::initializers::pipeline_layout_create_info(&fft_inversion.descriptor_set_layout); VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &fft_inversion.pipeline.pipeline_layout)); @@ -1319,10 +1307,7 @@ void SubgroupsOperations::create_fft_normal_map() VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft_normal_map.descriptor_set_layout, 1u); VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &fft_normal_map.descriptor_set)); - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = {}; - compute_pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - compute_pipeline_layout_info.setLayoutCount = 1u; - compute_pipeline_layout_info.pSetLayouts = &fft_normal_map.descriptor_set_layout; + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = vkb::initializers::pipeline_layout_create_info(&fft_normal_map.descriptor_set_layout); VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &fft_normal_map.pipeline.pipeline_layout)); From ef7159eddb35b72d1fe31baeba7d84fae11f8a6b Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Thu, 2 Nov 2023 14:38:10 +0100 Subject: [PATCH 49/53] [WIP] reflections experiments --- .../subgroups_operations.cpp | 292 +++++++++--------- .../subgroups_operations.h | 26 +- .../subgroups_operations/fft_normal_map.comp | 44 ++- shaders/subgroups_operations/ocean.frag | 79 +++-- shaders/subgroups_operations/ocean.tese | 52 ++-- shaders/subgroups_operations/skybox.frag | 8 +- shaders/subgroups_operations/skybox.vert | 13 +- 7 files changed, 262 insertions(+), 252 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 289e331f6..4f2493be1 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -20,7 +20,7 @@ #include -void SubgroupsOperations::Pipeline::destroy(VkDevice device) +void SubgroupsOperations::Pipeline::destroy(VkDevice device) const { if (pipeline != VK_NULL_HANDLE) { @@ -76,11 +76,11 @@ SubgroupsOperations::~SubgroupsOperations() precompute.pipeline.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), precompute.descriptor_set_layout, nullptr); - tildas.pipeline.destroy(get_device().get_handle()); - vkDestroyDescriptorSetLayout(get_device().get_handle(), tildas.descriptor_set_layout, nullptr); + tildes.pipeline.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), tildes.descriptor_set_layout, nullptr); - initial_tildas.pipeline.destroy(get_device().get_handle()); - vkDestroyDescriptorSetLayout(get_device().get_handle(), initial_tildas.descriptor_set_layout, nullptr); + initial_tildes.pipeline.destroy(get_device().get_handle()); + vkDestroyDescriptorSetLayout(get_device().get_handle(), initial_tildes.descriptor_set_layout, nullptr); fft_inversion.pipeline.destroy(get_device().get_handle()); vkDestroyDescriptorSetLayout(get_device().get_handle(), fft_inversion.descriptor_set_layout, nullptr); @@ -125,7 +125,7 @@ bool SubgroupsOperations::prepare(const vkb::ApplicationOptions &options) create_semaphore(); create_descriptor_set_layout(); - create_initial_tildas(); + create_initial_tides(); create_tildas(); create_butterfly_texture(); create_fft(); @@ -194,7 +194,7 @@ void SubgroupsOperations::build_compute_command_buffer() VkCommandBufferBeginInfo begin_info = vkb::initializers::command_buffer_begin_info(); VK_CHECK(vkBeginCommandBuffer(compute.command_buffer, &begin_info)); - // buttle fly texture + // butterfly texture { vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline); vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, precompute.pipeline.pipeline_layout, 0u, 1u, &precompute.descriptor_set, 0u, nullptr); @@ -202,18 +202,18 @@ void SubgroupsOperations::build_compute_command_buffer() vkCmdDispatch(compute.command_buffer, 1u, grid_size, 1u); } - // initial tildas textures + // initial tildes textures { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, initial_tildas.pipeline.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, initial_tildas.pipeline.pipeline_layout, 0u, 1u, &initial_tildas.descriptor_set, 0u, nullptr); + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, initial_tildes.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, initial_tildes.pipeline.pipeline_layout, 0u, 1u, &initial_tildes.descriptor_set, 0u, nullptr); vkCmdDispatch(compute.command_buffer, grid_size / 32u, grid_size, 1u); } - // tildas textures + // tildes textures { - vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline); - vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildas.pipeline.pipeline_layout, 0u, 1u, &tildas.descriptor_set, 0u, nullptr); + vkCmdBindPipeline(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildes.pipeline.pipeline); + vkCmdBindDescriptorSets(compute.command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, tildes.pipeline.pipeline_layout, 0u, 1u, &tildes.descriptor_set, 0u, nullptr); vkCmdDispatch(compute.command_buffer, grid_size / 8u, grid_size, 1u); } @@ -342,28 +342,28 @@ void SubgroupsOperations::request_gpu_features(vkb::PhysicalDevice &gpu) vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); } -void SubgroupsOperations::create_initial_tildas() +void SubgroupsOperations::create_initial_tides() { - std::vector set_layout_bindngs = { + std::vector set_layout_bindings = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 3u)}; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); - VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &initial_tildas.descriptor_set_layout)); + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &initial_tildes.descriptor_set_layout)); - VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &initial_tildas.descriptor_set_layout, 1u); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &initial_tildas.descriptor_set)); + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &initial_tildes.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &initial_tildes.descriptor_set)); - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = vkb::initializers::pipeline_layout_create_info(&initial_tildas.descriptor_set_layout); + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = vkb::initializers::pipeline_layout_create_info(&initial_tildes.descriptor_set_layout); - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &initial_tildas.pipeline.pipeline_layout)); + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &initial_tildes.pipeline.pipeline_layout)); - VkComputePipelineCreateInfo computeInfo = vkb::initializers::compute_pipeline_create_info(initial_tildas.pipeline.pipeline_layout); + VkComputePipelineCreateInfo computeInfo = vkb::initializers::compute_pipeline_create_info(initial_tildes.pipeline.pipeline_layout); computeInfo.stage = load_shader("subgroups_operations/fft_tilde_h0.comp", VK_SHADER_STAGE_COMPUTE_BIT); - VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &initial_tildas.pipeline.pipeline)); + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &initial_tildes.pipeline.pipeline)); fft_buffers.fft_input_htilde0 = std::make_unique(); fft_buffers.fft_input_htilde0_conj = std::make_unique(); @@ -377,10 +377,10 @@ void SubgroupsOperations::create_initial_tildas() VkDescriptorBufferInfo fft_params_ubo_buffer = create_descriptor(*fft_params_ubo); std::vector write_descriptor_sets = { - vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &htilde_0_descriptor), - vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &htilde_conj_0_descriptor), - vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &input_random_descriptor), - vkb::initializers::write_descriptor_set(initial_tildas.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_params_ubo_buffer)}; + vkb::initializers::write_descriptor_set(initial_tildes.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &htilde_0_descriptor), + vkb::initializers::write_descriptor_set(initial_tildes.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &htilde_conj_0_descriptor), + vkb::initializers::write_descriptor_set(initial_tildes.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2u, &input_random_descriptor), + vkb::initializers::write_descriptor_set(initial_tildes.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &fft_params_ubo_buffer)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } @@ -394,7 +394,7 @@ void SubgroupsOperations::create_tildas() create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dy); create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_tilde_h_kt_dz); - std::vector set_layout_bindngs = { + std::vector set_layout_bindings = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), @@ -402,20 +402,20 @@ void SubgroupsOperations::create_tildas() vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 4u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 5u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 6u)}; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); - VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &tildas.descriptor_set_layout)); + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); + VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &tildes.descriptor_set_layout)); - VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &tildas.descriptor_set_layout, 1u); - VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &tildas.descriptor_set)); + VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &tildes.descriptor_set_layout, 1u); + VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &tildes.descriptor_set)); - VkPipelineLayoutCreateInfo compute_pipeline_layout_info = vkb::initializers::pipeline_layout_create_info(&tildas.descriptor_set_layout); + VkPipelineLayoutCreateInfo compute_pipeline_layout_info = vkb::initializers::pipeline_layout_create_info(&tildes.descriptor_set_layout); - VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &tildas.pipeline.pipeline_layout)); + VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &compute_pipeline_layout_info, nullptr, &tildes.pipeline.pipeline_layout)); - VkComputePipelineCreateInfo computeInfo = vkb::initializers::compute_pipeline_create_info(tildas.pipeline.pipeline_layout); + VkComputePipelineCreateInfo computeInfo = vkb::initializers::compute_pipeline_create_info(tildes.pipeline.pipeline_layout); computeInfo.stage = load_shader("subgroups_operations/fft_tilde_h.comp", VK_SHADER_STAGE_COMPUTE_BIT); - VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &tildas.pipeline.pipeline)); + VK_CHECK(vkCreateComputePipelines(get_device().get_handle(), pipeline_cache, 1u, &computeInfo, nullptr, &tildes.pipeline.pipeline)); VkDescriptorImageInfo htilde_0_descriptor = create_ia_descriptor(*fft_buffers.fft_input_htilde0); VkDescriptorImageInfo htilde_conj_0_descriptor = create_ia_descriptor(*fft_buffers.fft_input_htilde0_conj); @@ -428,25 +428,25 @@ void SubgroupsOperations::create_tildas() VkDescriptorBufferInfo fft_time_ubo_buffer = create_descriptor(*fft_time_ubo); std::vector write_descriptor_sets = { - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &htilde_0_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &htilde_conj_0_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_dx_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 3u, &image_dy_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &image_dz_descriptor), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &fft_params_ubo_buffer), - vkb::initializers::write_descriptor_set(tildas.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 6u, &fft_time_ubo_buffer)}; + vkb::initializers::write_descriptor_set(tildes.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &htilde_0_descriptor), + vkb::initializers::write_descriptor_set(tildes.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &htilde_conj_0_descriptor), + vkb::initializers::write_descriptor_set(tildes.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_dx_descriptor), + vkb::initializers::write_descriptor_set(tildes.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 3u, &image_dy_descriptor), + vkb::initializers::write_descriptor_set(tildes.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &image_dz_descriptor), + vkb::initializers::write_descriptor_set(tildes.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &fft_params_ubo_buffer), + vkb::initializers::write_descriptor_set(tildes.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 6u, &fft_time_ubo_buffer)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } void SubgroupsOperations::load_assets() { - skybox.skybox_shape = load_model("scenes/geosphere.gltf"); - skybox.skybox_texture = load_texture("textures/skysphere_rgba.ktx", vkb::sg::Image::Color); + skybox.skybox_shape = load_model("scenes/cube.gltf"); + skybox.skybox_texture = load_texture_cubemap("textures/uffizi_rgba16f_cube.ktx", vkb::sg::Image::Color); generate_plane(); ui.wind.recalc(); - log_2_N = glm::log2(static_cast(grid_size)); + log_2_N = static_cast(glm::log2(static_cast(grid_size))); input_random.clear(); @@ -456,7 +456,7 @@ void SubgroupsOperations::load_assets() { glm::vec2 rnd1 = rndGaussian(); glm::vec2 rnd2 = rndGaussian(); - input_random.push_back(glm::vec4{rnd1.x, rnd1.y, rnd2.x, rnd2.y}); + input_random.emplace_back(rnd1.x, rnd1.y, rnd2.x, rnd2.y); } } @@ -492,7 +492,7 @@ void SubgroupsOperations::prepare_uniform_buffers() { skybox_ubo = std::make_unique(get_device(), sizeof(SkyboxUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); camera_ubo = std::make_unique(get_device(), sizeof(CameraUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); - camera_postion_ubo = std::make_unique(get_device(), sizeof(CameraPositionUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); + camera_position_ubo = std::make_unique(get_device(), sizeof(CameraPositionUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_params_ubo = std::make_unique(get_device(), sizeof(FFTParametersUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); fft_time_ubo = std::make_unique(get_device(), sizeof(TimeUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); invert_fft_ubo = std::make_unique(get_device(), sizeof(FFTInvertUbo), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, VMA_MEMORY_USAGE_CPU_TO_GPU); @@ -507,9 +507,9 @@ void SubgroupsOperations::generate_plane() uint32_t dim_gird = grid_size; uint32_t vertex_count = dim_gird + 1u; std::vector plane_vertices; - const float tex_coord_scale = float(grid_size); + const auto tex_coord_scale = float(grid_size); std::vector indices; - int32_t half_grid_size = static_cast(dim_gird / 2); + auto half_grid_size = static_cast(dim_gird / 2); for (int32_t z = -half_grid_size; z <= half_grid_size; ++z) { @@ -597,7 +597,10 @@ void SubgroupsOperations::create_descriptor_set_layout() 4u), vkb::initializers::descriptor_set_layout_binding( VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, - 5u)}; + 5u), + vkb::initializers::descriptor_set_layout_binding( + VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, VK_SHADER_STAGE_FRAGMENT_BIT, + 6u)}; VkDescriptorSetLayoutCreateInfo descriptor_set_layout_create_info = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_set_layout_create_info, nullptr, &ocean.descriptor_set_layout)); @@ -612,19 +615,21 @@ void SubgroupsOperations::create_descriptor_set() VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &ocean.descriptor_set)); VkDescriptorBufferInfo buffer_descriptor = create_descriptor(*camera_ubo); - VkDescriptorImageInfo desplacement_descriptor = create_ia_descriptor(*fft_buffers.fft_displacement); + VkDescriptorImageInfo displacement_descriptor = create_ia_descriptor(*fft_buffers.fft_displacement); VkDescriptorBufferInfo tessellation_params_descriptor = create_descriptor(*tessellation_params_ubo); - VkDescriptorBufferInfo camera_pos_buffer_descriptor = create_descriptor(*camera_postion_ubo); - VkDescriptorImageInfo normal_map_descirptor = create_ia_descriptor(*fft_buffers.fft_normal_map); + VkDescriptorBufferInfo camera_pos_buffer_descriptor = create_descriptor(*camera_position_ubo); + VkDescriptorImageInfo normal_map_descriptor = create_ia_descriptor(*fft_buffers.fft_normal_map); VkDescriptorBufferInfo ocean_params_buffer_descriptor = create_descriptor(*ocean_params_ubo); + VkDescriptorImageInfo skybox_cubemap_descriptor = create_descriptor(skybox.skybox_texture); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &buffer_descriptor), - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &desplacement_descriptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &displacement_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &tessellation_params_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &camera_pos_buffer_descriptor), - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &normal_map_descirptor), - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &ocean_params_buffer_descriptor)}; + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &normal_map_descriptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &ocean_params_buffer_descriptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 6u, &skybox_cubemap_descriptor)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } @@ -671,11 +676,11 @@ void SubgroupsOperations::create_pipelines() 0u); VkPipelineTessellationStateCreateInfo tessellation_state = vkb::initializers::pipeline_tessellation_state_create_info(3u); - std::array shader_stages; - shader_stages[0] = load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT); - shader_stages[1] = load_shader("subgroups_operations/ocean.frag", VK_SHADER_STAGE_FRAGMENT_BIT); - shader_stages[2] = load_shader("subgroups_operations/ocean.tesc", VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT); - shader_stages[3] = load_shader("subgroups_operations/ocean.tese", VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT); + std::array shader_stages = { + load_shader("subgroups_operations/ocean.vert", VK_SHADER_STAGE_VERTEX_BIT), + load_shader("subgroups_operations/ocean.frag", VK_SHADER_STAGE_FRAGMENT_BIT), + load_shader("subgroups_operations/ocean.tesc", VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT), + load_shader("subgroups_operations/ocean.tese", VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT)}; const std::vector vertex_input_bindings = { vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX)}; @@ -712,20 +717,17 @@ void SubgroupsOperations::create_pipelines() void SubgroupsOperations::create_skybox() { // descriptors - std::vector set_layout_bindngs = { + std::vector set_layout_bindings = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, VK_SHADER_STAGE_FRAGMENT_BIT, 1u)}; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &skybox.descriptor_set_layout)); VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &skybox.descriptor_set_layout, 1u); VK_CHECK(vkAllocateDescriptorSets(get_device().get_handle(), &alloc_info, &skybox.descriptor_set)); - VkPipelineLayoutCreateInfo pipeline_layout_info = {}; - pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; - pipeline_layout_info.setLayoutCount = 1u; - pipeline_layout_info.pSetLayouts = &skybox.descriptor_set_layout; + VkPipelineLayoutCreateInfo pipeline_layout_info = vkb::initializers::pipeline_layout_create_info(&skybox.descriptor_set_layout); VK_CHECK(vkCreatePipelineLayout(get_device().get_handle(), &pipeline_layout_info, nullptr, &skybox.pipeline.pipeline_layout)); @@ -734,8 +736,7 @@ void SubgroupsOperations::create_skybox() std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(skybox.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 0u, &skybox_uniform_descriptor), - vkb::initializers::write_descriptor_set(skybox.descriptor_set, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1u, &skybox_image_descriptor), - }; + vkb::initializers::write_descriptor_set(skybox.descriptor_set, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1u, &skybox_image_descriptor)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); // pipeline @@ -747,7 +748,7 @@ void SubgroupsOperations::create_skybox() VkPipelineRasterizationStateCreateInfo rasterization_state = vkb::initializers::pipeline_rasterization_state_create_info( VK_POLYGON_MODE_FILL, - VK_CULL_MODE_NONE, + VK_CULL_MODE_BACK_BIT, VK_FRONT_FACE_COUNTER_CLOCKWISE, 0u); VkPipelineColorBlendAttachmentState blend_attachment_state = @@ -760,9 +761,9 @@ void SubgroupsOperations::create_skybox() &blend_attachment_state); VkPipelineDepthStencilStateCreateInfo depth_stencil_state = vkb::initializers::pipeline_depth_stencil_state_create_info( - VK_TRUE, VK_FALSE, - VK_COMPARE_OP_GREATER); + VK_FALSE, + VK_COMPARE_OP_LESS_OR_EQUAL); VkPipelineViewportStateCreateInfo viewport_state = vkb::initializers::pipeline_viewport_state_create_info(1u, 1u, 0u); @@ -782,15 +783,15 @@ void SubgroupsOperations::create_skybox() static_cast(dynamic_state_enables.size()), 0u); - std::array shader_stages; - shader_stages[0] = load_shader("subgroups_operations/skybox.vert", VK_SHADER_STAGE_VERTEX_BIT); - shader_stages[1] = load_shader("subgroups_operations/skybox.frag", VK_SHADER_STAGE_FRAGMENT_BIT); + std::array shader_stages = { + load_shader("subgroups_operations/skybox.vert", VK_SHADER_STAGE_VERTEX_BIT), + load_shader("subgroups_operations/skybox.frag", VK_SHADER_STAGE_FRAGMENT_BIT)}; - const std::vector vertex_input_bindings = { - vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX)}; + const std::vector + vertex_input_bindings = {vkb::initializers::vertex_input_binding_description(0u, sizeof(Vertex), VK_VERTEX_INPUT_RATE_VERTEX)}; const std::vector vertex_input_attributes = { - vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, offsetof(Vertex, pos)), - vkb::initializers::vertex_input_attribute_description(0u, 1u, VK_FORMAT_R32G32_SFLOAT, offsetof(Vertex, uv))}; + vkb::initializers::vertex_input_attribute_description(0u, 0u, VK_FORMAT_R32G32B32_SFLOAT, 0), + vkb::initializers::vertex_input_attribute_description(0u, 1u, VK_FORMAT_R32G32B32_SFLOAT, sizeof(float) * 3)}; VkPipelineVertexInputStateCreateInfo vertex_input_state = vkb::initializers::pipeline_vertex_input_state_create_info(); vertex_input_state.vertexBindingDescriptionCount = static_cast(vertex_input_bindings.size()); @@ -814,56 +815,56 @@ void SubgroupsOperations::create_skybox() void SubgroupsOperations::update_uniform_buffers() { - CameraUbo ubo; + CameraUbo ubo = {}; ubo.model = glm::mat4(1.0f); ubo.view = camera.matrices.view; ubo.projection = camera.matrices.perspective; - CameraPositionUbo cam_pos; - cam_pos.position = glm::vec4(camera.position, 0.0f); + CameraPositionUbo cam_pos = {}; + cam_pos.position = glm::vec4(camera.position, 0.0f); - FFTParametersUbo fft_ubo; - fft_ubo.amplitude = ui.amplitude; - fft_ubo.grid_size = grid_size; - fft_ubo.length = ui.length; - fft_ubo.wind = ui.wind.vec; + FFTParametersUbo fft_ubo = {}; + fft_ubo.amplitude = ui.amplitude; + fft_ubo.grid_size = grid_size; + fft_ubo.length = ui.length; + fft_ubo.wind = ui.wind.vec; - FFTInvertUbo invertFft; - invertFft.grid_size = grid_size; - invertFft.page_idx = log_2_N % 2; + FFTInvertUbo invert_fft = {}; + invert_fft.page_idx = static_cast(log_2_N % 2); + invert_fft.grid_size = grid_size; - TessellationParamsUbo tess_params; - tess_params.displacement_scale = ui.displacement_scale; - tess_params.choppines = ui.choppines; + TessellationParamsUbo tess_params = {}; + tess_params.displacement_scale = ui.displacement_scale; + tess_params.choppines = ui.choppines; - TimeUbo t; - t.time = float(timer.elapsed()); + TimeUbo t = {}; + t.time = static_cast(timer.elapsed()); - OceanParamsUbo ocean_params; + OceanParamsUbo ocean_params = {}; ocean_params.light_color = ui.light_color; ocean_params.light_position = ui.light_pos; ocean_params.ocean_color = ui.ocean_color; - SkyboxUbo skybox_params; - skybox_params.mvp = camera.matrices.perspective * glm::mat4(glm::mat3(camera.matrices.view)); + SkyboxUbo skybox_params = {}; + skybox_params.mvp = camera.matrices.perspective * glm::mat4(glm::mat3(camera.matrices.view)); skybox_ubo->convert_and_update(skybox_params); ocean_params_ubo->convert_and_update(ocean_params); fft_time_ubo->convert_and_update(t); camera_ubo->convert_and_update(ubo); fft_params_ubo->convert_and_update(fft_ubo); - invert_fft_ubo->convert_and_update(invertFft); + invert_fft_ubo->convert_and_update(invert_fft); tessellation_params_ubo->convert_and_update(tess_params); - camera_postion_ubo->convert_and_update(cam_pos); + camera_position_ubo->convert_and_update(cam_pos); } void SubgroupsOperations::build_command_buffers() { VkCommandBufferBeginInfo command_buffer_begin_info = vkb::initializers::command_buffer_begin_info(); - std::array clear_values; - clear_values[0].color = {{0.0f, 0.0f, 0.0f, 0.0f}}; - clear_values[1].depthStencil = {0.0f, 0u}; + std::array clear_values = {}; + clear_values[0].color = {{0.0f, 0.0f, 0.0f, 0.0f}}; + clear_values[1].depthStencil = {0.0f, 0u}; VkRenderPassBeginInfo render_pass_begin_info = vkb::initializers::render_pass_begin_info(); render_pass_begin_info.renderPass = render_pass; @@ -1026,12 +1027,6 @@ void SubgroupsOperations::render(float delta_time) draw(); } -std::unique_ptr create_subgroups_operations() -{ - return std::make_unique(); -} - -// TODO: move out usage to the function arguments void SubgroupsOperations::create_image_attachement(VkFormat format, uint32_t width, uint32_t height, ImageAttachment &attachment) { attachment.format = format; @@ -1093,7 +1088,7 @@ void SubgroupsOperations::create_image_attachement(VkFormat format, uint32_t wid get_device().flush_command_buffer(cmd, queue, true); } -uint32_t SubgroupsOperations::reverse(uint32_t i) +uint32_t SubgroupsOperations::reverse(uint32_t i) const { uint32_t res = 0; for (int j = 0; j < log_2_N; j++) @@ -1106,11 +1101,12 @@ uint32_t SubgroupsOperations::reverse(uint32_t i) void SubgroupsOperations::create_butterfly_texture() { - std::vector set_layout_bindngs = { + std::vector set_layout_bindings = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 1u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u)}; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &precompute.descriptor_set_layout)); VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &precompute.descriptor_set_layout, 1u); @@ -1143,19 +1139,17 @@ void SubgroupsOperations::create_butterfly_texture() std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor), vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, &bit_reverse_descriptor), - vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &fft_params_ubo_buffer), - }; + vkb::initializers::write_descriptor_set(precompute.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &fft_params_ubo_buffer)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } void SubgroupsOperations::create_fft() { - std::vector set_layout_bindngs = { + std::vector set_layout_bindings = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), - vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), - }; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u)}; + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft.descriptor_set_layout)); VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft.descriptor_set_layout, 1u); @@ -1180,7 +1174,7 @@ void SubgroupsOperations::create_fft() computeInfo.layout = fft.pipelines.horizontal.pipeline_layout; computeInfo.stage = load_shader("subgroups_operations/fft.comp", VK_SHADER_STAGE_COMPUTE_BIT); - std::array specialization_map_entries; + std::array specialization_map_entries = {}; VkSpecializationInfo spec_info; uint32_t direction = 0u; specialization_map_entries[0] = vkb::initializers::specialization_map_entry(0u, 0u, sizeof(uint32_t)); @@ -1204,44 +1198,41 @@ void SubgroupsOperations::create_fft() create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_x); create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft.tilde_axis_z); - VkDescriptorImageInfo image_descriptor_battlefly = create_ia_descriptor(butterfly_precomp); + VkDescriptorImageInfo image_descriptor_butterfly = create_ia_descriptor(butterfly_precomp); VkDescriptorImageInfo image_descriptor_tilda_y = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dy); VkDescriptorImageInfo image_descriptor_tilde_axis_y = create_ia_descriptor(*fft.tilde_axis_y); - std::vector write_descriptor_sets_asix_y = { - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), + std::vector write_descriptor_sets_axis_y = { + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_butterfly), vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_y), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_y), - }; + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_y, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_y)}; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_y.size()), write_descriptor_sets_asix_y.data(), 0u, nullptr); + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_axis_y.size()), write_descriptor_sets_axis_y.data(), 0u, nullptr); VkDescriptorImageInfo image_descriptor_tilda_x = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dx); VkDescriptorImageInfo image_descriptor_tilde_axis_x = create_ia_descriptor(*fft.tilde_axis_x); - std::vector write_descriptor_sets_asix_x = { - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), + std::vector write_descriptor_sets_axis_x = { + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_butterfly), vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_x), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_x), - }; + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_x, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_x)}; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_x.size()), write_descriptor_sets_asix_x.data(), 0u, nullptr); + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_axis_x.size()), write_descriptor_sets_axis_x.data(), 0u, nullptr); VkDescriptorImageInfo image_descriptor_tilda_z = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dz); VkDescriptorImageInfo image_descriptor_tilde_axis_z = create_ia_descriptor(*fft.tilde_axis_z); - std::vector write_descriptor_sets_asix_z = { - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_battlefly), + std::vector write_descriptor_sets_axis_z = { + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_butterfly), vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_tilda_z), - vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_z), - }; + vkb::initializers::write_descriptor_set(fft.descriptor_set_axis_z, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_tilde_axis_z)}; - vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_asix_z.size()), write_descriptor_sets_asix_z.data(), 0u, nullptr); + vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets_axis_z.size()), write_descriptor_sets_axis_z.data(), 0u, nullptr); } void SubgroupsOperations::create_fft_inversion() { - std::vector set_layout_bindngs = { + std::vector set_layout_bindings = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 2u), @@ -1251,7 +1242,7 @@ void SubgroupsOperations::create_fft_inversion() vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 6u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 7u)}; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft_inversion.descriptor_set_layout)); VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft_inversion.descriptor_set_layout, 1u); @@ -1272,7 +1263,7 @@ void SubgroupsOperations::create_fft_inversion() create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_displacement); - VkDescriptorImageInfo image_descriptor_displacment_axis = create_ia_descriptor(*fft_buffers.fft_displacement); + VkDescriptorImageInfo image_descriptor_displacement = create_ia_descriptor(*fft_buffers.fft_displacement); VkDescriptorImageInfo image_descriptor_pingpong0_axis_y = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dy); VkDescriptorImageInfo image_descriptor_pingpong1_axis_y = create_ia_descriptor(*fft.tilde_axis_y); VkDescriptorImageInfo image_descriptor_pingpong0_axis_x = create_ia_descriptor(*fft_buffers.fft_tilde_h_kt_dx); @@ -1283,7 +1274,7 @@ void SubgroupsOperations::create_fft_inversion() auto fft_page_descriptor = create_descriptor(*invert_fft_ubo); std::vector write_descriptor_sets = { - vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_displacment_axis), + vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_displacement), vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_pingpong0_axis_y), vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 2u, &image_descriptor_pingpong1_axis_y), vkb::initializers::write_descriptor_set(fft_inversion.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 3u, &image_descriptor_pingpong0_axis_x), @@ -1296,12 +1287,12 @@ void SubgroupsOperations::create_fft_inversion() void SubgroupsOperations::create_fft_normal_map() { - std::vector set_layout_bindngs = { + std::vector set_layout_bindings = { vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 0u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_COMPUTE_BIT, 1u), vkb::initializers::descriptor_set_layout_binding(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT, 2u)}; - VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindngs); + VkDescriptorSetLayoutCreateInfo descriptor_layout = vkb::initializers::descriptor_set_layout_create_info(set_layout_bindings); VK_CHECK(vkCreateDescriptorSetLayout(get_device().get_handle(), &descriptor_layout, nullptr, &fft_normal_map.descriptor_set_layout)); VkDescriptorSetAllocateInfo alloc_info = vkb::initializers::descriptor_set_allocate_info(descriptor_pool, &fft_normal_map.descriptor_set_layout, 1u); @@ -1322,13 +1313,13 @@ void SubgroupsOperations::create_fft_normal_map() create_image_attachement(VK_FORMAT_R32G32B32A32_SFLOAT, grid_size, grid_size, *fft_buffers.fft_normal_map); - VkDescriptorImageInfo image_descriptor_normal_map = create_ia_descriptor(*fft_buffers.fft_normal_map); - VkDescriptorImageInfo image_descriptor_displacment_axis = create_ia_descriptor(*fft_buffers.fft_displacement); - auto fft_page_descriptor = create_descriptor(*invert_fft_ubo); + VkDescriptorImageInfo image_descriptor_normal_map = create_ia_descriptor(*fft_buffers.fft_normal_map); + VkDescriptorImageInfo image_descriptor_displacement = create_ia_descriptor(*fft_buffers.fft_displacement); + VkDescriptorBufferInfo fft_page_descriptor = create_descriptor(*invert_fft_ubo); std::vector write_descriptor_sets = { vkb::initializers::write_descriptor_set(fft_normal_map.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 0u, &image_descriptor_normal_map), - vkb::initializers::write_descriptor_set(fft_normal_map.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_displacment_axis), + vkb::initializers::write_descriptor_set(fft_normal_map.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &image_descriptor_displacement), vkb::initializers::write_descriptor_set(fft_normal_map.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &fft_page_descriptor)}; vkUpdateDescriptorSets(get_device().get_handle(), static_cast(write_descriptor_sets.size()), write_descriptor_sets.data(), 0u, nullptr); } @@ -1345,6 +1336,11 @@ VkDescriptorImageInfo SubgroupsOperations::create_ia_descriptor(ImageAttachment void SubgroupsOperations::Wind::recalc() { float rad = angle * glm::pi() / 180.0f; - vec.x = force * cos(rad); - vec.y = force * sin(rad); + vec.x = force * glm::cos(rad); + vec.y = force * glm::sin(rad); } + +std::unique_ptr create_subgroups_operations() +{ + return std::make_unique(); +} \ No newline at end of file diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 60cb03ae6..37ee6f310 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -51,7 +51,7 @@ class SubgroupsOperations : public ApiVulkanSample void create_pipelines(); void create_skybox(); - void create_initial_tildas(); + void create_initial_tides(); void create_tildas(); void create_butterfly_texture(); void create_fft(); @@ -60,11 +60,11 @@ class SubgroupsOperations : public ApiVulkanSample void update_uniform_buffers(); - glm::vec2 rndGaussian(); + static glm::vec2 rndGaussian(); struct Pipeline { - void destroy(VkDevice device); + void destroy(VkDevice device) const; VkPipeline pipeline = {VK_NULL_HANDLE}; VkPipelineLayout pipeline_layout = {VK_NULL_HANDLE}; @@ -124,7 +124,7 @@ class SubgroupsOperations : public ApiVulkanSample struct TimeUbo { alignas(4) float time = {0.0f}; - } fftTime; + }; struct Wind { @@ -141,7 +141,7 @@ class SubgroupsOperations : public ApiVulkanSample struct GuiConfig { bool wireframe = {true}; - float choppines = {0.75f}; + float choppines = {0.1f}; float displacement_scale = {0.5f}; float amplitude = {32.0f}; float length = {1900.0f}; @@ -155,7 +155,7 @@ class SubgroupsOperations : public ApiVulkanSample uint32_t grid_size = {256U}; std::unique_ptr skybox_ubo = {VK_NULL_HANDLE}; std::unique_ptr ocean_params_ubo = {VK_NULL_HANDLE}; - std::unique_ptr camera_postion_ubo = {VK_NULL_HANDLE}; + std::unique_ptr camera_position_ubo = {VK_NULL_HANDLE}; std::unique_ptr camera_ubo = {VK_NULL_HANDLE}; std::unique_ptr tessellation_params_ubo = {VK_NULL_HANDLE}; std::unique_ptr fft_params_ubo = {VK_NULL_HANDLE}; @@ -171,7 +171,7 @@ class SubgroupsOperations : public ApiVulkanSample VkDeviceMemory memory; VkImageView view; VkFormat format; - void destroy(VkDevice device) + void destroy(VkDevice device) const { vkDestroyImageView(device, view, nullptr); vkDestroyImage(device, image, nullptr); @@ -179,9 +179,9 @@ class SubgroupsOperations : public ApiVulkanSample }; }; - ImageAttachment butterfly_precomp; + ImageAttachment butterfly_precomp{}; - uint32_t log_2_N; + uint32_t log_2_N{}; vkb::Timer timer; struct @@ -243,14 +243,14 @@ class SubgroupsOperations : public ApiVulkanSample VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; Pipeline pipeline; - } initial_tildas; + } initial_tildes; struct { VkDescriptorSetLayout descriptor_set_layout = {VK_NULL_HANDLE}; VkDescriptorSet descriptor_set = {VK_NULL_HANDLE}; Pipeline pipeline; - } tildas; + } tildes; struct { @@ -292,10 +292,10 @@ class SubgroupsOperations : public ApiVulkanSample std::unique_ptr skybox_shape; } skybox; - VkPhysicalDeviceSubgroupProperties subgroups_properties; + VkPhysicalDeviceSubgroupProperties subgroups_properties{}; private: - uint32_t reverse(uint32_t i); + uint32_t reverse(uint32_t i) const; VkDescriptorImageInfo create_ia_descriptor(ImageAttachment &attachment); void create_image_attachement(VkFormat format, uint32_t width, uint32_t height, ImageAttachment &result); }; diff --git a/shaders/subgroups_operations/fft_normal_map.comp b/shaders/subgroups_operations/fft_normal_map.comp index 162a03dfa..1965e58e2 100644 --- a/shaders/subgroups_operations/fft_normal_map.comp +++ b/shaders/subgroups_operations/fft_normal_map.comp @@ -1,5 +1,5 @@ #version 450 -#extension GL_KHR_shader_subgroup_basic : enable +#extension GL_KHR_shader_subgroup_basic: enable /* Copyright (c) 2023, Mobica Limited * * SPDX-License-Identifier: Apache-2.0 @@ -16,13 +16,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; +layout (local_size_x = 32, local_size_y = 1, local_size_z = 1) in; - layout (binding = 0, rgba32f) writeonly uniform image2D fft_normal_map; +layout (binding = 0, rgba32f) writeonly uniform image2D fft_normal_map; - layout (binding = 1, rgba32f) readonly uniform image2D fft_displacement_map; +layout (binding = 1, rgba32f) readonly uniform image2D fft_displacement_map; - layout (binding = 2) uniform InvertFft +layout (binding = 2) uniform InvertFft { int pong_idx; uint grid_size; @@ -32,17 +32,33 @@ void main() { uint N = fftUbo.grid_size; ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - float tex_dim = 1.0f / N; + float tex_dim = 1.0f; - float left_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-tex_dim, 0.0f))).g; - float right_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, 0.0f))).g; - float bottom_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, tex_dim))).g; - float top_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, tex_dim))).g; + vec3 v0 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, 0.0f))).rgb; + vec3 v1 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, 0.0f))).rgb; + vec3 v2 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, tex_dim))).rgb; + vec3 normal = cross(v1 - v0, v2 - v0); + + // float left_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-tex_dim, 0.0f))).g; + // float right_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, 0.0f))).g; + // float bottom_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, tex_dim))).g; + // float top_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, tex_dim))).g; + + // float z0 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-tex_dim, -tex_dim))).g; + // float z1 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0, -tex_dim))).g; + // float z2 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, -tex_dim))).g; + // float z3 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-tex_dim, 0))).g; + // float z4 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, 0))).g; + // float z5 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-tex_dim, tex_dim))).g; + // float z6 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0, tex_dim))).g; + // float z7 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, tex_dim))).g; vec3 result = vec3(0.0f); - result.z = (bottom_y - top_y) / float(N); - result.x = (left_y - right_y) / float(N); - result.y = 1.0f / float(N); + // result.z = (bottom_y - top_y) / float(N); + // result.x = (left_y - right_y) / float(N); + // result.z = z0 + 2 * z1 + z2 - z5 - 2 * z6 - z7; + // result.x = z0 + 2 * z3 + z5 - z2 - 2 * z4 - z7; + // result.y = 1.0f / float(N); - imageStore(fft_normal_map, pixel_pos, vec4(normalize(result), 1.0f)); + imageStore(fft_normal_map, pixel_pos, vec4(normalize(normal), 1.0f)); } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index 937306998..3e1435db4 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -17,11 +17,11 @@ */ // layout (location=1) in vec3 normalVector; -// layout (location=2) in vec3 eyePosition; // layout (location=3) in vec3 lightVector; layout (location = 0) in vec3 in_pos; layout (location = 1) in vec3 in_normal; +layout (location = 2) in vec3 eyePosition; layout (location = 0) out vec4 outFragColor; @@ -32,62 +32,61 @@ layout (binding = 3) uniform CameraPos layout (binding = 5) uniform OceanParamsUbo { - vec3 light_color; - vec3 light_position; - vec3 ocean_color; + vec3 light_color; + vec3 light_position; + vec3 ocean_color; } ocean_ubo; -// layout (binding = 7) uniform samplerCube skybox_texture_map; +layout (binding = 6) uniform samplerCube skybox_texture_map; -void main() +void main() { - - /* float material_shiness = 27.89f; - - vec3 n = normalize(normalVector); - vec3 l = normalize(lightVector); + float material_shiness = 27.89f; + vec3 n = normalize(in_normal); + vec3 l = normalize(ocean_ubo.light_position - in_pos); float i_diffuse = clamp(dot(n, l), 0, 1); vec3 e = normalize(-eyePosition); vec3 r = reflect(-l, n); - float i_specular = pow(clamp(dot(e, r), 0, 1), material_shiness); + float i_specular = pow(clamp(dot(e, r), 0.0f, 1.0f), material_shiness); float lightPower = 1; vec3 specular = vec3(0.3f, 0.3f, 0.3f); + float radio = 1.0f / 1.33f; - vec3 R = reflect(eyePosition.xyz, normalVector); + vec3 R = refract(eyePosition.xyz, in_normal, radio); vec3 reflection = texture(skybox_texture_map, R).rgb; vec3 color = ocean_ubo.light_color * ocean_ubo.ocean_color * i_diffuse + specular * i_specular; - outFragColor=vec4(mix(color, reflection, 0.1f), 1.0f); - */ - vec3 normal = in_normal; - vec3 light_pos = ocean_ubo.light_position; - vec3 light_color = ocean_ubo.light_color; - vec3 ocean_color = ocean_ubo.ocean_color; - float ambient_strength = 0.91f; - - vec3 ambient = ambient_strength * light_color; - - vec3 light_dir = normalize(light_pos - in_pos); - float diff = max(dot(normal, light_dir), 0.0f); - vec3 diffuse = diff * light_color; - - float specular_strength = 0.5f; - vec3 view_dir = normalize(vec3(cam.position.xyz - in_pos)); - vec3 ref_dir = reflect(-light_dir, normal); - float spec = pow(max(dot(view_dir, ref_dir), 0.0f), 32); - vec3 specular = specular_strength * spec * light_color; - - vec3 result = (ambient + diffuse + specular) * ocean_color; - result = result / (result + vec3(1.0f)); - - // gamma correction - result = pow(result, vec3(1.0f / 2.2f)); - - outFragColor = vec4(result, 1.0f); + outFragColor = vec4(mix(color, reflection, 0.1f), 1.0f); + + // vec3 normal = in_normal; + // vec3 light_pos = ocean_ubo.light_position; + // vec3 light_color = ocean_ubo.light_color; + // vec3 ocean_color = ocean_ubo.ocean_color; + // float ambient_strength = 0.91f; + // + // vec3 ambient = ambient_strength * light_color; + // + // vec3 light_dir = normalize(light_pos - in_pos); + // float diff = max(dot(normal, light_dir), 0.0f); + // vec3 diffuse = diff * light_color; + // + // float specular_strength = 0.5f; + // vec3 view_dir = normalize(vec3(cam.position.xyz - in_pos)); + // vec3 ref_dir = reflect(-light_dir, normal); + // float spec = pow(max(dot(view_dir, ref_dir), 0.0f), 32); + // vec3 specular = specular_strength * spec * light_color; + // + // vec3 result = texture(skybox_texture_map, ref_dir).rgb * (ambient + diffuse + specular) * ocean_color; + // result = result / (result + vec3(1.0f)); + // + // // gamma correction + // result = pow(result, vec3(1.0f / 2.2f)); + // + // outFragColor = vec4(result, 1.0f); } diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 4cf8006e3..084c3d5e2 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -16,62 +16,62 @@ * limitations under the License. */ -layout(triangles, equal_spacing, ccw) in; +layout (triangles, equal_spacing, ccw) in; -layout(location = 0) in vec2 inUv[]; +layout (location = 0) in vec2 inUv[]; // layout (location=1) out vec3 outNormal; -// layout (location=2) out vec3 outEyePosition; +layout (location = 2) out vec3 outEyePosition; // layout (location=3) out vec3 outLightVec; layout (location = 0) out vec3 outPos; layout (location = 1) out vec3 outNormal; -layout (binding = 0) uniform Ubo +layout (binding = 0) uniform Ubo { - mat4 projection; - mat4 view; - mat4 model; + mat4 projection; + mat4 view; + mat4 model; } ubo; layout (binding = 1, rgba32f) uniform image2D fft_displacement_map; layout (binding = 2) uniform TessellationParams { - float choppines; - float displacement_scale; + float choppines; + float displacement_scale; } tessParams; -layout (binding = 4, rgba32f) uniform image2D fft_height_map; +layout (binding = 4, rgba32f) uniform image2D fft_height_map; vec2 interpolate_2d(vec2 v0, vec2 v1, vec2 v2) { - return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2; + return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2; } vec3 interpolate_3d(vec3 v0, vec3 v1, vec3 v2) { - return vec3(gl_TessCoord.x) * v0 + vec3(gl_TessCoord.y) * v1 + vec3(gl_TessCoord.z) * v2; + return vec3(gl_TessCoord.x) * v0 + vec3(gl_TessCoord.y) * v1 + vec3(gl_TessCoord.z) * v2; } vec4 interpolate_4d(vec4 v0, vec4 v1, vec4 v2) { - return vec4(gl_TessCoord.x) * v0 + vec4(gl_TessCoord.y) * v1 + vec4(gl_TessCoord.z) * v2; + return vec4(gl_TessCoord.x) * v0 + vec4(gl_TessCoord.y) * v1 + vec4(gl_TessCoord.z) * v2; } void main() { - vec3 world_pos = interpolate_3d(gl_in[0].gl_Position.xyz, gl_in[1].gl_Position.xyz, gl_in[2].gl_Position.xyz); + vec3 world_pos = interpolate_3d(gl_in[0].gl_Position.xyz, gl_in[1].gl_Position.xyz, gl_in[2].gl_Position.xyz); - vec4 fft_texel_at_vertex[3]; - fft_texel_at_vertex[0] = imageLoad(fft_displacement_map, ivec2(inUv[0])); - fft_texel_at_vertex[1] = imageLoad(fft_displacement_map, ivec2(inUv[1])); - fft_texel_at_vertex[2] = imageLoad(fft_displacement_map, ivec2(inUv[2])); + vec4 fft_texel_at_vertex[3]; + fft_texel_at_vertex[0] = imageLoad(fft_displacement_map, ivec2(inUv[0])); + fft_texel_at_vertex[1] = imageLoad(fft_displacement_map, ivec2(inUv[1])); + fft_texel_at_vertex[2] = imageLoad(fft_displacement_map, ivec2(inUv[2])); - vec4 height_texel_at_vertex[3]; - height_texel_at_vertex[0] = imageLoad(fft_height_map, ivec2(inUv[0])); - height_texel_at_vertex[1] = imageLoad(fft_height_map, ivec2(inUv[1])); - height_texel_at_vertex[2] = imageLoad(fft_height_map, ivec2(inUv[2])); + vec4 height_texel_at_vertex[3]; + height_texel_at_vertex[0] = imageLoad(fft_height_map, ivec2(inUv[0])); + height_texel_at_vertex[1] = imageLoad(fft_height_map, ivec2(inUv[1])); + height_texel_at_vertex[2] = imageLoad(fft_height_map, ivec2(inUv[2])); vec4 fft_texel = interpolate_4d(fft_texel_at_vertex[0], fft_texel_at_vertex[1], fft_texel_at_vertex[2]); vec4 height_texel = interpolate_4d(height_texel_at_vertex[0], height_texel_at_vertex[1], height_texel_at_vertex[2]); @@ -84,8 +84,8 @@ void main() gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); - // outEyePosition = vec4(ubo.view * ubo.model * vec4(world_pos, 1.0f)).xyz; - // outNormal = vec4(ubo.view * ubo.model * vec4(height_texel.xyz, 0.0f)).xyz; -// vec3 lightPos = vec3(0.0f, -5.0f, 5.0f); - // outLightVec = vec4(ubo.view * vec4(lightPos, 1.0f)).xyz - outEyePosition; + outEyePosition = vec4(ubo.view * ubo.model * vec4(world_pos, 1.0f)).xyz; + // outNormal = vec4(ubo.view * ubo.model * vec4(height_texel.xyz, 0.0f)).xyz; + // vec3 lightPos = vec3(0.0f, -5.0f, 5.0f); + // outLightVec = vec4(ubo.view * vec4(lightPos, 1.0f)).xyz - outEyePosition; } diff --git a/shaders/subgroups_operations/skybox.frag b/shaders/subgroups_operations/skybox.frag index 118451338..2211a185f 100644 --- a/shaders/subgroups_operations/skybox.frag +++ b/shaders/subgroups_operations/skybox.frag @@ -18,12 +18,12 @@ layout (location = 0) out vec4 outFragColor; -layout (location = 0) in vec2 inUV; +layout (location = 0) in vec3 inUV; -layout (binding = 1) uniform sampler2D skybox_texture_map; +layout (binding = 1) uniform samplerCube skybox_texture_map; void main(void) { - vec4 color = texture(skybox_texture_map, inUV); - outFragColor = vec4(color.rgb, 1.0); + vec4 color = texture(skybox_texture_map, inUV); + outFragColor = vec4(color.rgb, 1.0); } diff --git a/shaders/subgroups_operations/skybox.vert b/shaders/subgroups_operations/skybox.vert index a67c45fd2..8fb0c2bee 100644 --- a/shaders/subgroups_operations/skybox.vert +++ b/shaders/subgroups_operations/skybox.vert @@ -17,18 +17,17 @@ */ layout (location = 0) in vec3 inPos; -layout (location = 1) in vec2 inUV; +layout (location = 1) in vec3 inUV; -layout (location = 0) out vec2 outUV; +layout (location = 0) out vec3 outUV; -layout (binding = 0) uniform SkyboxUbo +layout (binding = 0) uniform SkyboxUbo { - mat4 mvp; + mat4 mvp; } ubo; void main(void) { - outUV = inUV; - outUV.t = 1.0 - outUV.t; - gl_Position = ubo.mvp * vec4(inPos, 1.0f); + outUV = inUV; + gl_Position = ubo.mvp * vec4(inPos, 1.0f); } \ No newline at end of file From f6b125ca0736c32cd3b089d97de5f01c17443995 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 8 Nov 2023 10:43:50 +0100 Subject: [PATCH 50/53] Convert a README.md file to a README.adoc file --- .../extensions/subgroups_operations/{README.md => README.adoc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename samples/extensions/subgroups_operations/{README.md => README.adoc} (100%) diff --git a/samples/extensions/subgroups_operations/README.md b/samples/extensions/subgroups_operations/README.adoc similarity index 100% rename from samples/extensions/subgroups_operations/README.md rename to samples/extensions/subgroups_operations/README.adoc From d09532f721616f877c845f17ca1165d5e002a312 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Wed, 15 Nov 2023 18:05:24 +0100 Subject: [PATCH 51/53] [WIP] Add BRDF reflection --- .../subgroups_operations.cpp | 27 +- .../subgroups_operations.h | 4 +- .../subgroups_operations/fft_normal_map.comp | 38 +-- shaders/subgroups_operations/ocean.frag | 262 ++++++++++++++---- shaders/subgroups_operations/ocean.tese | 22 +- 5 files changed, 250 insertions(+), 103 deletions(-) diff --git a/samples/extensions/subgroups_operations/subgroups_operations.cpp b/samples/extensions/subgroups_operations/subgroups_operations.cpp index 4f2493be1..891f3ae3e 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.cpp +++ b/samples/extensions/subgroups_operations/subgroups_operations.cpp @@ -593,7 +593,7 @@ void SubgroupsOperations::create_descriptor_set_layout() VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 3u), vkb::initializers::descriptor_set_layout_binding( - VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 4u), vkb::initializers::descriptor_set_layout_binding( VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, @@ -627,7 +627,7 @@ void SubgroupsOperations::create_descriptor_set() vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1u, &displacement_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 2u, &tessellation_params_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 3u, &camera_pos_buffer_descriptor), - vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4u, &normal_map_descriptor), + vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 4u, &normal_map_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 5u, &ocean_params_buffer_descriptor), vkb::initializers::write_descriptor_set(ocean.descriptor_set, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 6u, &skybox_cubemap_descriptor)}; @@ -1066,6 +1066,21 @@ void SubgroupsOperations::create_image_attachement(VkFormat format, uint32_t wid image_view_create_info.image = attachment.image; VK_CHECK(vkCreateImageView(get_device().get_handle(), &image_view_create_info, nullptr, &attachment.view)); + VkSamplerCreateInfo sampler_info = vkb::initializers::sampler_create_info(); + sampler_info.magFilter = VK_FILTER_LINEAR; + sampler_info.minFilter = VK_FILTER_LINEAR; + sampler_info.addressModeU = VK_SAMPLER_ADDRESS_MODE_REPEAT; + sampler_info.addressModeV = VK_SAMPLER_ADDRESS_MODE_REPEAT; + sampler_info.addressModeW = VK_SAMPLER_ADDRESS_MODE_REPEAT; + sampler_info.anisotropyEnable = VK_TRUE; + sampler_info.maxAnisotropy = get_device().get_gpu().get_properties().limits.maxSamplerAnisotropy; + sampler_info.compareEnable = VK_FALSE; + sampler_info.compareOp = VK_COMPARE_OP_ALWAYS; + sampler_info.borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK; + sampler_info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR; + + VK_CHECK(vkCreateSampler(get_device().get_handle(), &sampler_info, nullptr, &attachment.sampler)); + VkImageMemoryBarrier imgMemBarrier = vkb::initializers::image_memory_barrier(); imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_GENERAL; @@ -1326,10 +1341,10 @@ void SubgroupsOperations::create_fft_normal_map() VkDescriptorImageInfo SubgroupsOperations::create_ia_descriptor(ImageAttachment &attachment) { - VkDescriptorImageInfo image_descriptor{}; - image_descriptor.imageView = attachment.view; - image_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; - image_descriptor.sampler = nullptr; + VkDescriptorImageInfo image_descriptor = {}; + image_descriptor.imageView = attachment.view; + image_descriptor.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + image_descriptor.sampler = attachment.sampler; return image_descriptor; } diff --git a/samples/extensions/subgroups_operations/subgroups_operations.h b/samples/extensions/subgroups_operations/subgroups_operations.h index 37ee6f310..3b59af252 100644 --- a/samples/extensions/subgroups_operations/subgroups_operations.h +++ b/samples/extensions/subgroups_operations/subgroups_operations.h @@ -147,7 +147,7 @@ class SubgroupsOperations : public ApiVulkanSample float length = {1900.0f}; Wind wind; - glm::vec3 light_pos = {100.0f, 15.0f, -1.0f}; + glm::vec3 light_pos = {100.0f, 15.0f, 10.0f}; glm::vec3 light_color = {1.0f, 1.0f, 1.0f}; glm::vec3 ocean_color = {0.0f, 0.2423423f, 0.434335435f}; } ui; @@ -171,8 +171,10 @@ class SubgroupsOperations : public ApiVulkanSample VkDeviceMemory memory; VkImageView view; VkFormat format; + VkSampler sampler; void destroy(VkDevice device) const { + vkDestroySampler(device, sampler, nullptr); vkDestroyImageView(device, view, nullptr); vkDestroyImage(device, image, nullptr); vkFreeMemory(device, memory, nullptr); diff --git a/shaders/subgroups_operations/fft_normal_map.comp b/shaders/subgroups_operations/fft_normal_map.comp index 1965e58e2..97cbdbae9 100644 --- a/shaders/subgroups_operations/fft_normal_map.comp +++ b/shaders/subgroups_operations/fft_normal_map.comp @@ -32,33 +32,23 @@ void main() { uint N = fftUbo.grid_size; ivec2 pixel_pos = ivec2(gl_GlobalInvocationID.xy); - float tex_dim = 1.0f; + const float offset = 1.0f; - vec3 v0 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, 0.0f))).rgb; - vec3 v1 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, 0.0f))).rgb; - vec3 v2 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, tex_dim))).rgb; - vec3 normal = cross(v1 - v0, v2 - v0); + vec3 v0 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, -offset))).rgb; + vec3 v1 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, offset))).rgb; + float v01_length = length(v1 - v0); - // float left_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-tex_dim, 0.0f))).g; - // float right_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, 0.0f))).g; - // float bottom_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, tex_dim))).g; - // float top_y = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0.0f, tex_dim))).g; + vec3 v2 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-offset, 0.0f))).rgb; + vec3 v3 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(offset, 0.0f))).rgb; + float v23_length = length(v3 - v2); - // float z0 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-tex_dim, -tex_dim))).g; - // float z1 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0, -tex_dim))).g; - // float z2 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, -tex_dim))).g; - // float z3 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-tex_dim, 0))).g; - // float z4 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, 0))).g; - // float z5 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(-tex_dim, tex_dim))).g; - // float z6 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(0, tex_dim))).g; - // float z7 = imageLoad(fft_displacement_map, ivec2(pixel_pos + vec2(tex_dim, tex_dim))).g; + vec3 c0 = v0 - v1 / 2.0f * v01_length; + vec3 c1 = v2 - v3 / 2.0f * v23_length; - vec3 result = vec3(0.0f); - // result.z = (bottom_y - top_y) / float(N); - // result.x = (left_y - right_y) / float(N); - // result.z = z0 + 2 * z1 + z2 - z5 - 2 * z6 - z7; - // result.x = z0 + 2 * z3 + z5 - z2 - 2 * z4 - z7; - // result.y = 1.0f / float(N); + // c0 = f(n - 1) - f(n + 1) / 2h; h - length(f(n + 1) - f(n - 1)) // left - right + // c1 = f(m - 1) - f(m + 1) / 2h; h - length(f(m + 1) - f(m - 1)) // top - bottom + // n = cross(c0, c1) - imageStore(fft_normal_map, pixel_pos, vec4(normalize(normal), 1.0f)); + vec3 result = cross(c0, c1); + imageStore(fft_normal_map, pixel_pos, vec4(normalize(result), 1.0f)); } \ No newline at end of file diff --git a/shaders/subgroups_operations/ocean.frag b/shaders/subgroups_operations/ocean.frag index 3e1435db4..ed92b8a1b 100644 --- a/shaders/subgroups_operations/ocean.frag +++ b/shaders/subgroups_operations/ocean.frag @@ -16,20 +16,27 @@ * limitations under the License. */ -// layout (location=1) in vec3 normalVector; -// layout (location=3) in vec3 lightVector; +#define LAMBERT_VAL 0.3183098861837907f layout (location = 0) in vec3 in_pos; -layout (location = 1) in vec3 in_normal; -layout (location = 2) in vec3 eyePosition; +layout (location = 1) in vec2 in_uv; layout (location = 0) out vec4 outFragColor; +layout (binding = 0) uniform Ubo +{ + mat4 projection; + mat4 view; + mat4 model; +} ubo; + layout (binding = 3) uniform CameraPos { vec4 position; } cam; +layout (binding = 4) uniform sampler2D fft_normal_map; + layout (binding = 5) uniform OceanParamsUbo { vec3 light_color; @@ -39,54 +46,205 @@ layout (binding = 5) uniform OceanParamsUbo layout (binding = 6) uniform samplerCube skybox_texture_map; -void main() +struct ReflectedLight +{ + vec3 dir_diffuse; + vec3 indirect_diffuse; + vec3 dir_specular; + vec3 indirect_specular; +}; + +struct Geometry +{ + vec3 position; + vec3 normal; + vec3 view_dir; +}; + +struct PointLight +{ + vec3 position; + vec3 color; + float dist; + float decay; +}; + +struct IncidentLight +{ + vec3 color; + vec3 dir; + bool is_visiable; +}; + +struct Material +{ + vec3 diffuse_color; + vec3 specular_color; + vec3 attenuation_color; + float roughness; + float ior; + float thickness; + float specular_f90; +}; + +float pow2(float x) { return x * x; } +vec3 pow2(vec3 x) { return x * x; } + +vec3 brdf_lambert(vec3 diffuse_color) +{ + return LAMBERT_VAL * diffuse_color; +} + +vec2 dfg_approx(vec3 normal, vec3 view_dir, float roughness) +{ + float dotNV = clamp(dot(normal, view_dir), 0.0f, 1.0f); + vec4 c0 = vec4(-1.0f, -0.0275f, -0.572f, 0.022f); + vec4 c1 = vec4(1.0f, 0.0425f, 1.04f, -0.04f); + vec4 r = roughness * c0 + c1; + float a004 = min(r.x * r.x, exp2(- 9.28 * dotNV)) * r.x + r.y; + vec2 fab = vec2(-1.04f, 1.04f) * a004 + r.zw; + return fab; +} + +float dist_attenuation(float light_dist, float cutoff_dist, float decay_exponent) +{ + if (cutoff_dist > 0.0f && decay_exponent > 0.0f) + { + return pow(clamp(-light_dist / cutoff_dist + 1.0f, 0.0f, 1.0f), decay_exponent); + } + return 1.0f; +} + +vec3 schlick(vec3 f0, float f90, float dotVH) +{ + float fres = exp2((-5.55f * dotVH - 6.98f) * dotVH); + return f0 * (1.0f - fres) + (f90 * fres); +} + +float smith_correlated(float alpha, float dotNL, float dotNV) { - float material_shiness = 27.89f; - vec3 n = normalize(in_normal); - vec3 l = normalize(ocean_ubo.light_position - in_pos); - - float i_diffuse = clamp(dot(n, l), 0, 1); - - vec3 e = normalize(-eyePosition); - vec3 r = reflect(-l, n); - - float i_specular = pow(clamp(dot(e, r), 0.0f, 1.0f), material_shiness); - float lightPower = 1; - - vec3 specular = vec3(0.3f, 0.3f, 0.3f); - float radio = 1.0f / 1.33f; - - vec3 R = refract(eyePosition.xyz, in_normal, radio); - vec3 reflection = texture(skybox_texture_map, R).rgb; - - vec3 color = ocean_ubo.light_color * ocean_ubo.ocean_color * i_diffuse + specular * i_specular; - - outFragColor = vec4(mix(color, reflection, 0.1f), 1.0f); - - // vec3 normal = in_normal; - // vec3 light_pos = ocean_ubo.light_position; - // vec3 light_color = ocean_ubo.light_color; - // vec3 ocean_color = ocean_ubo.ocean_color; - // float ambient_strength = 0.91f; - // - // vec3 ambient = ambient_strength * light_color; - // - // vec3 light_dir = normalize(light_pos - in_pos); - // float diff = max(dot(normal, light_dir), 0.0f); - // vec3 diffuse = diff * light_color; - // - // float specular_strength = 0.5f; - // vec3 view_dir = normalize(vec3(cam.position.xyz - in_pos)); - // vec3 ref_dir = reflect(-light_dir, normal); - // float spec = pow(max(dot(view_dir, ref_dir), 0.0f), 32); - // vec3 specular = specular_strength * spec * light_color; - // - // vec3 result = texture(skybox_texture_map, ref_dir).rgb * (ambient + diffuse + specular) * ocean_color; - // result = result / (result + vec3(1.0f)); - // - // // gamma correction - // result = pow(result, vec3(1.0f / 2.2f)); - // - // outFragColor = vec4(result, 1.0f); + float a2 = pow2(alpha); + float gv = dotNL * sqrt(a2 + (1.0f - a2) * pow2(dotNV)); + float gl = dotNV * sqrt(a2 + (1.0f - a2) * pow2(dotNL)); + return 0.5f / max(gv + gl, 1e-6); +} + +float ggx(float alpha, float dotNH) +{ + float a2 = pow2(alpha); + float den = pow2(dotNH) * (a2 - 1.0f) + 1.0f; + return LAMBERT_VAL * a2 / pow2(den); +} + +vec3 brdf_ggx(vec3 light_dir, vec3 view_dir, vec3 normal, vec3 f0, float f90, float roughness) +{ + float alpha = pow2(roughness); + vec3 half_dir = normalize(light_dir + view_dir); + float dotNL = clamp(dot(normal, light_dir), 0.0f, 1.0f); + float dotNV = clamp(dot(normal, view_dir), 0.0f, 1.0f); + float dotNH = clamp(dot(normal, half_dir), 0.0f, 1.0f); + float dotVH = clamp(dot(view_dir, half_dir), 0.0f, 1.0f); + + vec3 F = schlick(f0, f90, dotVH); + float D = smith_correlated(alpha, dotNL, dotNV); + float V = ggx(alpha, dotNH); + return F * (D * V); +} + + +IncidentLight get_point_light_info(PointLight pl, Geometry gem) +{ + vec3 len_vec = pl.position - gem.position; + float light_dist = length(len_vec); + IncidentLight result; + result.dir = normalize(len_vec); + result.color = pl.color; + result.color *= dist_attenuation(light_dist, pl.dist, pl.decay); + result.is_visiable = bool(result.color != vec3(0.0f)); + return result; +} + +ReflectedLight direct_physical(IncidentLight il, Geometry gem, Material mat) +{ + float dotNL = clamp(dot(gem.normal, il.dir), 0.0f, 1.0f); + vec3 irradiance = dotNL * il.color; + ReflectedLight result; + result.dir_specular += irradiance * brdf_ggx(il.dir, gem.view_dir, gem.normal, mat.specular_color, mat.specular_f90, mat.roughness); + result.dir_diffuse += irradiance * brdf_lambert(mat.diffuse_color); + return result; +} + +void main() +{ + vec3 total_emissive = vec3(0.3f, 0.3f, 0.3f); + ReflectedLight ref_light = ReflectedLight(vec3(1.0f), vec3(1.0f), vec3(1.0f), vec3(1.0f)); + float metalness = 0.91f; + float roughness = 0.91f; + float ior = 0.5f; + vec3 specular_color = vec3(1.0f); + float specular_intensity = 1.0f; + + vec4 diffuse_color = vec4(ocean_ubo.ocean_color.rgb, 0.5f); + vec3 normal = normalize(texture(fft_normal_map, in_uv).rgb); + + Material mat; + mat.diffuse_color = diffuse_color.rgb * (1.0f - metalness); + vec3 dxy = max(abs(dFdx(normal)), abs(dFdy(normal))); + float gem_roughness = max(max(dxy.x, dxy.y), dxy.z); + + mat.roughness = max(roughness, 0.0525f); + mat.roughness += gem_roughness; + mat.roughness = min(mat.roughness, 1.0f); + mat.ior = ior; + mat.specular_f90 = mix(specular_intensity, 1.0f, metalness); + mat.specular_color = mix(min(pow2((mat.ior - 1.0f) / (mat.ior + 1.0f)) * specular_color, vec3(1.0f)) * specular_intensity, diffuse_color.rgb, metalness); + + Geometry geom; + geom.normal = normal; + geom.position = -in_pos; + geom.view_dir = normalize(cam.position.xyz); + + PointLight point_light; + point_light.position = ocean_ubo.light_position; + point_light.color = ocean_ubo.light_color; + point_light.dist = length(ocean_ubo.light_position - in_pos); + point_light.decay = 2.0f; + + IncidentLight dir_light = get_point_light_info(point_light, geom); + + ref_light = direct_physical(dir_light, geom, mat); + + vec3 irradiance = ocean_ubo.ocean_color; + vec3 ibl_irradiance = vec3(1.0f);// envinroment color value - TODO + + vec3 radiance = vec3(1.0f); + // radiance += get_ibl_radiance(geom.view_dir, geom.normal, mat.roughness); // envinroment color value - TODO + + ref_light.indirect_diffuse += irradiance * brdf_lambert(mat.diffuse_color); + + vec3 single_scattering = vec3(0.0f); + vec3 multi_scattering = vec3(0.0f); + vec3 cos_weighted_irradiance = irradiance * LAMBERT_VAL; + + vec2 fab = dfg_approx(geom.normal, geom.view_dir, roughness); + vec3 FssEss = mat.specular_color + fab.x + mat.specular_f90 * fab.y; + float Ess = fab.x + fab.y; + float Ems = 1.0f - Ess; + vec3 f_avg = mat.specular_color * (1.0f - mat.specular_color) * 0.0476f; + vec3 f_ms = FssEss * f_avg / (1.0f - Ems * f_avg); + single_scattering += FssEss; + multi_scattering += f_ms * Ems; + + vec3 total_scattering = single_scattering + multi_scattering; + vec3 scattering_diffiuse = mat.diffuse_color * (1.0f - max(max(total_scattering.r, total_scattering.g), total_scattering.b)); + ref_light.indirect_specular += radiance * single_scattering; + ref_light.indirect_specular += multi_scattering * cos_weighted_irradiance; + ref_light.indirect_diffuse += scattering_diffiuse * cos_weighted_irradiance; + + + vec3 total_diffuse = ref_light.dir_diffuse + ref_light.indirect_diffuse; + vec3 total_specular = ref_light.dir_specular + ref_light.indirect_specular; + + outFragColor = vec4(total_diffuse + total_specular + total_emissive, diffuse_color.a); } diff --git a/shaders/subgroups_operations/ocean.tese b/shaders/subgroups_operations/ocean.tese index 084c3d5e2..16ad12ba4 100644 --- a/shaders/subgroups_operations/ocean.tese +++ b/shaders/subgroups_operations/ocean.tese @@ -20,12 +20,8 @@ layout (triangles, equal_spacing, ccw) in; layout (location = 0) in vec2 inUv[]; -// layout (location=1) out vec3 outNormal; -layout (location = 2) out vec3 outEyePosition; -// layout (location=3) out vec3 outLightVec; - layout (location = 0) out vec3 outPos; -layout (location = 1) out vec3 outNormal; +layout (location = 1) out vec2 outUV; layout (binding = 0) uniform Ubo { @@ -42,8 +38,6 @@ layout (binding = 2) uniform TessellationParams float displacement_scale; } tessParams; -layout (binding = 4, rgba32f) uniform image2D fft_height_map; - vec2 interpolate_2d(vec2 v0, vec2 v1, vec2 v2) { return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2; @@ -68,24 +62,12 @@ void main() fft_texel_at_vertex[1] = imageLoad(fft_displacement_map, ivec2(inUv[1])); fft_texel_at_vertex[2] = imageLoad(fft_displacement_map, ivec2(inUv[2])); - vec4 height_texel_at_vertex[3]; - height_texel_at_vertex[0] = imageLoad(fft_height_map, ivec2(inUv[0])); - height_texel_at_vertex[1] = imageLoad(fft_height_map, ivec2(inUv[1])); - height_texel_at_vertex[2] = imageLoad(fft_height_map, ivec2(inUv[2])); - vec4 fft_texel = interpolate_4d(fft_texel_at_vertex[0], fft_texel_at_vertex[1], fft_texel_at_vertex[2]); - vec4 height_texel = interpolate_4d(height_texel_at_vertex[0], height_texel_at_vertex[1], height_texel_at_vertex[2]); world_pos.y += fft_texel.y * tessParams.displacement_scale; world_pos.x -= fft_texel.x * tessParams.choppines; world_pos.z -= fft_texel.z * tessParams.choppines; - outNormal = height_texel.xyz; - + outUV = interpolate_2d(inUv[0], inUv[1], inUv[2]) / 256; gl_Position = ubo.projection * ubo.view * ubo.model * vec4(world_pos, 1.0f); - - outEyePosition = vec4(ubo.view * ubo.model * vec4(world_pos, 1.0f)).xyz; - // outNormal = vec4(ubo.view * ubo.model * vec4(height_texel.xyz, 0.0f)).xyz; - // vec3 lightPos = vec3(0.0f, -5.0f, 5.0f); - // outLightVec = vec4(ubo.view * vec4(lightPos, 1.0f)).xyz - outEyePosition; } From a9ca1fb41bb597134d05d30f466e91feb3ede001 Mon Sep 17 00:00:00 2001 From: Patryk-Jastrzebski-Mobica Date: Mon, 20 Nov 2023 09:32:58 +0100 Subject: [PATCH 52/53] Add pipeline barriers, update README file --- .../subgroups_operations/README.adoc | 30 ++-- .../subgroups_operations/image/image.png | Bin 0 -> 2051725 bytes .../subgroups_operations.cpp | 137 ++++++++++++++++- shaders/subgroups_operations/ocean.frag | 145 ++++++++++-------- 4 files changed, 221 insertions(+), 91 deletions(-) create mode 100644 samples/extensions/subgroups_operations/image/image.png diff --git a/samples/extensions/subgroups_operations/README.adoc b/samples/extensions/subgroups_operations/README.adoc index 0c5fabba5..28ee3fff0 100644 --- a/samples/extensions/subgroups_operations/README.adoc +++ b/samples/extensions/subgroups_operations/README.adoc @@ -1,38 +1,25 @@ - + # Subgroups Operations - ## Overview This sample demonstrates how to use and enable the Subgroups operations feature. - +image:image/image.png[] ## GLSL Shaders + In shader enable `GL_KHR_shader_subgroup_basic` extension. TODO: add more description - ## Enabling the Feature To enable the subgroups feature in the Vulkan API you must create a Vulkan instance with a minimum version of version 1.1. Enable the extension `VK_EXT_subgroup_size_control` and get the subgroup properties of a physical device + ```C++ VkPhysicalDeviceSubgroupProperties subgroups_properties; subgroups_properties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; @@ -43,6 +30,7 @@ device_properties2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVI device_properties2.pNext = &subgroups_properties; vkGetPhysicalDeviceProperties2(gpu.get_handle(), &device_properties2); ``` + TODO: add more description ## Additional information diff --git a/samples/extensions/subgroups_operations/image/image.png b/samples/extensions/subgroups_operations/image/image.png new file mode 100644 index 0000000000000000000000000000000000000000..0c649ff131a6ba066ecf54fef7d30bb8a3bc01b8 GIT binary patch literal 2051725 zcmagF2Ut_V)-H?{6(J%jA|(jYJ1C(hA}UC4(yM@ol+YoNqS6E@B3+~kBE3m30qKOI zAksUb*CY^92$yrtz4yQO{NMTR^JM4Qv$EIB>@_p%UGJJngr1HX#gI<(@qxV zW8pgr&wUFwz7!fdZH%z^s1luDy?k{s)w8Gh>h&jAJEl7x>)m_A@s8F0K9dyauFmC% zcfT#$j-*Jo^VXN&=deD$EI3b>!~8M3QFjOt_Ib6?kVrf}&T!g3I6*_xz3()*sUGek zGMiVbGY*Q^&4RX@Pt?F0_F&qM^ulYx{XMf-ONwr{iuvprQSa&KHdLBX3$z#T2cilO&4QSG>fv1T2@0_JNh*gX6HQ2*cc z{O=o%nAMCRcT%WLK-C&+(#ft3We}U{lwl@r-C&tLlKsa%R=EK_xC;M^yLr+EHM1u5 znaFwk-+lVub^Ysxcyl*!tT7o)NIuEOhNF&vqPmb{f){b8k^?cpB60Gy>Fz-ha&%P; zHBUUDVf#}@ZX*8s3OT=k5Q9#rLtIjAizBRCadv%(Jkt$T-|uai!vU%EH{JeQeH#Vj zrU?>@TaGWYu|Cc&v)&2XYon*K8%zmM{y)q9+gqF(y$Q096UZ=aSz|!k%6l%-uq~8W zpp7-LVeI*@|7%eGU3C}IbXZ+x6~n3AFztVGOq3`Knh8%P{y%h%^ftyTusy4&HrAQr z80bk679mF)1W;$N%DXV#dTWJ@fW-)`$#LJ1#ZKdH3vS|nPKkeS{hu=o^#G1FL5ylD zCXg-Wi2lg{${s5+ZKEfSS9WfH$0@1N-_5Gj67EOZ(mhGQ#;I{dbpLnVJHcc#<+beX z_`@kD=GNg$-L&7a5}716bdIG2Cxk#mIgrW$h*4amNmCB|zp*KjG_QtSR!hpHt!G;A z9I`70&F5u+TK1zc`6zOJ@(`NqC}#I423rNDR3)3tE?r%<+C>s^?GS=027LfMw!q>w z+f8USW#%zp(lR@3Eg3~hP8?eZAO_7gPS<c7}~L&Q`;Ri0~m?1&~s46GuCjzSh{X z*_Z@8*$0C3XX)<%d&o^U;N_STY}k$}M}LOA_cu7`zZ$aB(Uf3?&k+%GI6fO3A4UL$ z6Oi~vHK;e>MPBd=(%cF01VQT1rFLlSm(g><2*0tr$1~_grNg5oOlq??2UG4P%9o+z z>2r2yQz~@sUO^ijEwdJhe(ofu>#cHxeZH7ttciJ7b zU9@3MP;mQq|E48Q?3KC*6mI!h<{hMD+d}_ogSw(Nq(mEM@=posPb`qujxs%UD%0#7 zDY9$bgiMnbTq3sWU;gR;_hv}>x1e0Efcdvr8f6!f5MZ}p4b?|Twy_&z5=BsXfK0iN z*Mzq>?1+IMHdsF-`w05t|GK3~1Z{PAEz_njp;N5M`DKvCrq#)jLrA3nCO+2EhAV*j zx_&2G_xO*lRYuTGz1IaOV$&kIn=7c;5~yTZ=V7;6TP|nxX72y=od4D{8>Vqc3i%abv@^_yy610oh4N4ZsgnPj=c1~w5hKM>LDsKHLFr)|Hd{3cA?a`}Q%LELR zjyyVcmZfF}ZWLgF)L>LObv*VsWyD}-j%dY$%Cg#Rwb=`)fUOP9sqsR97Q~vQ` z(yLwyoX8&xG_k7oUx;i>#?qk^ui{B2DK&F~WVz7qU29e=<{O+z4}|aiqCR8T+YG?3 zUfUh@4^Z6;N3Mm7Qc7*Yx5u#Mva|oeH32#HMjwDj{0VEnaq^cIshfe&yp<`}8n;S`Nq`;qV6jI97MT6`+9Z1cck%8gT~>=e#g4Eg4A=*TTNq2+3~jSh zLyL`O?oc|B8(UD?0J4UG`iNT9Bxh24GmAeuk|r{bC$M<%E}mKs+4{3x?)8#O!5u#~Pxn5+>Q=bykqSrB zj}wOGwu6=qw=MkP5N!e?1HI-)y%a?Ft)@237V+`HD&%I-y?rI`qJ1Z~7oOMexoKW@P3GDRYm`@oDnw{GySqmKa6 zM=beZvj%t(`6VitZub6Xzyk#vCg+zPDXniWJeL0w^X~Z6F7Gv)=V*tuQ#11)&rbN5 zA$p!z9}UqfA=E2l+ZS7I?{u<kBJ$XFNPxKZ8j>2Eb}e-ZA}lzg@?JoX~?rxVcl1r*l2 zCfCsgJ6koTPdghy)`1B>+WN; zc6-{xT?s<|*2{kA9TmKV0-g0rdnKYt!qeu`z?8)En5OE34`-k1He%NlH+gEpJ-Sjx`1Ex5b)+X@9!MA^JEHcrZT$MI&Kj*7Dw|UIfS2 zoUDW0$wZ5L5;$ap~XN&w=L3Ku@jK;pvkh~qh?rj>JvD^-BPS(8@X)j4`E ze3~*{{a$K)(A)MaOeF#5!fRhyueH92VX#)OAudJ-7tpjgxo4V}CvmBGeu<6quQrh~ zS5J=N3Y-Ty{iO0wwGndnn#L?l_B50mndszkhx);D?yr9l=QkxYY%U@6YJk8Au0!Jy zWYA9_bcoY$&G+`&(z4W;%YQw!W#Jy4%RO<}u4s0w!lfM)%c>1%3#k48tVeHnGKfBg zN~mSRaf~Pe22I8EZF(pj)cA{M9tr^);XjKY4x|J?&@ryJ4AGA31s>{S(yZ}un5Ecs zGo3GT)nCAfKzUExve$A8FF4^*>n@xalpH>k;W^Im7LPEW4aJ>R<&^=NmiW+;e zk6VJ!mN5HE-N~BC;?cmf?EnfElZgv&)T*PIN^`PxE8JIy*jDz@o+@Sh#y=s zKl2RmAAs)E(sK^J&x)!TAhr;jF#y^;U=j^5n?Ekd&!{}EE<=GbJ^kyi?&+>P-N9+1iN+17N(_t9l5iL+-U{m zdD>P2S*JB?Bswhc#N)Y>W8Rr`MgOOq`)Qcn!&cy;?YazjNk-U*4Lh_wLiz|g_@=}0 z4Y90-%mvt=Ml3Cug}noj2ZChAQbg0E;1zo0JpI!N4kN@*0rWyA3?_O* z#_Zt2yQhJDxSGP`yB`#{Aj;u((4qF^Rwgwb^#X>bVkPFBb&*rWST#;0{?sw+;QM8U zQpmPmeyCM{Wd78<`-EEE6N0Y)m;beR|1I`B4bLTjdyW!23%KhY;NuUdeF=COkrkg; z2e?Q3OGNIT`glscBE3PhOunu}tTOHBUm>CzilCOfE5Uez1;N|BmhtF$hFxTL?MWHd zSB^%WqfUW8*0G)j&tMk%8>9rAy$pY}6N#W36c6o(nzmWohet+(Y-mGFvxDS3WQ#ix z_+H7$qo@7+FZzf7nM46ZTX40fp*96RS4K+Atb!l8ST~=}ftC%uz=0y03V4r5rl~dM zZrSwy>cfrfaIQj2S*q7LsLDaw3!o@`suqz3sCcQcH3Iu@+7P!MPBUPiMIS*<7cd9; zKDoAOb}lPwTf5*tn6P{y84zIjS}|erKVb1cX~LOKO&Wae->mXooe=*qq!hIjTvHySdv)fPbUsU3SJah)FOxO~vjK?_`IKaq#mg=8 z`)}!W!~}HHEp>ht{oFy@P+5nFhOGNq>7T;NmoQS%GI*Y!K31=bTU9r_rw;0oOI=>u zlfJ*Jv9D82N-ZB8eDP581@m(Wot$^`DoDoGl1 zk1Ms(gqf6C8%~TRvyVYj+#><_@AX%_TcfXpIp(J~W;BL?vOSmVRDI>V&MQchFW|59 zC*82#@!_C=`OF zVafUhtV$Fc#+biyzi$7iZZ0pigXy55K*_yuHbtOe#vwxi=0&g@_T#+U z2g#Ow@{MpHV12NGb<}NoAx}A)z>-H5F{jY5gDrs~($=>7h-5EKAAHw?e8mE|?)&mN zJ+uzcgkyAdkh7OFRr9Y}tn{G_jcqOp&vFMJwMx)@W@Oh>&6kEuk?qda8H3K6mKI&b z4Oo7H0~Neq!q&1!FsUQ?R%3+pKR~YiuS&k(9AaEAbS?Vs2Wk5lBD(MonU51_f&0TZ zhdB4gsNcZLzO$PYW=h-0gWhy0T_ge4b!?UAnR&CUJI2|1e>x7lv%gRe+WwVA+fF`! zcnh3SR|E5MBF6DQ*8Y|d)}w^5`A6MBTeibQ!8D{m#;3Hb?|^i9D_oF=e9&Pw*6}p) zq(tuF#k4yO6L@OC%#1;T~|VzSGWm{ zRf4~S9o?;-NTpA|$TqAiDPdJ#Y1sgGlrJN_m9=GR{%I)&n>FS%9GepNP6SrHagq6@ zG_@n=W!smYVhHcHQ2oo@`>K-Cz>?SQ-)@YW;&@ZF`eZ1M{MBl@+Y!9qfk`?|SnwtR zg0@|fO`12JV}fG~(C+_mZ|I8wV-`aayRhA$B>$GBZH&bnbpu zlH7}?PHL6AMT{ZaIj^DN;hFg+99m`L_EwysF}qraD+_;7zxj3`*#{MbLc`&kIxb=% zTnV#;wRWfKm+7N{M;{N8r_85JN8Edx?j6rlJ2f%h=B&TZ=#=YzZ1IGU8NC}X#Y_Km z=|fWYyOU(hopwn9B@Tg<38!SI445B`Jh7VOk0!2TId=nV|3VyMz(L)C#)Pi5#vZS= zt7}h%S*<#iUTqV}5;=oOs>~_ajR$@38u52WJLK8OG3QN=VUP<#?L-BrU`*mwb0D?` z=QoupMmF_Y!t|{A)Id*Bs;ls8@{sR{ze3w*_!Ku2KUMh(Y_^#3#(sA*qKjm#=L`wB zZip!vUT>OfEIm#>s>%=mQG<5*tb)ne0$ia9PKP{kK!=1kzaY(JZ@yRI8jv>|UN&E7 zzQ%$$JUZTDi3>fPEu6bNy4lfdSPE*;AS6BsGT(MIlJ(Kp=laOyx6w9|H1~X6#YDo! zywtkkhZ1UY#p|&%xA~R9@?N!r#tO@%mXN^^wqxMT2ie|Zx{de6Rjk9a8Ojx#aSd6z zccZviozJbW^9(n1_?*pmHp|&1(jt|wR#u6FmT$XDt;dpU*S)SiOl#;fH{HG+q5Eez zZQ8#wKQg1Nf)AhT&_&~iYpy1zdq#0`R$*IHUU8z&Hqd#_2Ys{|hpLELHNi8mmF2iY2mlL=fqPh%Lt}U zD4|NT?O#ERT@@5s$A{IJ4YjDdWGVYd@<@KuoPm-C#PbtOo){|l0dWO?!Mpn$Vbt)o z)6M{gJ%ls;v)?*R3$?yKK@3d+mi}#7+!#G*IbVq`MR3^dEe4t8Ox3o7uC}pAtKOoO z(biiwkQrWLRgW+8^nFfVkU<0_<X+kLfTa$>jbY10tbvh4fzfl6ht zZmn|D@cZDml_~kDQsqMuV=qjv(O1ckUc_J`2NiJ=Rozl3hiID|_HSRewwnBcHz6?iqTM zFTWvoS4JZI`uJN+Yqt*Oh<#5_aMT-+`Yl$Tz~S-xg&sKfDnaUZ*RB}SEAM3ZP`*!1 zs$UmJ!n??m#jgPv#Ko1q17t)*z+>X`XX(4vW*H_g+2p^i?cijfFij(Qp6k6Ig6Uz5 zuMHUqQu9x+LyV3RdRn%L-8u`tsfCSUVwshLoTV+YLc%Vt68~^l)*FE!4 zQp4xyh0-HlF_a?|k`~dNQz-x1^q^+haV}qQM&|i-@z8|XZ?r;c`?~Gz45`UC&_vg#rxuCd`Cb^8ikR{auRN;M&^mdpKir z+}F$s=WW_l_;dMkjk)bKoT^#+%6N|*wA6pr()j0lr-?S5+pLnJqY{Q7k@i9s`&jS* ze@sJx!C;bKwm1h5hvKZ#Cm7S+Ft~bmfZ!uGI(e$F7w1TF_VO8?M!OV9oMU6DUHw89 zeoQLY-pm8~19k4;xE1?0XvtDPl0wr+Dab_R1VYB!UBs=Fui<^yE$`cD zkhFdmYu+5SgOt86GO~-{4ur zMUy40Dv@AFPkV=Pl?BhOMb^1Qb;A}CKneIF$)?dB+c4>L)~&_i&lTIRWATYigZG*3 z{eT+jV^@slc?U#a3Nv#8l%G@1u3bPAa}MeNY4DM_YPRG4Kb!Th%%={3H=xCtssbEj zs84)_vEkc7wZh+t1vT#wC;HLFW(ji&v+3d{Y#%{H<8ieYi!n%^h-+oTb`867W*B`V z&+Xl!54xm^*7LK(Ry(`&5%3PPe++nn_pR-oG#yHBk_X_B*kT=MR7Og_7S!FGcYKf1 z$3AIqjrMb__dDqGvhrD)9R28X?q|d*k{gjtdFZo6H&M^{=kHqf)`#)%ub?xFJSQCQ zi%G5Q$Ft$d&Cu(Pb7ce#O|uSSOB0hD9HQb&KuC<`k+Ug}$4p*T40q97g7mb-d&;(U z=x@>o#cFQ8+i4_hZXE~gmF`+RR*P09N*0Q%jQ~y32R6kQK+EQPH8{zP>!S@ zF6>W|f(6<*D7FVEWdD<1cshS0SdbXBbNuS6 z;wFimIPZ-XY1u3d{ef zC=s{@s4}#Thp(elSC+VfU+S|)N^nu3yoYRvlzOUJZchg1q=48*Bq zFV|hSu88%d*Jm9)`6X$Hb;>PHxvRu9$(d^E*h1HXT6C*zVrx9p?#tcQjJawf(KDd@ zkWHq9yVgkAmq=?MX92a6C%hLG{No;8PE6dibd%g+wv%41O{b`Zut0mBb!1KyD6|DL z9=&>v)G(21V_mZ48{*?)mkD(mjtb%ydZsr)NL)#mazLu8CjZRzz>9oT8}L?CGMYfX zZ|GW0Q7!uORToSOgOCf8oBQ2^7-fF$M*4>aNlP-XJZhA_r}^1|nWg{mZ479_>DM*q zYnK#*g&Dtm9-99q>u7JI?G*DBH&kugy^IE`?by+KZFP)azAF4dS^&&VvoHRh6_#$8 zTdjkIq&|*(o=gS4>v^R=8u48g(K5ya(Hq*@M?Oj$kh=8x4l`%AeF?v4NI7W1 z_$22fv7TMLkmF^@E+HLp?d0K|D?d&4dq@1#G}w;vHwO+hC!zos(_kHCU6C-CD)nTI zJKhAR)Y~%G(|x3T#bnAy;X)-#9T2v&O7=1EBN4MeR}-G_iHslE=9POOzoz(P?>mR3 zWlQ6+X}t^DIs#7%CUc`;8g(;68XPv`i$XyUL+<{j&&H)=AF=|7E0MN&X&Lbf5Cxgfw)hT3%Cz6P9^euQq(F zBslfk6hn#fly(=U~09L zJ1p)Dvd6IRmAeS0RD?sPuKneFaD{>!_N}0{JS6jg>6P3?uC7A{QQsu z=fhuz@p<3O(lW`*LPp{>Q}-UVr2FQdOS-(2{DCFw_VW5^i-Mz-p985-VOY}K04V?r5f4SI?@N} z@T=1N`@>ed!?x2uv;A;&tdQ!4%0-+d#4<^**p(Ramw5huW^S^0CVqN+1w&^IXy(r# zD6BWrZGIMn3S-+Ke}y)CtCPac#iUCCyAAnJuF)m}tnx{y`Dq(pC3c&NL z%~6N$G+jpuzbs*I0D=Z$%X~K!etuX=5@b%zTmP}|@+-{Eex`ab#J%I3X6K64%OXI=h&zR6J&cY-*0S{LWf=H;?!*5!6tz z$xDhao6^K#f}>M-4l>u$reC$JgVe-rn&xYBOP!l$430QM^(7#pM({@oz3eQxd{DSE zre*a4@Nq6`S@Kz6VxNcdT6dBvljd)g;t>-?Q{vaFoqQ$GzutO%+8*zfK88{r;$BO8 z^FadMM8meK1KcEjG%u;T`l*=__o)a-|SYfxzo3~53V&Fu?@|F43))yZXs_w>%L6#-j$zD zdT4iRvV3`acI$)PS)8&UgvuE;W0bJ@~9 zM4H<9WfLwY0DWY&68$(e8=l%m51vI4Sh)=7agW8rJ~*3&51U}nOMBtZfY;lW0`%X} zKFF`EY+0q7KY_J4x{d{4RIhM+bG+&_&07@(VWEC*qXYl6(Gqoog0pX4&Xii*(OIupJfb9cPb0wUSx0n7O;vlkU~#ry1uBD||q^AbY^N8~7K zeCOtjxwuZ5L3@mJ7iMn9 zjcA*E6mC_G>st%Dh5)w1Q3XGOfXsm&7 zb7rV4gJ)LVk5?e6%4R;F#V=?EtieB&MatNByU{b7%pLFPOV|=)^7`v< zanc=b^BmnTBe_5<7!J$08$AaXh?U7~Lfh{o!#AaI-G4JY8BbIz;;;TmFo`!{@o4ZLLEu z1W{dDfYvtqo`QZ0=?9sFeen+YpIXDA9$p2uS%Nz4Xx!eiO2(B~#X|=Z*j{&?Bud+B z3>fEXT$wJycI~yjTkQGXj_uZI%jfGW(Gg>yq^VLy(=p|WcrI{?q#~eyJip1aP&~=( z=0h{BnDo`FtIV2PgE`wzda@j@L$-ccQD<*iUj~PR-I1LC0|^zQk*gW=vq8CDL4PG}dcvUF^SW z-2X_kzSHe1rnYmIKI_ZWCx-s*Gnp}~^WSj1n{m2Q<)sk6Z!v}JE;qRExIEQcdX$y1 zP%W#Kq)(o+I*>}f7gWnm;;C|#uKGU;|A6;Frq&OR=NB`t)q5-~dg(8- z^Z5TLa8gOmiQPh+dlZuH_at>S0wlWy`}{fD3dmh>wc0UZcg0CL!RsW?{NXKFS}cFp z3#Pcr)HF&l=v_-hg7yoj<+JccOhUM{Nv8&~rD9%5KHv&k(3TKE7!^D=excWKBV$0g zcgBT-_2oiv4KGv2(`@{UjE*u`&VYfLS61b?zsk&|a(JYmfc>M&7nVIbGy)?IBthOq zE{R?dQ_D|3XB0NK_3lQOVZN(u{h98SjB^@Ra`hTvJw=xu?;B23HXyJ(L zxBa#!q_&?--QxyR0qBlL@b)O)Z>{hy0r+jrX?V~;cyk}^U%1o}>I9X5(n4)aOFg(d zEjW-C!g%+x2j}E2w;QPVl?>j*#<6)ks2KfmdrQu*4gR_+XZ#oTq1n6DrJUy{j@nVy z6M>h#J<-l>+NyIQC8|P;KXyXo-F+f|y)E`1HNV68#HFln#qNeX`Kv@l=`4!e_TtcE zIxZf^}|^&wFvwZdM#tTfgTBf=c@+ z`!Hh#Gy`VC`{E@1nDXXXNv7e~%uKRSk)ND)BRT|Na&^K=i7H&$CGmtdBh;NJNnB0L za~N5#a5*;DeJ;tmopG0a?3hiR>5IGdYPkT^#gdq;dVlcG4me>rfH%Vzn#ht~`S~#^ z2o+kY(fA#{5#p|%E-4f*%VRy4LJYR=6e0b3D*v%hhq>V-GT%c7J%>J~PGk*)K&H2(Sf`GrYom(-n{sa5Ng;g8`_J<119 zG7#JcZvbEA?rrEEI$Lkj#LZH5pEH?x1$_820kkfXSUM7smyT=fE+n(K1xKl+>4Y1A zqZao*)SEuG**?cfBrdXgu^X1+LX>p9Z-1*idZdfgcoU2ac!it~tM(A@w?VmG5bp3)q3>&EfU8Tf=WV`Oir$d|y}_Me zc;yVmjyOS(2nexQJ!ji>5gs}@lu~@IL!?N|@HGE(qi(atWb06*Ho9_)(Cn4s6qj12 zpym54%l_iBVR?1mT=r*>pU~w`VGOsEZr-52Q?DzNxaDhG%YlDk2=9D>Ub(3oF1L3< zQmulow0bbLd*npYg{wo{JH+!O)a&hr18HoWaUp5lPpFptd;dN5wc`1F*m||nQnnD* ziIK_X{QOPhnSpsn^@Ne!h{`_tU2TFbGbzHFZ4K>Uw- zLnW&l{$lnk(dDGB%DI$By^GpHsYWfV*FqA6DZ=&x4Az0!xTjN*&;*-X%!^^+8bP%( zA1r5Vrm{ac#>`1_zTri_(}=ks*Z4s9m$T@(G(jrr3o=@C4>v956MTv?+&(Nwkfg%Cs~RMcU*vFyDQ6vsvkk)q$~DaMy~ zQfs4F1e#{+0W2j1AfJjPVEMWNc)wV(N@V&p>+QiGN zP(euI4D^pw*jg~{aoFT2XyN`~42NQiX!kwEje-W!y6SSav}6JMVbg_+Q0m`X_`pew z(r2JU%`4SZc5m0kIQ9%aMvO8@*Nb}%Hol$UM>OZpiS99|(I1mG)*C7muEd~VTXv8V z`%%Z-l;PT|fX@;3GOYQnv$TMYn0B5Dj+i1~RmME6Cl;Q*^v}I@dKM^ZGbpLH2|Iup zRruHd6G7WJ6q5Y&`kgGV8}B^&c_>ixnN>#W?+=hDi-JJ^1CRb9nMviFJMyM3<8}(3 zn`&!(Yggfs$?U~S5kuN~e^s@9R?|5S@!SbF7?V$Eu(yixy*0zDpkdmUc_5zI5NKPk z8&n$fXmcu>fi-D0LveM8f5l(7RLtaF&?e6sYn~xtwab*<3Nlu5==bN@ru@OWDv!@y z2R+6G*(xd-5EgwY=FmkE`vf(ltzXuDUXalx88!7}Qz6_?oMA#kt%9r7y!8I`t0%WY zzPcQh2m^WkoH+~sP&uS$6`8+wbNJ@x&jW0WYFay78i};9-e57K_X_q3Hj|f|pTN2@ zxyN4TU&UB8sg)UK^q`i@WCa_s6M@%`HGFZ5PRt5Y^jhFI4Sw1yjAm?6n z+K8ts)>;}j+BCwQ>$Ck7k3fX6LSU$iZi0`%HL? zTeLWZ#bdxLR_%AlkKY^e$>r=1>|+D>T6w||V2;m#h{q&lxdTp&q8I>N2GL&b#fXDD#9vm76Eswpwud)HjdBp@1PbR!!y|OHv zI}e#Do!x{8T<1aaKBsrsTZAcvzl-v%1(=0Bu;H1ihQ?3?^d7$)5vRog;e`Md1b3)t zBLG%v%yFeJz+YpSB4C~Hy7OJ^CQxNe{<)vNlE4d6j?HowpJb6l2%8s ztE}oD&A+Y6KVTxOY-5)>*_H4a)YHXbw_wA+E!CyoIACbFZ{Kn#v=kl??DQRYwPpGu+9c(M+xA=7~ZSm~u;D zo~-x&XbShR{l4YhmeM9nW#gQ6PIs=477DcBG-x~I8}c)`hc$*Rwu;j3HKuIOKJfK4 zK9rGHv@`W94$lsD8aYh&Y4`7jv}~ihgi2pXob26#M8z?+1}Ui}w9jWXY>oLYuQt8l zFQyX9zd`O@95DKe;MlNX%e_S|dNJl_)>|PdqVzJ)9h7_|SKcHd`chN{Cnkq%-0aGn z-Wlp_ZmbqJPPqL&U~TToS!YYzM3*XliT57SY~k?}3@--S#T8xoI?kcrIT&Yh2~uWK z0ZsL}Ts3JagW7a3zeahTU1iYN^U%dKL+xq6p{dE7-$mHI7*cX16;RiNk-htxV z{7NL#yJ8y`+89D_F?LH>8|X0In4f_QD_1h0J8N-V@CbpNhXce9cir+u))#MTDh|U# z0}Q&EYV`+ifG%)+x>X14SwE6pxaqQZPn61%`~(7vzFlvu5FYU~NOAh|)mp3GkHHt0 zLl>tIv7HK6wj1hqROz5GdhfBQCz0eQZb`75tDzYIvmc9&zX+s>rzY$SUD;E1cY~Yd zF60AvU#0DMyIw1K-^iGlq|h^m_&v=~sV9MlW=z1p8ih#U(kU*G)N!ar927_g!@tjb ze_u>dyyUxsstx-`gSsa91W1~YLx`ocnsCL|tXz+n%j6q!0n;8^p5S(z64uy~Xt!tp z3C)WsP;Dpmep%AWcDb|6--w55bXv;p$h+mQS6p^o`_!Jh*Y@-?V-_fNa+w!RdHh;3 zUnlcIp>cG|)LU zDbKh6IMF}XwGlC}8}pU>OVx;JxEdiWDVy5v5EQJ$A1BU=bP|ux)Qs{RYkbW=egziZ=5o(l71ei-gmd5d^%~MTiHudX>?2fx^0`z z=)d};u?uebivC-Vw|9x>c=N3sfdrjDpbOf=x^>Py+Pd!Chi`9yRGV*+!_=5AQ~iz@ z!6%9N$El4b$M(%}I>?qdO6xg{=Wp?|Jsccv&?Myh97e&v5T@t6N6-5^&tfc# z(=y+Dwf+>HAIu#o?T|?z%}$0*?PaB2tJ^%@A+4U4PyfOo-@ht+RphJ~EG0kVw;n=N z$w3RU=;c`RKFr2ihj4vN8w7_u>Wayvk0kTAVAUE;O(NW5(}&K9>>=b5`UVi2F`Yq# z3$Jh10qP|7E=DU|+H8Hs6bB;R&%28IfeLL)#@{GI#A5>UAHtb(xS8G`anwAMu8cyo zL-UaHPEJvAi6rb>rS(n?)p{i5>HYv@eO(|Szh;s9^6@l)lrx- zwh`UrG884BKsgWtUNQ^)a_Ks%{Qa8JO61-v$TgJ3((1_r9YeAY`JFE;R+|aAQ_I!s z^o{Z{2atPdl`iz?Pq*7cq%eaqugRz`zjUvFoV;seNljlyu&Q!C=paShT0ti@st76R zzC9()r%O(psxIhz8Z#~iIsWbIEU~M2&wGmw$!i)VEx=t?&3GM*fp&c++k$1(6BVRc zpGr7FJo=#O>E?r9V3I-Sflt%5PK4yo1nh%{4;Hk6WTsQhWOr-w><-KUhjA#aX7r>5 zHbqo(J=W(B*M}JCK!?ZOUr)Ct)o-x43D*@Wg{TNU2y>{{ZDAy<|;iKT~AZL9Y#3%Qp__h zDjB7=zelV%kBP^d#;h=$h)hbcdP_W3?`m74Z};n)POWOP=E`xzgBCtW(y zhkt{r05+?I)=O}k=%a6akE4PhRtPY@;Z^@Wix8Z~AXs;F+sHoH(*Pbt(O=~IF z_nA6cS~msqg&V{|aO!fKMX7Ev%n4;RQCB7)KVL&YH*S{Iy>)Tre^tI9;G>|nS~hIk zN{xJA#b4K@bMG!pzBbN(-;{BRcFVY)C7+rOux>X#FIcvb z;C3#N5w`eiSy8UT(!B?UnVU)~|ac3-W)B3SDqA1=|kPKkPKv+`MOr9f83EcP>RRZ%mgb< zLu?jBls~-P9snk?);NHE&&SS4VnEVd4G-FH*VzYzrqb9<)z;P=rg@s z4qMix9BR|Dk6s72ev20| zkL7&rc-#j2Rr!ysLrL3K7;cdwpY>uULvvloFGOvkut=#OA@_K=J#L! zI1_!u{m_tT2t{_|PEUDj;-x@c0qXXF53_Q%x#RKH;~TS5`RY1{Em?~}cS_^~vhvN| zOPcxlIOzN27EghrD&2EnM**`O3ticB34z>Djf3nz%q0-S2!EzwpW)Hkm6EED0xwrF z1IA@VIoP9};M?@A`IJ)*k%2gr2Wsp)w{8x12dIiETyxag0Lg8EgHsF~(gm)`*}kc@z3Q9n za$YR0Q*!c3^LkgB#pults)d)irmgovOG8SzQHJ~V=M=6vW`nZOi4$b;$LY+szTwrl zek5wWs~bOW$hHP8`B2CBi$Qx0yw8ie-WEirBbP{f!Y0poW);;ezfFUG0?Y`8r{Uiw z_-e+Xrqn&VdW5=SJh=0~p6D1EHoud9z-fx%`3~X zF+=*AE)9i|x&oMwK;_x0C)Zt=Qx8F&K}+xhfN7zHb2p}af@;6Mb(eFP)&fZ>-!|VuKp$Owz}0a zym)$XoxP`K?){=6q8aiq)`^4o_J*@kz8`-i|~>R?T>5TRV+|~7=1rn zQmKPqm;8hEVYeuCpZXC|FQ#HvD$L3QC5t#x7=@T}e`I212rxrosM@}n5m^p_*!#Mr z)i;R~F(cDd51tHttrO`-#Q^;=5}VHs7>IESMluy9CC!-uyPm6{+>Y2e_0id;IjsZ4 zjg4@)Gf1#tj(8oFnUT5XltQS!!%7`^=jmq|$vQAGpxK2{ z2#>A*QSdkI++j|7;79*`tLZcV3;bu*$`IFy6=19_uD;ax1i6zK-|9cRm@i+Vx1(s6M>w*i8R)&)R5eU*see{(!eVv52>} zXvU3*dh~*M(mUWPN5c3Wk@clW-;jhx*KM&nM&XfO%HBIB+rc8SXLW(QNi%K_CZlD= zP-iE#sH*exkxsSBi>$yiT;Klew5v$GxH>DiuCJqfrZ>Iivozd$n*W^72CB~S;Lmxa3Jq~4V8>JLCIWvO)j7V?azY)*?O1%8A6A+uC#3u15*8)Sfa3!0?I!NJVd1K4H;KDC7J}SVFgbbv6Kg#O#J;G%wf{ znA5uH|0AxiCP&Z31Qnrujsu}H)O?B;S*TLq)s;_C#|3fI#q1Npgi2= zgryJXHFY%9UERzRW7viNi0Iy)jYDnd92egxoGod6BEIrcA)zn-#k)V9$JYyK%k`=x z+k8e#5N`4-YlmGM1u2B2Z^tC{yw{v?i0FOrS9Os?GI%sJx}|hL1eEq#3%(ZK6bL)2 zkcp`L7Y(v(tP|C3#)@G^b`iB>z0(Q1>^I(+rK9Wo^PjJ*r0`~~Y#ef>Y(LUnr|kPV z{eGvE3UyIn!hKcG(7by#)xu)vK8%sEg2@beeShT~pzi2mj!)$D?SWG~TK^vatw2)0 z5?X|O*n$qgI%R$78lRxLm9@F@?DZY}gSCzy>Oc123;&js7PsZb#S6>1=gu$BJoD^w z?!rYMNO6M|-OB4~udS?w)@4;!{aZJ%F2aF>^dY}_kelW3mCy7_*HHPanc3qgVard& z`)YZ?jMn(7C*`p=yAQ~uZ0lBavz5arA53wRTDpRDZ0QFaV74CU{f){uPaH=15FrOr z=o#Fojc$VsxhYjTi#TyRA$x_XA~1IlbQ<^-uhwzlwfbyy$LUQek#!&Pbw9`r^@J-g z*T3W;AoH=8seE%V72PL(yeXwTs~=Ghzzsd8_s!-UOmRTfdK%%r`A0lResg)bA-M8} z9F9uHybXjVIc~lcbzAwjXgCzOn!^C{?0Z?XU2c}=CVBT$$leXnnfk3I*% zlT9k=Y%Q(sK0Wp_{`kk9NG(^}M?BfP&7m5r9#8Jm97@q|(D(CuLp(TznSP$WPJOaR zWG|4Ln?RhJU+d(^N%o;{iWpxWvr5x9NRv;-bLNil)jB`n%oizJWSJHZY9IKH-l}qK zW-M5aqh0AEc}Q*?N{OEmY4|R?L4rMW->?ize(-7m1B^r|#DSZnq-$XHVKbz|Bhf=Cz_a)B>%pJV>##^c6 z&%x9*@A}ZO`YJN_q3^^r^|xGW%((ydffRdN4y5>G-=)jTMLr)`2UF~C-M7{D86UgS zp8I9`BW<64=`jBxvxFx;2bCF5KJ=|QGH*&{uIq5Okz_D;b#f7Iz)9K^RUxy8lRsHc_ml%{$l)jF=!gk=) z!{3IBs!|6Z?MFC%|iNnzb?{MEm)^4R4U(eFmzQ~91# zn>1I@_lKE#Gnr>s%6uDs$5=~sjqp@!r{|mKDE;8H@70LHx8j$ax_qw<8h?k)^X7e& zw3dY|EEmS9=Zn%YG(NA(^r6%vXU8Dc`Hx(x=UH6(>ItIIyd(^FTph!Fl^qb?aRNNbdKh*X*+<_}tvs40tINuz{t9Nq z|MB(zzS(G7d#)k7{KY}&ZZze%<3S0s`#)xU4hb3l}J2zpY^Knpx@8rY3%#^L` zqZv^ao!clTCOW;R;K6r0v8p?N%ha93D&JR~-Iaw>1yyJKWet+>}WgO>Lg& z`etZPdKFbj!O4%jaL8M<&KsFrt%M|FBVqAM9@#XippkB-&cmkLswYz1bNG~|+SI&t zc=;uKjob5J9|o9)U12hv*1RT7FMOkH=Rq9al*)o}i-nGUha79cTgz0Mia+xeL8P4K zQK9CeH`p!{p1h!F6DYY^j!r4q(k_Mhzvi;ZJHEP-UQlT^4O!bg%nF`jt*F4=6ks(8i1ret+qC$4%eKQ-%HAakrb& zcb^0;A+bO5hca(P;mAftHLTc`J;4jv!8M)X9jt#hf=Cv)eW+IR3G1f zK>=}}MfTJ$b0}#MeqhaZ3>M?%EE@q84H$%Lbb#6nO&GGObOvE{Q-y(kcC2$t7~i6S%j{!wa_L&_&(jP#-Id%4lWr#~@AA5`6FE7uHnXIwCg6TKwtWYLcBWXNKd zJ<5O2ZWDL=#{*jhmgxX|9THMIcude9a=-f3uUejX#bXKFuwlc74+5Th`pWW-w?DN! z_v|yC&%LgDZOOdr<;T-yTWU7;1?bQs@b!-;-T| zmh%l5$ZK)dW9x(Z^XK=Fk;{qQ2ON^EbtJ|2p_JEcWvD{n(Fa(L&kb6ok%`RF(X7{q zTEAg6Y(0ff)~TUTxJLW#fI^Uq`9zN>o~`3WS0RpdCefO2`a=8(g4v*piLO|rynGV+h#;nxmy}c`LrtSkI(9bf@fBc<6`xk4ervD8zvv zab3RUoif(9EO<+bg8|-5x#<$TH&g%gjq>WB$R1wF8-CEc=+{1w66L@~^_7_~tfb!z zec69glTat;6`#&`cE{rX(&dY)k9toaUVXDr{Q01``m-Ns5{<(O`nbY)LyyDQ-0W9< z$X9>nvG|Q{>}Gc1)FU_f6|QpGn;fr)?U)Nzx$F{SsmHqeG;*{oMb4x6P#En`IVQ@d zt;!51bQ>={KY1H>CW@K;FLWgKNt3 z*h;$qz%4sndpRjT|D{RnsQluN?=cMym9?SP`oYLo*mQZF3gExcAJChiJv z#I{>1t~JJCJgq(l!gf3krd-jy*%pgn+1m6q|L~35&e%;I5z8YJ15@S39zJ_)BcPPnd6Yb~7W>|%%Sl;Y&owq# z7jG(_52YTKlIMc&i3rR#n{xYFZWtE1QQg21Mg%mQu_rw7T44I0?2})LvwcOv8fLwP zhiRjQ5jU|A3MN;Q+rP`Qz1%j|GGfM!eQlzkEl~VSH6EmyRDtY-S3V{zjbGn;IP##2WYNtMXw1!)+D&E;>#A{BJ6Mnb_jj z{_xXOaqM3>U|l_?AYpdRc9h#SrSrE7d)zzGg)Sen#|J)6TYj;(pQBGeSt)rD&YvfM zHIJE+q)ZD|kh3RG#fqCO3GO&D?@r0*yxMM~{VSj0Bv}CbTNt3@fCy6gvl?XEH{dKGIV*KTb#=I+rQlWjk%}Vq1~ohc=qc6Spgy+7K$?C zuCR%mkMuqA^!(>VWt|JXf!j;79(qjqSC|zpEfhYS(jsC7uqJlLkt8e!sOm$0gmpRc zmWjwB$M8kGUQ}vsmK{_yL>;5@acPxTN^YdJR$I3*mqL=pYGyEPl%@Q3FRL|$n&P+e zG~`x+sdM~$Ms}G_D9FovIuG1(RXS6*txUHKGIAM_s}EGzJkU05u)AH94~sSNkg@Yl z#x{&H)6Xb6n_?eIu}D|@`T#)$#wQ?^#)ncOzbQo-6JIGWMTGygU020Jd8@v$js|9e29OyU*rL( zD^~K2d?O3XOK##)FE+Q-fyiC&COq&9U6HVCJ&n+(z0MWdd2c zJj{ge`qjS!;IZp#e&bQ?!Fa&`$nwfpzG8Xo?BmPvllOQke8Ywf8$N)zeRWyh`<`c( z_q^*}%Y};2!90;%uvfj$H?qRcKk~(rTy>Ei% z<~=v@bFkKOKt^6WX+yVT?Ok?Il9yo}N$c@#t;ejxUgt?8nT@xje1keS;gKWda4^(2 zY{-Xo4ZVyw>w#4Ci@uR>xhY@F==9ZgZa=jXhf?x49b~#0K6oevQ4g8KfqzWrZCxsT zLLtfKkiHgOu}E^@u+4gjWV4Qm$i@9jc>~KDou%YP)&!+LR$Q6Utwf2E_~fa6Mvo!C zqtdz1iH}Qu984j*p{qiYH8Q7Nr^L`fJ8rcqQNF zhj^69LB4rG#%1OPV-%z+|(t3iJ#E+TKt7;(2la6HQrJFbMq9+1XKxHx%@g{liT zi>;rRtaBj6VbuJ_5@S7|Ms%K4FCU~#hfzNG@d3~c_tU;5eYe|9 zB!r)dr!vErW7zF9h1E*L7$MT=kN)oe$()e9;3K@O{Z~Zp6`p@P0<3oK<`-v`!lf;) z%N}W1%P@~NO-?XOJZ&rIz3?Kz15*w5aPN@D;B6{x<+FI0_;r~7*ZL(WaqJ}i&U~3M zVe(p%J~odN=j+D8sJ7A=9CMIO{4=+wpD{O8e9FZU1}aZjDIQFj51W_VTVNh7s{YRy z&K!f8`LokFFV`*h0rL~SOf}}m0g?0~#z!8QD1OP0@$0gShdKnq?lR%OWSPo2Wy&J2 z8lx#EWqQrW#IMGt{gKD#f58{tea*{X;fCmiA;QSUz6C^i|Nc+?vUb(JSFPrS^n zf9f#&6H)dU1N&~{r9|3ixP~Fib4OCrQ;AC z-fA;)gfT9q=sqbbM^0#?r9~e)iYeajGw+H6-&MC(I2-bDxjj}hA!n@Lmm0vGj`5h|`#qL<)=P4vBePu@xTz^C~o4YJDS&ZPLc`~`9 z`u|W9MjfZ!Z5P^R>A6i}`sTmn>(4-_4u0Yev+9F*FkKJ$lngNuzoc!GJo<^j9MD85 zo|So&1j_60e51GzMHoe0`4ZO(H2s#oLgvs6@#Fl|FS6dQpQwQR#7fDNK1{uitACI; z?Zrlf_VQr|{igG&VlY_0dqc{Bk~bXiE!wa98Es^s2OoUU8`a}HeT)2*jU%I?pxOmA=AR@19 z1h2ZT-fYQ4cCJWO=&?u7E{~i&yPSRKjBidE)K7*#v7x+pLY&V>UR7NzLzTr1AaC3~ zUTF*ZSrkN%dDl1X>nts_e)G>Q?|tvP zvo?ASlwRcdGc?iU(HV*-ujoBcKcJI! z(|U(=48qV~Sc7{_L`U%rQt1;MCOxk8>d{9YUe2C&~7j-VVp}Pd~f7>+SDco;!bjd0P3s=hS~ZXokHFX`geStVQRnre_+(mx<%WZk257T&JnP3Z^BUv7GzR2W$U4_l2g)2bxiFPq z=+<&f>8wMGXx|5=J3Vmb^m0o5i#?LRbRgbnr@j$?1$|R~zWFY`$UpiXH_g+}PbjYIiVRgh@}!SAOw_WO^{fF`DfLRdWpW{?WZ)1?+#Ls*;6`pT$9NTA=Ck;v zkMNsDK9EwsK^8f5&BJKE(MkVPdhhG`jjiI>tQNoi^r7;^M4v}jrd_Fb-IK1FL|r&yjQqt`G@7m z*~Fhr0#Bc(kB2V>W6h%Y{6sdk3fZkHLBoo`>X44dNDz!v}Pk77ye z2_p}YdBD^ql)(5`hGh=7knAeck$rfMqwH&78w2Lcrj0%KLHmIWChP~ToC3M>zlZWU z7;pe#Bc+TMX~u=YXZ~a2>Ho-L#2E(uyF{Pr`IEJ>?TI@QdVidQrp(>kGw3~Y+L1n) zHk_#;7da}sx(wUk^X-|E8?v9Tw{NYd$9Z4L(%9`4nNAY7DiTc+Ed$>SD9! zGq1oXkPvnk(`Ce&o=k;hh(9Y( z@G8?cqM00}B*uPQm7^Yt16fecWX+diTz-{Zg0_E`i+?jvhA|6eVBhbsIN&gU7dBA= zJ{prXsJQ;011Ec1G3O{Rf>Eq3A{%8k4!Da3R8PzOg^0H4_GwB|UxppCEy96vbs6mm zrA=qMY=QG6?}=U%iI+o;;@5ppeT;rqea;6VH|szIw+3I*XrR|v@j*-(^C9_(56gL~f0xU5$8(TYbid{um!&yIVMC538}lurGLb*lvUiy6Wg0Pk-SG#q z&(1?pd>g87+1_V#+3rJacKa7ewmzH@wZe-M%3_AhnA7=;7dPJh6p9$F3nr@SIa~h_{VychHp+I51wenv=XZbW?h#Mr3ZK0j!Br?!I9%{ z{?|%a7E>B-PO<6rkHucP?9DmKH4C;GBbw3-Id;B6XT95_*0p-}?Lw19*CRCe5t1f3 za*Y|JS2UEi{`fDi6=)Vf60ni^gp*BV_$M7C)z3pjp@^D!tf(q+PpSJ z=r7&ov{#o$!JviW;?{Z-jy3%}{WpEMc$9zF$?{M>@JTryGg=3_exr;o!-ft2itvu7&MohG_q&#J=bqCVAbneNIO~>gDpnj%dpL$itOw@9 zb`seMguUrE-U;vY@sbORX?Sum*7;n(Zuc4vh~qWEYsS28#BYS}Q1q6pql#NPs~o1? zxShU&ixKOe+EcJTqI13=q&Hl-e0h2H>1USn&z<)}EF6B^tb-%pnDlyDZBWz@n=a1> zKc(yXposOkb!mkU9%0d(hHubvZ}~+ph|XXgMSDfcCjq()dHUfW%265Ci%3`NzTXnJ z9zYuLi2kZHUayr$m+Rt0Yxxs8&P&CM#838ng^V-E*g0PTzJWF0Dcl-u9(Z4)_*xCbWUAt3q4W@+QGYh>B)?I0V)Sdb z&CH`I{8I&6Y{p(L^WaO#$k+9}P5Kb!(WV?o z@o?$<**>>H9a8(VeLP@T{WR^6xr4fn`APkg!&l1oZ%AFawp`L&+P@!l;lian}le}A30`DOkX{If0?H0LnUkYHqVSR@xplJ zu#EjyWW|F3H3n;M=r`>sckn6`e#~Qd*x?yB5sOFA{8qwFG>CMz&rllD^;mbknFnmf z-Ul2BuQ{UQ51R65@?XjrfCc{c_!@OGb$Hyi9ELgH7r}lGbJFG%zor+$%#t{b8Agrr z@zvP3*6(2s#B-js!|&L7#UD;Aq}|QYE_cx6JL<4{C8^_u+^A(&t5gsuZ^fr3=ZxR@;cb2tCw?+*@3<2I+c9v*-r+m(?d`xdO!hvch&j^~2h+(a zUhxwDm{w9W7i&T;`}4MIa!#2tlH0QU%qw51DrzAUc|ntjGTI8MpR~~8a=(;Wo}{#L zT#F40jmwjqE$R|%8HPe^!d@UAV-(M_JIaVKP%#bLD|*Z|477i!!(_@H`IMKCDDk&j zrWf@oUi_*Q@z*SN8tUBXNMt*Uamf<*N_m)rMYRw2*f2-=y`3~l>+k)tfFtvky~6S- zzWBL}GRYUlR)t1X4MDnmgZ)U9c9GkSwD4!5e*6XSnlh9BP!uX}m*;rP`c$1ZQ=PBN zRJ#7gJj57Iq8O{VdvTj9*_|+mO#`{sqq70QynJ8%Vr962>U8^6!Fno%}Z%UmwnRcnNA^N1>D60*b z5AIM1Wv(|v$kSr;1@8Nlb*HM>Cpq04m1ve}Uxy$34cqXy9diVZfW8#kP72xmuRqLO zYRo4dZ>jNwJ$?Jkqo?h+VZ(+E|0?kIUwGGY@#6XA(#4C*<;$0s8#k|e?p<#-^{mNt zI0o!%zYd7G8;zNJH+)68GgTf$&68z#E-g!6JePW1g;>wfB}87^+TU^dhGN!V)+E-v z)+_v*Qlk5{7AivYSJoq~b=(N`n6>{y0Ol*5Ufb5jV~0l-5T?&bRx8%+as8?Da?L(}#mLm0u{j zPSO@x4y6yOjMhb6S0Auo9#5TAM;}B9DI1v~f68iIN`X~h=ZAi#ym+vYdZqv6Fn}BG zrg}3ab<6`&C!}xXla72g8CggFgU{>->JwtJkDG4Dv-B?FVeUZbs~b75*kj_hf7yo{ zJN54N#VlF)=B0Q;M!db4z8rngpLDCf?FS2e!(YV1q)O-fRRBEpo+gJ6c~dO%K#k3Q z0GR;7T*n=A_r@)8Qj=iZd==4`b8setf7-XoAe^y!$MTB|$=45U(QjN{C^tMAYmxgX z&-IGjr`v%AIcO88&|@YCoaI?iNtHww#YsK*a>D(VEeAr(b$pG4H>L6!y>)Lz`#7Jf z4{~)5%8K0s%$cM;yQH1mxf7 zz7nJP!7RH28|)w_%s~ZnHu9%Tj4r?IQ=BHdZNcss1H<2lBYrypa>7uB`}!i7A{AFF z%^B^x7 zLUJf|`V@y!WLiX|W|+co%l}iKeYO~>`nxWxFm_N}cTgIq%?Q$FrPqG|W7VL;7o+0G zJ^AloBsAt=+a26-C|P#rTVB@DiK#o=lwzLQkCPF_5_og8Nb#6CHPyW;{IYE$`@&x>P9 zZj7vC*~ujGw-^0bV+TJEwz5Mxz}H3p#BHUg$3(~hl|B5*aJu*ycCT1NWFzgGNa#?R zcy*ti;i$UUTbC&qPLf|-voV|I*kzZUF$6U~c6*?WM7Bc62p%x7?OO{~=qaFC4OfcJpE~R}^Zu{r25u2`+y%*2nt3SEDkjpGjLG2Bc#y2QP)8T?; zSh954T@x#rx5i zQBp)w=H0H9zS|FQr79>ADXV@s+X?}jxm5usUf(>LpKMIpt_zzS4yBOUT1J$N)H^(9 z&V%?=&cQ>eGD)ut((khgjv{kM<))q-z>T(n6wiv5${`+U;KE}=cI4R`G3Uvj{5xJ( zI5KMD8u`)I%sS+6YF;$)o`LCR0p*|jWw2S^on@<2~BdN_IpCVBr$2H2} zvzqX0dHRXPJP~OnPx7VS&Y$=qe|nP}4Zb;|O|HX2K9H(S8_HncP@+Ao(}|~+X=IbK zZDN?>o{>7v{s-giLMf_#E2<8$wPo1hf5mFw*{+q9G^uma;oUL-eD0|a?Vfn`a&3oEmUdULG3M=DlFJ(2dSOkK zhgoEey^IQ(9G**ionw8BKG6CE>m_q--(*=Iwu#=^zh~kHGPGVjC_2ue6zd;Rlw|qLR4y8n_uRb(V9ncdj7z?@O6fYlXRWXE$+R3u-BkafUjpitvhztt_i355MT2_u&2bL1`XDm(T(_N_UHJfsjUJ;Ayrxg&6KM2je+th!<9f(SzoXAsrW{HsEhWMiV+8*BtRb?9 zzHK&Crpl-^lmo1T52DYi|CecEJjfKg2BvjoS<=2d{1%@3fxB*F+SfX(@{T#>E9(A2 z-=TkU_?z*Tr|-MW0O6MvbUR?}3%r@H%y01^FmMX~m9`Jo7;DIrx1vrxAVSt05+Qfy z4~0=T`Y~h0<%)OrQTLhpWMBPuQ@r1d^R`T6sm97>G?sRvwk&$pJJsQ0hg z4+kED$ccW>ysJJxf2I$UH>7+hReTWVrr&Dl{w$sfc)^=p~TxcjHJEQ zM>F4N&d20|*`Du6gS_2`#5bQA?87J@07Gi?90YUoJAJ`(d|J!Y%@`iSZ{`kot8%b+ zd|ze|M%;s>3;!X->^@=}R$t1*ch7Vrn=yhNhoMqj0A-}irq%6mn^Sl^%YHZ%RH!)n z5y>LPxb9>1J*=EHwJ|x{I)w_&$DEh!?MUEor><=l(Z({}=hUBipO3x!;20mG0g0Zr zfii#9K@No5^LgeY$8(yr^$h1}KGMH-%d9nAeLQ`=NOKA6x9?>@xNT3oPQUiqDhw1q z;hBd&vc6`WWo%>(&b-;ohCk=6GE!#E!LpmDDl_wl3QUDd1}UrWRZw2*3LdXT_OJ5$ zpuOb>-+bX}tf(;P1S~>U4q2Fa;zOp~_>E6#sbD5kb2Tazxgek+9}7!DLwHhJ{M#y+ zw%2F8m~c>z97pVsW`^BO801;B4|q`}RT&dJnosfU&!`pe9j66h;VM6Mm%)eM=71k~ zw4cf|6^~thv(e@U1yle!j173jXw#mPyUfFoEGIa@O2dkMavQ_4T8VQnbQMOKlgF@P zZ+Cgx1my-Uh6_~f)$OPL7&GK{eIXF7tRu?;)6m+7s$VfSv|#2Fem9rrZ^*nZQwj&> zUAg^PI`v2QQ+S3&`c;n+O+ZJa>}7l~PKZz&2$!!5MRFMgRG*!?QopWmMKPZujrF@y zFz#xMkI&|b$2%fE^EZA6%DgHrV~BLrv1M)}WqQoC$=HE!6e1K|lz%B0{D#$0DOC5J zfbyGGd}>cKpLo1a*>Q7O6BT7j3Fh;Hcccu#7yRA5Enblu;}~9wjV{-VjPjy9Pi|dq zxh+q(KfKDc;GKcb*e8Hu@rvW=TqS}-vk^GdQ`kH00bSw@mD%nJ(y;1b5B+H~X4=hx@;X#4)=loD8<<17D z6k@l@yog4&`gfkqwsDx>gp!VJ$zXB|nbx(ALE@i!prA%cRi9D)7z(Wf`1{)86&^iK zlpo_68L*K>;Z6G7*w4X0)(x5x)8$cCm6rurvXk7*qRT3Ug7$Emq+D;lnj|~Ik*|5E zyjbWdjBVnPBbZawp~5|T)=oeiPLCN^^4f#vmi)V}D=M;$-_NS??QMX+Uw%yMv+8xkmP$d@#3FWN~a3R0uzXp`;sw249#B`V?6fj*l4 zQHq#p?YEIf6shiOZhQExzCrCU*d(#hV{^yvIdQ{-&rGN@W&WB$AJCZeO+4y&Jex|& z?{O25e=tOA3;gK{pZfEL(6;g`vcKeP(|$EB@w2(s6`D$UVOV|>I_aUw*E-N9WwFd` zmBn`ZrEPX8aUI1be=CU7H^2D&&V>sXm*<{6w_JZVpPbpSVZ(-h6?pcUr{n32$R3SZX)-{A53Q8y>NZp`Zm zL>^V9F1SbfI2>L9Hj+MTkq?KowzK|OPeQNMnz)?U`z^+OT)w`{V z5tJL0l9A*deQjz3(pA~k36IrHQtQn9)IMu+C|?%Hjr<(?utrb1E>!ynuS4ht)=Q$x zxm*=VJyXBvJ|jKCP&yw-k*0KtbzOGWLh(v^CI-JZiEA0$aP1-S-F<#Nbku&W={@G8}o=VhjYOz;xTPuI^^Nn)l=^Jrw zpS1nBfhjr&DqnI1woD!tOlX!>h;f+l6N-#_Zvn6D{d^b&(XWx8`f(0`(hsR4dBL7< zXsZ6y$G`t2`5-UqlS3dy5=~r1PLjJ1nnZ(7g>%7fYk9PPJVf@GdJILB^~GS3hNW2c zQF-qmkq<=Ab5}{Q%6h8Lu-ATAvO{O@x0pDLnh%OB_uAvRpK6Xke(p2$7s;1Hs4JJQ zEEmsf4!fxN>v9gK`nPx~n8R%PpWnt({w|}+;Z4;3WMY4=Z;T1gNz?wthr+qnd!r7d z_|z%CYm2^o{`^Xp#7lmcmNy9N(1-r4d?G#zB=Vr|NgnBg>62bF5=AljlS!E3!5=c| zemWmW`Asi{IS+V8UMd&9a}cFIDY<#fVtUMJ{EhwNH0I_8HlM|-c?^vWSoz4|*d&5y=CMJR*)BPAnNH<`Qra*a9{UblE@o5s0O84A4r@Ek7s6VWc*EF( z$Hj>AOF=l{+VqmK7dgFJru<#sim=-@y(VOT-v?42AMURer}K8(XpWT#b!#_a|yu2RpsGMzY~x4#SRKBDefKXBb6EqEL)L>OmS^)eBH%w!A~|FSmI;?3oWhjG zMx|}G!4-t_eE#y8xgS5Ec(PHm|sWSPUDSoh&!y=xZ^d&d!&h#as z)5X4n9p{`cmU7s;-FArD-A37Gq(MU%1^SJlB$gBHAerH@rpOiF#OeO0`BZS9qfa5T zlA8~vG}n=a&;Fn!62IH1S@d|Q5@y(sFy%cqNxsQc8+x}1q`pyo(p9B+V(J%(~KjLoe-VNv79bE!H8JUDJ4*6^ik*(;6>{1WCr zxlQCJxut(%7AeeSb-X0M2THiZd_Q1hRlT>|F*;Bta<;57XRjKA9+S^|v&v@s{P_#Z zxpU{8dc%ed8$Qr@{>rjkx^&S8<2P7JJ`Z zf+^&N?$iCnAWW1n<%NEO-huuhck7E4`^Dji_lQlI_Xwwi1Pxo^kTlKvK4@#Vy<6Y) zcHQHZdJJkkG5WgoXUQvi8fJ7e-)Jvw(~GH zH`S0YC6gC>Bn~6FSx(=ieEL5TXb*HfKg0%k&ya)2zBx`E=6Zo0>7;qf-qMMm?XF9? z@vvF4o9f*C`1o2ohRd1#FfT(0`9UfUD2bF=0ktOn7-_YYNDL$Wf)xYV){6#x(5yw_$?;2my9Lm8ydNXfKv43Gq z_$D@U8SPyHr#g`5e#s&|NDh1Ys7&$Y{S~s~eu1eEqp~OF-h%hzN^f~F&vDb6 zz9`;r@&@b;)j=|14Bzl!RE;<4$y~2=b1!N>R{n~|5xSzPH{ei5NcuR-#2*?+xjjCZ zzE=C{Z`Lp;9uLKS{ySyDMg5h3*KZE1$cLcIpQ$1H>}=dCyY?5Qe1M~@d$@=9Ky%Jz z#68SD)z^0QvGm259s?e)8WZk=D@9f$7ENu^y23bgs`c1z=D_M>J+B{V$1qg+XYJ~> z$viMm*Iezj3{&J!?_z2U_;9GkF>%9l_-&RA7Y`rI!{HSkeg8}2we=6{AT=%w@-a_Y z1I&xdjT;{4G8>fR`a93a#cQ4B2b-e4Z`RabYRP&S`8gl+4W$mipYMCwBagapR~!Y? zOr*igxT7x1XP!G3ixQ91Nf{4(;TJBi`6GX1dE__zh8LzU{nF*p-}t4=`S1MB<{7)YV-fe*Cw$y;{3Abdx%1Rh=En^K3xp5E{^8&8SC`wT zPAxb8=|5eb_~YNOocXn1v|Rq)zn#$3-toYZVOO0dd#F)mQ08PO3Wj}E-igop%;os& zU%woC-RqWPA1=ZaS?K34B;5e{R9^2ZoQ#5}^kxKi%qgURf15{^SA64tvpo23{OaY( z|L^;T-KwmYB5 zY{WSX4w}N9VDi~XTXsZ89Q6Twh9oDECT4lyFfSx3z8ELvA3xmK+po-6%CGX!__Ez& zsiFYK(SP&dKsj*RDogGK+s|V|{yjvTcVn^!6{<_evsnyS#j*IyehK$NH-6#o%*yLR6>AX=TnzBD^0=Cb|b8&Fj>QoJ#-O6 z2NqYEeFHua-(@2Tb->5z)+DaK+fR0H)U@F8^koi2viP&mM#g!nt-tYaMFa)FykJjW zP4Jc*4OzTl!ev(3BaIxiqm_4*2<5%@E4K>F8%{UMzHcZJ*wL3L7dgU%%c?T_$w+QGxLuS*hx>CN z-gxHeq$q^RcZ_Z2ohnvkfUVTe6?d59mH%OI?R^3JaYn#E;>UmIYTj+N-BE_yS34c8 z1MP@)1WxeN?UD`c|L9|nE|0(RkqX(cVZ(+G94?%@z5Mfk_LIx4J6ZeuAQ$UYg@gUs z61V5tOt|G&o~Kx^OwCr>TMPM#oRl_CyVQ@z&J+bUYKtUviCdJcv7bl-74-KV_oi3=B( zi_e`~p5x(z^B0$A&7oPd`((wW%iBX zjst&r3nHJILUZs9AnQDJ^H=rq4J^0G5$hPUT>2(5awHBQ; z@2TNkeW>RR%bU-B(pOCBD;|rA3(O<(kc>E3;mrjOhH@wxlfFc~;FU5N*Pe5jKRFcF ze2B>$)cQa3C1YdU$nJguRk_H*{nc%I(+6MU_oJ?c_*|bq)#n4+yxoxbleUv!5CMP6 zH+_-5S5)O8H(6Y+rRBKzXTGA}c(28rC7zLA+ta6Nj4GWE1L#l8KjPbck?|$oaQmjW z>a-pdtaHW;@-px6WyWNxNy4R_!Q2lTmDaI`*?whInY5Y5dC&Kz$RCSit(VSX1s?F$ zc3l!jSNyA7*Mq;dOTG>(Kc{eD>|LJGd`uY8oMcUxJXh>GJJM?1!3dURO1MT}{RS#< z@a`y8CwtBQG8B-kF}t5(_V|>k@i6Q~ozCST49i4+w43|7`q8wG(0)AzM%z;CVQ}8z zBjv2+n$*90-t2kQbA;TPTYFyZG|3}*RmFiAfD7|7@w(nqi9LNV{Cd4m-s9c}W3JZI zl$$xtWtL1+7G8~(&f z5BV4b6Jpm zcX61X`eDqrGf>I{cl)>9$c8*7)6Al=q_{J`%@jl)j1k5feHFV+(kL7j z>F4kV5AcFQbnV9S{MGBr9Vsa$DI}fuKh^*L|0|*r>V+udppYHPu@4Dl9jlO)tR(B$ za-2$q?0M{yP4?bSHs=^6dmZc8^O$FUy+7ao;Q7Pz@_am=xBKmWy-ND!<*5c=BvYR- zuZ$E%t}DOd?bz|p=r?j8>&R7U=}*g%`J~DVxcnbm`cWX$i=ePc3F`-6 zLKLGL%96kJ@bQ2Bb$EOmCG(>cIA-K+n31|~o)O|7ouj_O%!0Z7 zSg0wt@>#%UI@dkSAnb;gO8;m3{>fd9##x5(&!;H9RCklGjgNUNbTmJ1rN5mC_dal$ zHbpRivTLtG3T+CQfpPnPLa+HDpWEmWOb{xaQ4(j@$I(Z&1DEqFL8ARJ*N$lILC}82 zn^IdrzCWWH5loYf$9l9;rjv0|O>5vaQPGy3E~dq(M>SJY-GlFHJoNoV2TQJC1ns14 z3k>x~Ijcw0h8g^{0*Z^yy{fs1tG_RRBoFPLjx$c^dbS|6Pj98ahdqr0Cs9b{kDzj}{gw)FT<`f1GJ}_|IvmfKoyY`Hk#% z(I{hf4Q%@pxWT#nHJufsp?UV@m34o7yom9AK}sF~7-)_$pRGIfd>1mj1s!htd2&MN z(Fx{E4#;ZAJ2&BU?>g^|d=Zq$4LZLfVY)36$PVm=%GhA5D5rYR30n-SiqCs_uZsfr z=~4~aE(RnL_#wb#?S70(wGytsi)qDCDlpt3ALQ+L@isa1c$5D~JQ_{VCz z9aJrn$H2#nd<@RY-5Zu8o-MfCY)e%M%euLHto9CwF(8zS$%h#YH|(wY<&hk3AWYXB zTj!?t?IraoDqM!hjV>+}{RXt%e3-Hr$o<|q=FzDRm!^+Jnh}DJr)nkQyvjADMXdxj zUN-h;M|48U8UM3o6h2?CYZ^l|?;?v%J*3 zh)wo@Fve8J=zdNk=<{9j^^f#*It{ky?TIoj`_qnF&{&EV>=~s$ee_|U*MFfAX)r>< zrtn0p0XL#br>ydwXaoRqDR)v_kq%h_HrBkEq=2etx`cRQ{gEk}qOSY-@Ws4vOjUuWpRN~!;K~TrD@Na;a4vp01ExHM zYF;0?!h}-TIEnSJU-zL=NxY-7#81un)y=eAh;=>8bGpZ(ey7y2G<8&fA?l7@Qet|C z*aD7NJVL$VbpFQFiCBmOZm>a-d#ax#g3tE z+B1kBLCGjFBHa74Ene0=5cm`OCw$q0lW)(#*Ik z9?7nrpIK7{J-szEtt&gvus3Af@&=WIfAHCC1|S3fJsl5YH@=hRwx!d2%aem>)~4vj z@^emtrG{vH@`TY0Rv&{&8vT+say}4)@#SwNW}H`6Zyr`fvrvZ<=k8@Z30_nr|5bX- z`DP5f)a@0Nf0B~SX{I0Plq7CFn2?y)V)Q(SLjc?TAYYi?I42z#6qY=4F6Fpr5s-KL zMr2>)Z5D^8%d-gF3SG%s8W#uFFQ0=d`-IvUeV_+TF<1h(V!gg_MJ9!d^=OT#jrH-X z`o7G94Zgykc9=3DAqN7Q{G4zNRDbC@OTv81nyOYHpPb?J{98m86Uu+jW;uX$DFjH0bZ5;&n8)zT2O7Qnc76#@}JpX!)t{TVqeC-U5(>pT`|!1 zd0Vw3q;%AxDZiom;uG+kA26W=@F+r5ZSF7@vfE=*ui8K{0_-)#&oZQQBY8^(#F`9~ znY%wc&5ym=^uS{B`M10$+i6et!{6hd?!0>QJ%Ir~eE1cj@c7%Eose~Hh@bqC5B(a5 z|MG;unBeqp?eFne?tw9{@J=z#I+_!$ddeOw#?xWq1~<8wIcmc+i{UEkBc_+`&d~y} zH51Vg0aMIxgVu*(1BFCa!n-Ed(a;qw!iQ$dNSB$wxFpprejd|rb%dD2MwLWd4E>S$ zO;_3g5hFGoN114xy}fKR*+evmH{pctW+LUPB9F|~pi#fD%31#K^z&G@`M;7$z>^Y6 zYV*@@%uj2@B3@3HTJ|0;AAQ9i@pl!I$B~hrkAD+b+&62c)CX-zS(D#%;K^R>!;*dy zFFXHl>*|u4PV!nh13JoL^sYY(t<~oy=PO=!Chw*qQEq-kw?D2mS9?xBIW(nZf%#8= zH0#Ys!+YZ}Y89779bEqqWQr!3J3smsSeCVvxIK6nBzx3MRI>6k1&=k+B3qMrCYVtiP&@YYwlveZH%H-eP_kQGXl;=(dI{tGP zpsP#@yOsdPA6`fPV^-K}ye&RqlHq-6Cabw5ctdg{HQ&js!Nanrey$q++C4b`M?anI z>q1rPSo2z!K6Yq*V|0Q=!8SWn)?@H7jplAlh*CmD1sibS6^y!jE}vZSd&t6Uw_ip1 zyQr@!&%E{2=M`SXFv?{Q4FMt>9(NCW6(%Vv%ogU?Z|=L_@#kLy_g=$S_BJEt@xV z2W047AE|L_vY-`NWlT1MRujGU4mF(IGDq=03RN!kcW5w8$7%qy^Px-1&p`RX{lDD5 zYO)tlps{hy$$S?D_LbX-7zsi7HrlU9<*(zP4WG(Ko$ve-LD_BRtXK+j~E z@}|;@xO<@{qrb$^jl{vI@l(yMBw^`Q|3OVRXhnOSA|eW1wQG^9$R2^ zV2LyquVZuVwEpVb=6liTx#hF8tTBXue*ho>0@gvio9(6b_WJLIy%(}7q+Il35tE<) zgWUbz#o7l^mpC>g2;saCgCv&#XXaI=AKYz zAC``S-8cK!E@?7dy_xJokCk_-PRRa)Wu(e=j_|BCJ}Z)QxaOMv5A2(o95sI~BvwFp zTehiavchlT9RC>`=W9eY1mQj>5?YCEyCuj;f#oRKc{Yw9kL!~By=hQvg(sg< z%$VwVU zaG*mQ$qM`gvQddZ4Bm(A3``d4|C;TxIZ{jxg%M4~wLqkmr2F-rCr`{OK!o(uu~9W2FnH`fW@_jGdx>!-Tff};|Tmd{2w(2*5`KJ21{+n zs#BM3F4j2^o}h1wo5WDsD{NGb zC8686YU(7wA{x3Z^x*S`0~HIa%AlN4=CNZH)Dv_GwG&}8Oo^_jS7HuTAl|=7rAR_H z$kw$A5*U5ohDx=JRyvL)iz;T%GI}AbX*2tHt>*Mu7|p}QMy4~~K3!epM6C~~ z_#O}ydY{5A%j7BFT1>%>m2B|9e&tEE6m+s{g4M<8qg5Y}+mg5~m`jiSVoTp|5vU>a#HaSJ{5 z2w@UJaxWG;{}1n4EO_2_|Ecd-M*?3auTrD#fUvvMFH2z!m5l`Cb@>L;U`?uIgUk}- z*4aTcY=GQj@OQMLawk87Tk&$}n*-xJJz(~tuMWrj>>AR~?mf_C$XN~Ws;EEVG})?% zWp|$WtSgG}Ot68*apD7rvfYI!f`M+}Yb*9C;+)KW1`qq`e*vGqkHOB>toK z*A_c@6Osgwo5(SLW~D0IvLqbg#iZ3wZ!RaEejPSK_QgSE2t1IZ)l^SN~~xj`pA zznB9Qi(>0BY2~zAxBGXntE4f(V(|Dh>U&?x@~d`5^T!3+x0&MD ztB=y3#S5nk%!IZ!gI@E#bb^<8WYO}|v&zZ_@7cr>l9>xr(5%5)O0T8wFBPThd-sbF3twqA`^wmeAHMgp-9MEDW zA3IgwUUhyZ`?Fyz^nq&M^ygkWcO86#*p$b6o;*v~L^s)sM>6NXz`6SF+{HIm&D`P! zpWhj#$j~3lJ-hBpkqE*6sS?V)AvWdG(EmonD>3RU8uS;HL7uq~R%WtpM>iSo@F_j2 zFHV%9b~pJCm+QQF)=x=WgJP~#_&{i6dWvD;uBm9`zYH!j9g zvqqUrCTb=W*d`cfyoPa}wH(Nz>A$4O`%P5xSkhY~ zD`)+C^BwiRR(Ah)d(vRCO-pimbLkmffR-sYF!7+k|8S8|nLH7=)jSrsuFmJ(`&to~ zOPITf^|L9Ln3U_jWd)yim$TFYlmhKdZny~9d|C9eCebBr9@C)js(%J?1}zHmd}>7yr6Z5MQW z)JT?G1}0J%{{!bei;h#d1&YrVGIy(GUJWX?m>Fzl8^bmYu1XPD@yqim9NTgRC_8r+ zat${8h50AodgDd?e#Pa#Iip=5s$@0OEhDJ7RkD49>jB9h}lXF%Ir<&hW5k^2i}e zlk@b)+5>al8zzkAX{#OB3Z?J*wx$n5n-bLIeQ^(W?mYG|GpXdXDqrG5 z^Mx0Ev@dVB4RG1#o6n$l;_}vtH+fOGUqAR+&}S2R05eCX0e86*C)@_;_(Ij8cR?1W z2Pk2cUjTv-${z(kCI)gQDF49lW7n!tI-F&=d`sNwQABH0Pl&nF&Ax?|Q30l(=j1lD z{4HY+Pf}C+7R~l$W`lw;n=vma>c2x%hu#g3wek3u1HB~aLU1)ME9&M3FrLLOg8oJ- zSYFP~ZODl+;9>1)_+SE$-IL)#C;ums2Un$#T|&?Xyts5m-u5-t-n!A~Mp>5oNjnrP zL8ZY5;G;1?hc~PIjjZb}IdfLl#O2aQ#Odbmvot1Bm$7UEZZ7!g?gsqK|H6fg{X0U6l%tQ|j|@fborzVj#Y|!dV~}+xSg<_h1)?!%6+v9Xc3cFd zZmma|=v)MO8ok|qh>Qv{DGW8)6AfOu*z-8*qolY;BcT7%lbNX%6rDp4g@Avb(~P||g?W1IGpU=3Hld)Lj>7+!%1qrB$ zTGf{vb6!CyxwtU|XV)klHcyI!Nn18;t%op*C^`giNQcxrov9%F!k%C#yB8027J%j! ze-tCKKS>S;sW7$i;A_Da*2I?Jb=(EkB8KeS?}8+hz=fy{`^Qlf8UuUYV?x+m-;vgs^TJhgXO$Ge zgxgTv-9#F;D2;0$6MD=0B9_yl47IEZZk)+^0esqbeLa5I>vnZ@r!(i|alKjd&~}UJ zQ@Mq^%$XOMA1%!O%0#={9NSSMWOF!=IGV+GXC-bN3WK(a&({q5gF@yy;d*dM(DRjg z>^3Tl@o4ee&Aa>+$#$q{lHB>PE=l;yu?fppJMvDgss^zIa&oZk|Hm+C{#3K;> zEve~Z_M;GwE4P~r1(zPG`2Tsez!9yItfB!DYl0 z6NYDYs%~>}1~7#bFtUa|gYs%14@lS>`4VdtOut*5joJ>3i?+M|;XbGa+>iqELuhvC zK#r;*@qceH`Mz{y>ItN2STHfO*|zzpeLr4w`ubxHbW1oZZNfq`0M&+LI0=fad>P8u zfHK*TK5I2Ka5Hx*4SFM_KO3$*t+RBJRMvJsk&HBV z>SyNC)xYUuTxbcYlivc$=}AlR=iPUuUWq#3aznk`~1RVYn4-LtmovpmD=b8qeB^YuyGb=USbDfC`8~- zk_(yn*7Mf`0@*JJ@jgufG$19hKfjK{=U(zh|xsLh=YGw=4H2th)f5)m9N;9t& zKrASG2Mr7D5%YnxtQo1)z;f+X6^_gP0?!@|%>7-uT&>l_EwUp3DsB31WB%48D_*9$ zzFc-YX}RoVXlGv7Pdu7u66zu@b306(=aa|G?S8;KE+9d zEocQoa-$lB`_!H?vB{-uHRS%GzFyDRewsZz6O{G?K3=<|kmN9RU8cCAlyJA3M}#>p zmzu-ib%D8fr~kr_bw7xahI{nmqw!6%{Hq6a6@SfLKo1mE={7Q?ie9%?Red!PtDA{UYE;?ZuhS~kgarBM&eN}w={MIe* zHYRc_vl_@WV`uW+N5V6EbXVjtoD`j~#>7-xLgj_6_AGpX_7&6Xi!;ApSmeE-;3Y_J zR0vo#u=vP|%2(R|SR2E$i{g>mz3|q9&kcS+`>-YGo{NBlf&Ce~B)-gOAsW}g>V7?N zZ`IL@`_{ipcQhvvRmm!H>7pK{mWK8MlquqUKrah1mE|Cld=t1jAOtnE(;*T3>dO}n zI_Aekm1uu2nW#*{8yenc4B3^+U2<&l-;k0yl~SDAm1pwF&{ixWxf+THp&Nat&vH;U z6SjcouR7EAnF?@R`Gd`JrMZb*(LfA2CDx|^B3LIqw3rsrm4^VMaa92x~_Lq z0v5n*-shjVd}qc01rGzSn0VRFS6s(ZqL>RrR00ezK6=^A;bW6}QxOm|4J!XaGtXUn z6QyQw@rQLA@`r=_Uqa`=sHM_AwUMr=c`A!r`-{-vmr9(q>{xb5sKF}uJXvB9nDiJ{ zO0T{oA0!Ow<-;3it(AAMK)yJ^;lG4!QNMgH=WyY8TxcEY=PBVpm2oXXd@9vm(4|3j zr~StHOaHDCmO(A~2#-Pkm%rWh2BMLT?!)T8S{ol>3yycO^BD4ExW}c7NwCd-Xg9p* z+%>Ia2H1+bm=-wmwLUeSTHt;ML3o06ls}xZLpSY{t@oalMO}@zznW<;6jTQ~22f71 zPizh?lKq0$sr5={eblc(dlG$!Ac#B_Ox?sW*c>LITaXlx`2vZaT7#u%0j%_?vo?F_ zv;AB3cVZfXh9)KUjm~G0o9W3!K(K!kH9as?dJGKVY>qA(9t z+JghO8p!onlrdO5Mz~@aP6ypa;APOpNUUhG&AA?R7;-e%gq>1n7Dpafz_xM27<@}| z1M~n^2O9qvg3P7PKZHaNfN8?7{_q8j#|7 zKm+4avO?aDha9xB9;#&EE+^rWo`Nk$Q`fH0Om-yVf*v`HE@uapB920E1Ihr77tViI zR|4g=aD_^%0@H^U#*|uwVE;u}+F>FrntU01{~k)+S0yE0`s+=;5m$lGWPszWg{&fP zYbm2j5ooX)c}oNp=8O(*X7>U_J4Q{3(?z`1&JwA9SiC3AXEcg%e(JV^iEo`QI*+hpV-$K3U6lm)%e^@lUSe_}zGT3Rzm?$rN#oHV_#cZ?Y4fi=BV_${Ht{8WSK)u}4y z&u21j(}^IVw$09erlQ{e8=@{C&%Yrk>}W*{AsO3QGa{kYN;gJ&`OkE%!T*>mYOZbRZ7ISH*uiq( zn1U(u#U!F;@4Zz+d)&#KySA`RuwTFXb7$@*)V)3RVo{Z%WyY}JmCw>isqRYnhBsRY z7O^mT&ESpR4KWGhpi$-=;BzE@+qYzLKT5~#nIRLi`A>K*`)5mw3cW1CzkmZ>s5BisjJ3@QpXe>fRDt`icjMa=-)p96-T}DSwowoZj`? z6U+(llBCr4>pQa-QA~Swc1-d9Yx=RNqalsnAJ@2((kbNc(Cx}L=51y8k>TBbdFlSB zzE-FZx25e4y3Fgx>k8j#{kx6IMRwy}Zub*b9_;;)550RXTV{4t6nXmJzGKC28tg8= zVh=nlTI3(DVNPcuo`Lv9>GvxlZIs_BM{QKjy$R5GU$uMdo{CozFj?;As8VOfQ=1&d zOGGeknNduI4K>Nx3hYSa1O;3CLjL^YsxZTL`#Igtkbl^yyLNF*VDPHy7O4k&p??s+ zaI_*bKz<$cCOcZ^Uj+~x>ow%PzU5gvJ&zK_{eE8PqmMt;x<%z}L)ni60?svCv1|%^ z_V>hDpBeyKKny_XX2itOHiu9!c9BCEtV5YF&qzsJNYT2-2N) zbF5_)qC0Ct(B<>bzlOAS!wS}#YIaMaZg4EIC92Nl)#vB1G%U;Xrv0M+o9tEA2vd1Y z9#f^e<#K~uxZV!M&2eih*8jW_5@T>yqlP=OjX}A!^Y;EK-V3dp^(@uu@WNVXG{w=4 z%vpTeiVb_BwbHWeXFobztsYo&(@v-DUNnmfQ-1v4y$6qhzMybE<|mNn?7-*&gV%chfni<1 z#E(7s2>e}!msh82p;)I+y^SZ5R4nAA%-w1@c*2SAFLuf1s06-!LF$97(8E01)H3^< znj%}w4T1#k(@da(sVgy(6=ZJe5@fFr_BLQGhEl~BS%Nt=h5J$3F-WEJ9;9yo7Iqee z7HvOu$=E+MZHpg&CbcI4M0|7$ptON!p=2RSv4b~`(VZiJBUugAd=4W4&@!+icL^|i zCOhU+v+|h}br7;^wf|aW$B)E>#QPjtw2u$$SvNPfu7C)c=*$bVFPt0)st)Q$5WUcc zG8b9@#G)p#4uChDPE_Sy`fd-|<=(&Mj%zN4EFE?yfMrFak4XPU~mQj72kKV`=RXnJv+p1DA43rO>(O zG9NVDrmENq4rH~a+7@pJWraUpl`p#TwWD4f%imT6<7gbVz2_RLt@{|p}8gLB2SjZUc%vkxR`hT)^jJ%F2YPAT-J;1|yMh<7SGd_1&g_}+1(!ox zB8FvH4A`^VW5*RzJ=7x$@5m3X8sGa+U2ctFD(GHedRT)Bk@juYO`2Txt{!7Hn`p_2 z=Hk`3w=2M_E}9p+^@ps2@|jD9)qNfi_FIw`N;CX$-8G_#z`j^kT2<Ab=%Ec$@9~-^f;A@wB@2%xd3q7<3ltt|DD>0+2CT!;tasAyEO9 z^D8+egE+cjpR5NR9=H_#)Pg?Xs(GzzkYBh6NDrv3E{k1i+%5uGuu5I}2uJnGzSoQmMBe>dbA;;XGA;yw9 zUB~(Uq1BT1)S&Q&n-Z3fZATc_MBAHm6^<`YymvG6V!@YUy!+RQaw(mF3#Y<_wA#%z zAT3<{1!;((*>UqZ^vG$aN~GK{nkM-4p$)BGhin9$Na?>YFb?X1(~y|# zeUQQl@#+7me93X2<{Hwg5GT9e(@IHKulb;ZPimSGHOhI=E&(;3Us-9N>poUrd7=-w z#$fL?(7$QU4Yp@%-X4L*J{X+yS}_yjSr!PJ^in#|L@hwg7|^t;YS*u%a|KQ`E8 z>iYas-pc7|aMkrt2c`A@6eoYV4Vs16PIs|N;UrS;yDThlDFmF-v=t6zml6XizUvN(R$ou4sImsG8 zNG!W_8ZqZ=JY+wT{nV$Sj`_H7t_Ns`4tn9c(va&W9K9Y95S03|0X};(chEnqp?*L6 z*IgB^41!5@qZC(6R@URqZ5a(mz2*muYfSuws~CsrxJC6!79%F1h$q-Ljjh#@ID8EE z`sLt)K?`mAN|sO^C8pkF<}9LpqQ>`1*6_Zh#s7pj8al@pxOS?EVp1f@P)t-I}t}dNa<#>$+Xu zKv0)UzvqkTpty=7Co<|@vd;&Jk_XR}HImgTY&SS~+?yo-ojLvtz8`Txl`%L?z z$qa%_1LHmIG`1m+2QYl)+zX~rcX*>+xy|eBr{OY z0qbj3D2=J;&^J$}Q|%iN0WTU2DMA2PP*VVf1Ua<>IW|#iRN6@6sn+BzY;OCRyO<<( zRMIMI@`~q)6Rh6=Y(Cp;7)kE z?*T%l_$a4;TK~pyu-t?B)je8|DsH{XisDP6a~fB3jtlAKw)^8weZ=Ep`Tq9|#;ya( zOfl8JX%Sc08Gau^juv^-f9LcEsP|LimGkY9vv227#kWG$0_?)CXC0QEYjcXUYb1X~ zJAEk)H)=gCBt^e3Kg=E*APtY5=OIW>a!oTA_3&wutND-!03hq|4jU!QV37I#Me+5-i;(}WJ`Gsy*FExI;TLJuQ-CqI~POjzBJ!$sr1anOc`7{yNsKak} zr3*9?WIJ1(9KPb*h6_cwE~yDxnLQ_Z6eEt`@SU;<{-U2gZElzVkv|__^n}Sb-mWZN z<-RMzJ^#(8CiN)e=cP_M7MhorQ(dgOa+BQbw~y%ucCYv&IU=kB(GJjc9T;5KGu^ z+^}NM?1kR)@{XxKG);Bz1HDq&D?uR&v(on_=enI_H_*q-Gt4}98D34z35^K~T$@4p zB*9=tA&h*O0A#U`Ih$YW>;4(nVZ_xKt@tbFo^?`{4ABl*BG2Jt zHNq?IO7hk#p3sDN%Xm|1Q3YDIZG*-gvq}F|n+dZS4f5MH-45fjy6pg5yfaUh6o)MMPZ)o(FaBksIB#)wag8-vm4R&MHlPbWQ;-EP}#cO(}X- zI@|6!z;g*o{q}o5=b(PHZX2zqqL?4Vi>WOR;+u`T?)PyBbkz)0N!9O?$ zOR=RZiSBpJ922=%YR?Ce0p^%qwF+L%w>KO%$C9iZqV7{NMSV2s-kLlxc9nZ03O}&R zN(>$1-kM`l`{EaSEdBRoF-xG==E#_^Zyn3m=(lSx;$$J1BU|0DQ%=`D2gd@^?TVGg z7(tcPW*Y;2+SSI1`!Rqd6dU(wVCD($H<5b^D!4O?_ihmmVsf)_^auS21AnA>i zO~NL%e)0)Vw19fIKT8ah!yyyimgIjjOTI;K1X=fjjPZrCuiW|+bam!u-&Lh95&Zg* zKDSNxdBF#NJ|Bn0_l9l!icT$dVIt=7@nEO8?VCjJ^NvL}tLG~`*O$BMoS!^eTDPuj zby-knTA<-|Ircyk@^<>o2u-9mP^r`yG`)S)W^9ATRRs{0PQ&hYKC1N&ylW>H^Q+O& z63nPuvIMS}G65YvJ#2clnlQ=&h87Y8lz5x~z55>q9#`2Iai0>H+xXfAIN<)5~#TXnXxH^y&RA*B3-G z{0p=fr0*#W`6~XJoAU9j)@OPvi4gv|{aXM#gpD9M?Wg#(2^k$T*hS0iEX}^;C=d7E zto+P0r`3~b_}@LUSfb}$mK>77mbr9M@QQk-lor?|R9LXb=H&7EB90#XQWM^QizC`w z!&BgOZa-W6xm3>@6unBA!<1}FSBlh9v(llU!hlfILz`_~P~(zM;?eyRZm zaiSMUcbpe>!QI+TiMWHC0iDys@5g|)(mA`Gj*5Xg|ePgTZ}aB7#D`D`fWuZ{R4JnKwk1^ETF2^ZQBiq2t|9>^s#^93mYTQ@zkGZzdL?_ zHideY@Gii&XN|>OZ25KZ;u^o*C8T+jRb+US1eH{-Yq`Co{O-PgauX8vC|&Wy?J?`kgjy)%@fhwucJWE?61LaT=IV zqpiZWy^}9sn|wL{#a=aQpYOn4Bcynbnzs0;HjB9+tkDAao4+Pchr%4XZF;DY*=;p3 z&ldn>6i~fg`rAM@PV2hs9)J4}#c7x@uTWLohm4n4g;}UFZJ4IgeZ|!cnx~2UW&0_$ zSKm4Oh>Vq@|9osd*WU)L;STHA-OQ6aK8$1JGVJ^*?SF(R$bD`76 zE+GyHdO0MVcF@|*Ek^fOa~sBMGIn1c%>mhZS?0;KD*~G@rs^1Sn_J|3F&seGQzp2owqs$R(|?S#R}jW8vQXL9N98uiHp`Y^pC->E0z{{|aB}@n z$e%K_Q~Tef=B&faYbn&M=j>iKl=tBP@ z<3fV@oN%t*9w}+biA8*5($7}%>dN%7ykWM?O}A%m+1O|~K^V|`73`A7ocG<@#faiy zH1}F4iEfA#%2)Ppl(DYx!2S0-*R}7`KyUomc+(MkvJ*o2e=s(g94ELFBET&mxl0&- z)FPIr0Z;=imOW(3Ar*{mQj|HE@*6wE$X~xW7pg7s!Y?iSjS8xHwHgWuvsiN0!-XsI za>e)$s}&=cU7+j1;dcl(grD7dVX}I>yP#fISv7O-g=*VcAc2nH>Q+16!Agif*_g=B z))vaMDH{6o;?K5rgO~C<>#9^sj?llAr`L0DHM`BUXfl zd^@WLKu|}%T;}x8r>6TEGL8M|(-A6cU$yCGyVLn^wY-uIw-4%LgDrE z4g!suZJD%lE==FdRCADcKpeAlK2^29nK}PjA*Ozhi|j5ANDIhw99Pl@d>p@9t!QRE zE97rw zarV#s*=@)zw;srrMZ~OqN{3(CP^`V|;IWdT1XlJKMci6XfXc@P@*Z7vji6N2AF9V6 zUv6+})Ta)%=c2JLH=(>q0oyrx3(3-e<%lj(CKx0{WjA@NkfJ5`bqlJRs=~#qi}h_i z9tx;nH9!25_2M+h>g{d>Yq#7XPa0ltB^ZDDTULUVQ2nYI=4Ud4b=wCy23&nnIw?v&9(|F^s2JMvI#wpU{g;ya zI_m|u;juzzTlw6qCx|_&PSehDf?e|?!Tnd;kv?8n3wsAkdDeU1E&=~9uRROjuy&0o znnA(Xq3|!2OH|?RnODE~`G1{MdfbNN+#8I*;gmE|BW0o*>>}k+jw+^2cm0+SkmoTs zgZ4F6Pa=KFj&FvYxGATgl<)D>-T;Z$9Lz$te?)>5g<#8dUVW0ZDcnloO&(HOpIPY^ z0hgtP@0++Sr_ojQrfP&tLsK;52%T{qWh zhjSLY{eThN*u|X}vg{yfVU5421tnD}nwFo-1TY5;RM&Ymc~h470`_)lyKPLbjXXyBsy{}l>q-P{nhshT@GUqOckow(abT&!nFx>K7g zEdJLw#_=J~wr^E#ef$fNI}%kH3e}1s?P2ON$Wt*VL+3WEb4+O5Mq%(4ZVGS?hOZ!w zU!X-{llzdub^;mZ6m`kI?bww1YwMhZ`voF}!igZVsaT)NS}uH&ghkdW;ZBeZ%e-ha z*A6#ilC+uVPS!)dBq@GB+1*NdSD?~G)h}nHuI;8p|7@O);Y=aHyn#*dPNZhFcCvTw zFy{1Fz(zCm^t>nApMxBF_)~2keCGVLe_t-?1@$FzWJ&u(2L-=LIPwelSJ>+qV{F*F z_Mqf5WO8aqE*ijMto= zU)USuzZu&Ut=doXHS~{qg*nirPB-7w;@w^iwDkt16Vr*RrEjBkcD}rS6)aad``=8& zN3g_qVD;G(?IimSAeLD|`-?r61HE!<^;nU0UlXzWV zlS)X*aeCt;!6KBQ0{m(d%tWA|c}itY?I;-hJBw0%cRM7179I84!#pDu5O-CB5PO9( zXmO%P$f$lBwd!H-q^=EM=T|&*gKJ0^NtqjKQ67%@u*nBETo<(VYUNXjk%gwbI$A;m-)nip7y?8Tu>@+u^#g zafH)fV(O87Q!G6Avb$*^W@nFnhsRN3-%20Q@5#IHF*`)$qb!i#4vLmC|wu? zmH!&JbK39?gS=IA@5|4Ic(w=Xm?|}YvWtijY{NhoRl)<^6-BIksg5?6u%z|TYgo4K z#AvYH%2m&kUzpdpeHqBZZ+o94vRff6H@JboOtXZ)>6d~6pk8ENB9mkI_r1?5{ZXlN zix#T zhc3XqEewNS^d)mGK6m=l{{6(|Lg_0zSi#G8S2xUMQj*w`(bfiYcj|!A`Caqhq;#tvo7Dpruw$viIeO{8;9um2%hp`|oVW8c74`jpx_iR}!jz_C zRyR*imH?e$@R0AV|GO=M;Fg<^FVeo-4g513WpoUG5J#%~$UC==r*!WGHN zxXWiZw{EUS`aQy8J~@4SHKm{E`#}FFY>C%=v@aHjduOiIyZw1A(0@~@n-`J~{sRhn z=yURV!EBV|i62cOFcx!3(3~dKvW12Zvg3D~g4S4o^hR6?I7M9|!uG3}d#_0n`g8Gm zslxMhMq#GG5oI)IrhLjh#4ZbftoT-$o@U*OJml4iiG3~Xz+y;3e?eePhykl_@r{zn z^Q|%k)iDckNvDf*u(e9Jvp)@IlLwk0E9&|B70yR2!Ed+j!8}Z58D3Bwm|B)-wY7fa z)z)?Rd-f#{G|Ec#x@1h6F$)BX4f`(Zj8)ssh=V$xQYuW;a1qLm0ksIRdB;;<1M}sV zl@%Jd3w{(-??pXfevS9GaffMM~{`4jBBFf*?%~-V0UX zy(9IiW+Hsiec1uwEiA8@+~w@Y6^g&?NH1B9_jQu_4>i<$EwodV`sv2|FNUsOc9Y#v z^K)}a50YVcAFgIzx&Izk(UwO}q|b4kK6Gz!P$=EFg5=BZZ8^9PT3*(sT$FSg+9#2~ zqua10$&14*t-`7tN!|_5VOmN@T^&`tZZOi0SBJp@F7nBXVkv^ygjPqAOOm}{OE60b z${1L@f(nLFaknwYdg$GY`AGW3SXjvVH#3Mbqn48At3ks z-Z6+0gU&_nHeA$T7b+~GL8Jp*If$r-o=2M|`%!w3jTylgj2^B3HF!s;q!Gz|%HF;; z&&7FZqe7T%ca=XX0E)B22!(K#809r3`sQ=^wP55{Hkj0nje;Fg+cNy8S3wWyv8W(JvZ7x$d>d6dx>< zU}X{?BFvzh$ivCsdn4~~1H2Ki6Sfhu$}Fm@uthB()}8Cb9Tti|k;mR@>T@{fxtgwZ zSu%<}Ygj0{PrvVsqn!4~zIxW|)=^qsuUaf?w&El{5}|9%&E)+50Hi=$zcSl-dy4VR zQ2{S5o?AGKQn~P^N%eU?iHPj% zkK8nFyKP)G%2t`k$TF1t%I^7${P~>LEQiBdZYS)JaAWNcu@xWVx{EXq{Zf`S>Px~t z2hM`_6XkG<1u*uVg(Ux^2q`0EeXH-_Wm6o znLa>}-*}Sb_;g4`S+ogl1C82XPOi4>BhhxLkoxCvDtvPj;!Z8Rd2)=t#Elf{5M`s3 zi|&_Iu%c9**Lo8ao^ebhM6SE&I{F~YvZ~E7tx@nf3w?;9-Q8M(7bLI*+juoA{RK|VyK zJjnVg9ce`ezvpSQ?oHu!{3Q-mqcAhW8`*9Vs6g^!!tE?>#ZiU1Q#x{dOI@*mKX${iw~wVCMzc>b1Ss zgr5878!05uljOdb_lANKr{k9=aagMst%ZixEY?*&Wa22TYl&pu6N-ORYb6hL_&`eQ zH2RP8S6FU1kuTCy&hRjZ`cRi$bMWaD)JJQob*$14p<2iN!{4&^L%hTp>ni05%FcQ2 z-`qD8FT7I!(%DF-aOXg|A@@EV@;0di8xK;?RDUC8?<^eL?yTH@s(kWXSRmn2RA}`dsB* z`FGu%zw#XV{j?V$d-`ggAYX6Hg;$aD46i22>-sCL`4gGTGGphmBJr@;VN=y}o4S8C zf5jvFg8m$O-q^_@rypD@>cf&DC$VKcvHG(6D>6{nkSX@=tL6;(*&?*OYzQMjnEEw9 z4n>FJ9^MJKXJ9ZD6Fa;%zd?B)K>RDNE>4HL$n}>jTE^@rknKt7+z&j!;SjQ}o4sla zWJO-1Z>vxE(6a8OaxldH0y{b<{oD6ER1P<&*^A`R`jp?GvkNc^>1&y@$GQVm z^5&Q9ckQypxt2Zriw_6--{A*vh?lVzK47o)a%ddmUj73Vdar_kGO(~D1NvUeN&S93 zBxfAdSm^OkvIA)^+D7fgTIqE)^K(q*ytrvcx24)}@I^Le0UpA~Xe*gVeJD$R$FB~b z#1DOZP>omHCtiL32C6bBGW<^SF=V|W^9bXFwbE-MWd7-YFsp3fqw+^r>IVsYIEG$8 zJrZ>-t7O3gAkkweBlo+!F1Oz(FXY29WL9G#y6r@iQGYY+-*v@;kL~5Z#>)TjQ~f^w z3%>B~LnltUau#$8*22xCV}(Ei>9_p!PY=Ozabb`LsU}=EHfH?oM_l4`kqk&$2&x8) zIk_u*#52eQ^t%zwN2SZOnG#fJb5V z|A_DX-sRHI{LJ#M-~GFn`#$7DmFL?kL~yjdJj(CLYhL5upSpYQoSgE+zxnJmWem*q zo3P6GFqCX`v8V>{U&Cy+EC(FEfIR>p+JWPhsi$Z;+7GV=+|gu4v-o{)nd^>>d0ktQ0C2Te8^4+-0; z!wwUI`eVj3;Ue4>*ZsQsxesjkte*KgzT78;YWxP&agmg``;Kv-q;OB7j2-cBUXu@h z$vgRH!GL#7hT`3%NzVu}p9=`N%nW=p9R^>}s8V60{!SY3R zRWwMslrI^TyjqsbCqy)_vbr3qVU(lXeiEqUWL~Iq*I%X&zTkl{(&VDBEPg!)j(J67 zIaGVHxr%%+Z&w}Giu{dFdQ-Agp8h16_+b9_94{I8i7VuwFq8iHog}|Tul!jEotmO$ zlEzODs*PFv*7AtOO?v2g$M1E6{0Yv2mbXW4)DM~RrU*(hKVaGqL3op|HudKORDKrZ z^)LV;uNmfHn8dL&Y?<@E2KMKIOkB6f#)oC98;3xC%BAW`UHXQ@9o3H;1ihfcBlYEC zIXO?-6NTZv<)oE!4&B(uAoEfn=m%=wCM!Db87ZV!(Kl}#PfUUZuy ze)7y40Og;JfXecwAO|nqwv^%aqg_-!n`&gI)+R6gQ#emC<_iw}y_muNS5=Wu6iK(H$g-!q4JVvnZ>`QcsadQ5WbL%CD+q~M8D zC&b%*&Z}<%96Odnpq8iglOaR;LB?w8M!Q4~d`|K>`8#bCNAe*2Hf<}Ovnf(pj0c$^ z>`~n4zto{WGm(B9%F|8o_`LJ<)2nWq^$AA^IX;A(RY&L*uXy?L#LFIE9((N3<>cwp zixO#ml8j5h{1zpj-MDb+(s=TbC${i@WDhY<$al70hdpUF88o`SYxbbS2q zwdruogvZY*cK!7Cz^Mn8$0g%O9)5Uv=%F*qm;AfGmWUfRY}oLA!N2&4Us(Rm-~Qg{ z4VwJW8?q)b7p~@k&Ao^@5<^!a)%S{~NCd-9Cd zIOQj$z=umb7~%(nloxbD`-q1heKhOfVUXT%{+;hyo_gwSS`)R#iJsLucS-*6;Ww_) zBd2wo8lqdEcl+iuYrjnRwaz!Kon9NMqxBA@C%;+;=bJcWnmi>hrGMz*v&(7K@3iX2 z&1D`W@u&KfCpxYVVMN@p<}i)VdZ8Ctw?xOJ&lIuuK*zj(ZM}I*yX5{1zgcy1Is33= z`_O6YYE+7}^tR&D59x4oJAUw! zy;1r$x{hTBZ!+=Ajqm8}=nJ9S+{?<+1SEHLo_l^#Jg5SVF-&C8?2dv}3kK%g2;yiN5(Ea_^ zt>m-PI_5yeS;m_T``9c$BLM^vYBPOLE}7KPW0Crbtb@a=WPsgz`E)qddjQ(p`w{xG zOv{z=K)pQPqwfKaS!5}bPE9?0AglOpbNZj!3Qn_EsO;G-~InBm6RkdXLecB!otrs0`B`KI5G+-sMFvjp=^NyrOczw17yzT&-(!^0_|tTHf1K-%TF~ z|L%F}C#((3ebJ$th~tZ|?oZ-_u$Gg*7$Lc2pBo~~`BgsVm%Wk4clJ>|ANC%q?>%(= zIEYunxNMB@f@x0TCVzW!8HTXhg2^qb>sIz1P-HHnU#ztHS!}tKIp;s|HrrjWBaGOc z*ILF(_8(S)*;#eTbX{)5hIwkL2m-ab~ocQAWTX(K17 z?C$F|e@%Vgb;Yc7hVrDAmfN)MWSpljMW%{pGjssXG(5PBq8=0QV?LNGi>GxBiC)p> z;92B+M*h1zgkN|J&*3}fyoO@_|JnQVfa|iWtQTEL^=%rY6FWc!f^cjAJwU(PBi9QjfC~0{6+B!y3Q8Z4URnbrA=P|MRee>dO5Nx8 zJY&o`*WUa6>Z^+9{Bz&2_gJ$ZW6UwTG1uH{ZR-SjLnrf|LZ{hpxx)|6p;c>DuK}!W z#)LEOh-Z$iysMnTmfdb&OO0#nAyg5;k}mIY?nk$qWIr*xdcl4G)L=HoB&%94;gZ`oueD%K z*&E<5L|28@n`IN0jF<^;{_i~XMK~o#_3deVeDLPiE(cOT(_W$C-u&yo?kA_N3gkVB zx-?eQRl!v`sE_HyyQkUVifi2w=Cb6&G2jj!)g~Rb-+}Ve#<&$eg3um(*04O`>B}BO zi_h6~SEzLIF(GN9&;YC_q&SFqD6jV;uKFQ))0fllec-`Cj0`9aWAe&7Z2%STaj}jC zL-8a|W2dplQu-@XTMD4V9CnGn3@|?>b^n#^%`$qxM`Q>+BE!i4IYkTVhB|E=29n$5E+}w}>I}cOJkOh_Q*S4rVdvh@Q zLe*=j%n7iN0^N8?Rq0K#=18YLJVXUX^$<{149TS!*WT!9DqazgBIhV^!KDE zQAY792isL7bhU?7x6xDUDh7V(0UDy+6!6l9$eGP?WFF&78Qhi&&?YWt8o?O`+Lh1o z$I$ihC_tu?z2h9Qz@%N^EaZ2AUQz(hdj{I_5>6Q;>E`o;N#= zK6T42H*L4`wA7uqZ%=*dQ-pWsrWiLgZo5?*OwAjjfzG^fWCM5UzU|V*%iAUXH2qRu zvB3e({hHITh5Rc=^$U!)9v^v9N_|^0(LXRuxxv*&qU(f=YK!!(x)DO8(lC_SM6yx5 z_d*VuvWdKJd-%!&ens&B^rW%EjfiYk*%*fp`a?f$V0!E=`U+HA$t8M>ccSEyM0(f5 z{ZRVsvEFg!hmB%lCU_D?>e*TLfGmaGT^yc)xGS89ngFf1$;T!19(l9cU3zEJe(9p@ zy%T@qbq?on4(IShfv0VKGeLe&KZc$RV=s;PnZb}L(3d3D+!tyUx3~rcc45H?<5?2N zlq}Eb={d1X@vNz~eQF(8Z84{l*YtI1EM`u3K5~^SWM21u1g*cuTDQ2#OiA+ihOJN@ z(>e(c-0vUn@S}^%av#u~zCJ1C^^o-zzeSodto5QkS=TDf#Jf7IliZ+Lf0PpYkejEz z3C#f&yv#>+pkM*CjOHaiD-ZIr4y#;l08xLHRl1WEMt*K~H~l>IC1`rKH;_H@6JNyz z)5JL9>^e24|8>#f-z94$pZ^Yj2!&i7Ci+?ny{^dhO*_&?nvgc^K z-3`1&n9(NoJEmNG1?`dRoO~&RYuUQ3`yeCxCH7A10aPD8`jvgS+W(W+eZ)-Z%h849 zM#t6x^rv3aN9j*4GW2u^`&i0YaQh9`1umD08}5WX9YRY?ldAYtwM_=3vSPV;n9^x4l-uUh?77NvmX^9L#h1PaefV+s zhp9yw(4TxJKPQrp{K9UAjGNl4`UjHeCpsE?SNc_mc#muyae!8b>>L`h=R!{$bn{jU zeo)YJ9=bKl5)&A*d7Bj(?bc^P1TAp`owv$y*_R=I>gjtVAY)r{1WP8%HE{<;d-4Gp zTbZxz?$XHYViTHn*8v{+IxakMojk({8scAL3P#jqxU!Bg<$X*zLEmtSuj1C|g~zH3 z>x#Zr{VQ{GA>+>Dq1J#>nsrj`1A3mP4}mq`1&L1{>lbXHs5bneOFJ2BPrS6BWzw9P z`52k#>k~U1nJ1l=)c{}jBV;KVH8#B73mMa1V;HZB->q9B6IsxOWU&l|2mIjZI>Lwg zix2d&4wntHBJr|wvKB8o>@}`rXj=zybS)ju#&!IHPLpRB@0`Gl!~wXkVu7;g2K$%P zUXYCu+Bc(xs$~cjX)7F*Qv4Kx6cghiH@5K;Cv7HQwB;MaG>&Q;x^^GQjbphESK|dI z&uSgJbR@ON*024tUyY=eN?88fY$hx@h%1>Qr~F-qkc@u>fZ|zTxe- z$Q?bEYrp6fA9tI<7ep4a$lrX2x58%vv?n}#pEgGiT;&xtXOP=PuYt4Jk9c|T@5&rT ztgx5&7|trU0Q6~lbY}jIIJ$J)Ld#tH(rw}}QS!KL%RdqtpciM*UHr4{4xmUHOm~4I zV_d;wtov6VPBAucW#9V$&9_?$er`0& z^%!Sts}T^-^9Fj+#<5=^n5V1c*8F1mEd$J(x6nM%QTRu1=n|dDm7t@4T?J<9tmCyn z=N_d-pIO|I_;?@_|5#L1_jR!`I@bbAfc~&IsG?`=QJ{6Xb^Urx^Kg0 z;Z?ze6K^>kK87=*1!zAqwR;4kFdi^DT}SHHx@3N0Yzesel3zZmF4Z2ih1;R@0ueS0 z-n@7mhrabDIv9oSSKg%0enmOj`k{v&_Ms5J)a77^ja&YRSm*<({3_T#%+dVS`t<9H z1k|C&0j~X{t%8ptqx;n(ZkHH&bDYt~u&?;T;YK!Q+^kUnRc9#AbB@P{#>;MF%v`7B z^o5MK+`vlPM`sZ5#Z3TN%RnBvnG2E(&_bzK_rFHktm~(IA#9pE*RTfg?-B=y^Fhuf z#!iQH^*3(E8!MczJ`Oj`d|YV{HoXsAd1$-ue(C%2!%jJeb2x`{c#A>P_7$xubJIR2 zqs&7UCf;e~me)|h)ivxQ<=ESaX8==lP1BDJ$=K`HK<0kd0{O>U=e!8e^>k5z;`w%2 zp)QeC7u#8^r9yeU2BND`rq&pEn8hhz-R6d-^vwS<^1mq9Jgj}|kFDUvy683-GNm3Z z+rCaKpxNcK&|UGcjdw&=f8}G%9luy^8YN@s#O)hZ=xW$E$Y}kj4rMp32MVog;{u}S z&k>x6I)-b?bfCK>gqQl1ZQW0ea8C)l%1@qQ8)0u_YfB##qE4~3gRV36cikkDbrtDc zw|Isgty}YqeQ1)JC& zci7lIjH-hn4vkvJwz(O1Y8$}%l3x4XDSHKMckdfGY;`|Ze~~@`+LJcA`Ieh}2-QzD zj#u_^5XB)Q?I=0OZ+jbFs)PHaXfcl934M6^@Jqzrb8lJu zvHt%k_d$;-ljx?h9*`#_n zXB4FJd}$B&m!7){Ff7=w1 zSKQ!B|3Mb@@v$bcmh^h!zMnP9eGEh{XtCCp9Da@A!mo8ynsK?o$MiGya^O{mPmDSH zGnY7B^Mhru9LOLVmirLl5kA4is|)DJLVrk8XaW(*+i7dMXAgww$*w2ZOtPBvjQNiR zKdqsBesV))Mngl9MgZNoxuKgI#T*Xd&L&;#-n=9OQ4 zs3sipBXhd~^A+7j>aZ*ji6>uUdmbBp`6E7Jd;MSis}&L)6;s4v+$qw#xKkN`HC}## z6_skJJ_#N`>0#F;7g*?=4Q3$S;l;nSDU(@GXea52FKsb!jgwbRm9|<@04*bQi#D!= za9!PwgclHY>e6O*7nd9tOytM9Mq zallwaR`V55t`VRvHZYX#dWjbKm-pRG33_-%9v}n$GGPl#1qLmw}`KdAOFXc=N00@-}Mdq z@I?T0hk<3RdCT(}{j2H^9>8(T69oh+p#UrT5KhFcGTF-|7>fFcmTzRs&HNp|n_X^Y zY^N?0Xd^GY?hmC8A;&kp3BuUSfkNgxt5kVdwEapBwPy~evZ?aMO7XqO`yfH-F?4(L zowi98eHq|3E5N%1T|Z!7Lg2c#K5~HIckZf>_QK7gP9Cc0|9p}hZK&XSJQU@SOMh;? zHtHh%AV=Itl=z8>0!&pG>Sd#h`gf(2r+>xkwOfd+-l#lq#cMe0IeD7iA()jeElGTi zFchcLVBvzRo==vc-jIc^F!e!))1b@i60;E_8-0dNO4}p!BgQGb+UP^4Yw5zoP)(b`rFOH%I<9NtNL|aQ^%^W39b-6r{E~-HNi2xeLuA$#IKM1j;#c@ z_Y4Z1>h_>La~Sxbe^q)8=Wq_^@IM0X*L1=fY&SWJ+U@Y(l=dX z6Zqz+O9@U@MESS6*~(j8urdA1dadhy2sKgRs%6)u>(%RaD1*i87#M8EZ4PN}<*97-`}7kMeY4&_IFb&)wpl1c+t}_0sUx;x*9)1X zBlHx75s9OZ`(o)DI}iG{3DG-y&DdQ4He?_4_zCn|ZkAvBMlqp{9_m8{;gDB08M3-x zl?)lf)vC;)P4w1)hu=PadTYw;b-XrU*ozV_tWkt2T5y#&tfq-mfuU=!f(>>BX;B)VN{% zw2spM)VGjX&t{P$+N9wZtF6o?7e2>)JrpvvTQ=#I^2d0NTOxFUXwEoF>2e+WIt4&(X7A2}8ta zin>?^_kE#dVLpfqK9KTSJMUHbHI<%g6Yl#LV=YEzJ@Q(?wu z)+Z#iUZ5%7tb_hRg6E^YCL3G0-XG9*CVa0kWXVX(ru7!_(@j1r^1hVBG|ckxx{ z$t$=fswE@K^~i@Dj~eIVo~y)=*Hb$)dqCA`jVg&FT4^!1ZaOSi+SluF8@oz;>Nqyx$(d+puE9RIh9b}YyvRw zz1T{pEbcy#BJ9OB8?lL6eC2xc-Fn~EYx-NR5jT>$5-X(7f!@rjpK#M8haoKJY)1Lt zrO0pD)epR(l`L#DVn9*%sHXMmu|VJJFdt_~K3_V`(d8!tD05s@+5o$Ry267pxX2p=Y#L#YFve*w%?DJ@5_}J;v|Q8s|uR5*Lr_$qkw-nw##| zn#Ai(&fy%+;T--)05=I)gU5PWvrlFtT{B4)F5!k0s<@ffS*APFC^LB+gQsFo*dufT z6kkc1`{T0CG2h$%aC{IzRNv?XOKy^8zR&=&-evvm9|f@H5k_*`QP|kxWkH}kn&w%Zr5mAS?M<$FXU!8HVk_@;_}R<6?jMCSLPx-h!b)dD=hXfL)-# zaY;B?2AC`cIO-sta&32UfQWs=&FQvJ!Q7~e&4RwD4|YF$1K$+vzGs51r)mcy4tkC~ z>>HJ(XY^-23fsm}9@NwJXW#7ee-_D}q5_ojlP)VtUbltiOnWvxM2Ld2$7vm;84{={ z!Av)F9_m+^zL|YSux#Yz#w4$y_TiNLy>~jiN!d0bg7<1LqFnqJ)L!{c0qjlA3Cz#uKvM+l%L+@2W|Cq5^3mPnsI-otvJNV!BqacxBvgm z2Vli3hhO5O4{8~EQ?(y`uz$S9!ISqa7cXuXE(q_vxZQitg}ky)@s>k*Er-gcua-Wj zyW6bikIcR7zsCH{r}B|L_Ut`oI2h$1&3$l@okEtId8`i7(2-=p^f+B1uUedSPsM{XmwQvxm-1cuyTL3CJ~l>#(r+ znSbY9=`<;aD;;o|3RmPv8_)72HypWvbb@QlXxao zKOMy*@=Ub+VV)*l|2GSVMOwJ?zbq173JqgG%25MLQ&E;s+Vjm^hs%{VrAY4HP*i21 z!wnnJnoolW@k=%dST6ZWeAfmK{OAZp6WZ!1+(`ES>41a>=6`7XaGW7f^vVf}ZZKhAyDg2~z9Y5rRnx)RxCHF4_z|wS0ZmER z%*~_yLS52!D=dT0{0a`U8Avar@6eAhE*OIvqdXz?s3w_5>tKolA0JjJPx_jQihsr( zSmVHM`-R=t0JBR!wI3j9j4cI9pDr=VDRYlq3 zzUXb}4n3i-#CeV}{m8k8S&ys1aW2JlWb6Mp)9E;Kci?e>u#+|T85C2t%~lL+enV zHm3N<55x8w^{H=NM6O-YZ2h|a#iYyT=1XpNc*9a_w>P2clDdGWR$ITVw4@L$`c{6=>L z;uH2*FwZhbJr-)r$xa}SG4i11B|nw!AOD1}7+YNi(PDg|v)r&jXPWO6h7a|_7!MDj zXu`uc46AsW1`n#MK00KHj&yMjvlG9uSx`jRDYPI~v=&@_Ne!(2unPmHN;yZ(?{S4*1Gw2z;YJFGx9g?# zWAGb!93Yo@u?~_KTZQ@8PX@DAvqmYM8#;3xb+O%kz#zNkP4{Vi)>_RCS=R5}lTupu zdMz6_7E3=S&!u}>$`k5pWG>>GW)L211@34-Zw(o@?uFtYxJ?qWNJXMPRd_7Fh(_=ja< z-;u)zctQ)i*{?j~(9>@0Y49#b&7a5QOjF4O-Z^_+A7^fiq)t>bzUt)BpL ze`JiQko&azSoLYy?0!OuLn-YiaJ^?yy7vq!M_=RMlLIIYrFc3j2U53eH{ZnnRpoGJ z?kUk{e zYmW{<@rs>4+i^wgf8U^I&r$mh_dg#@(GTn9yyaH^6Cks7RDG()QQic>4=0gJW3%MS zUfD$pkk8+e6KA@Pt84&8tEWxqyz-QTFMa|7%J^gcnSVX6)YxWhSZAcW#DvHZcf}(e zk~c*JMa2%6rYhI8mb~SeWd*xHL(y2%9=p)sd`9vTJ#rqvK|H~;vJOBWqP*xQeLZW~ z#F?&dK(x~~>06$I<@SM;?}a$N+OgWSKP_Cyg|6wl?}uj}jQJ@g;W&`e_-AbTAj&Rr z^m&a}<_5+GKMX0k0duWB)K7JJO=moU_=R4Z7zdJ>erEL`i^p8?ho9Fd_!ZW83%~F} zH<>Fql+s+zy2YUs<8QbRW&PD$(sCe!?Gcc9V(2d-odyz<@eN%7sf#qI#~RzhxY_fw+-){q~=K^heT=hiFRecW8kU3=mWMhg+$KuE2Eo zil##j#x*ZzDi^5qK_O&Z(S+re-<5ZmY>6i!erNVKF$IU=Ga(XLP9!?%+Q;QInEm85 zWLlwu*CNq1!3iECB+nD(vlN@&mDW^3smO4spT&D7>?Eu}oLyj_?*Mjuy0HpIXrDeJIJv;n zvQ4>+^7lN_kLJc7bY99bskg8NAss$b{$S*<$rOK2tU{-gPksQIPz0YHZ*;ZDVg4P zn>KmS-};M_mVqqemm0Mw1I@Da5t&q0<@*UZ^qfsq7VSY$=z|n3h-{j2pi+k_S~UGD z4;!@ERQgpMh4x3S?nmZD1--YhGUCkoSvYtZ?uK zP#k%NT$0JLB1jg$a-%ji=&cR<-r(5ikn3^b*EU42emeR7yhaY%=yvVfHXc;>Zd$dP8?9UPjFAF;pxgS6+w*FP+qGe0v()Vk-oon|M2~l$rw_E& zo+NrPdU~^A!$`=VkNcN#suMSn9+IKt|Ly|S*LcMQ@qRK%ZvWztr?S!4jW=XdKK~5( zX4T;))Ao%Axf)6~9^KD`mh+a#iEg@&&}ZPq#*v#^{`Kp_{F;bg46%VsKG3{rJ2t9r zM{WS*CV&si&>sdLh8^u+eE=TR-LBgyVceT}O$-}3`jh$uW6-Y%ko&Z!K4p8_Q|{RA zyyK4Tt)BkW?QP%Yt+(ItJD$E>|MaKj=A3BZa=?G#qBf!TUfAxs=iaDa^3w0zZ)GH- z1Nszg!P7v|@BMx~VSZW3E9KK3BZ_fUVQo20n(VTa^)TH?yjhw<^`8-mi+3!nF=w)c6j zKe}CiXUd+#Ih?~eJP~+U)9Nq%(ywp-^`~C5{mQTWlIOceHLciX9t4>$nE*QsuIUi6 zOW1(}H+kbamJ8p4>1*)J<~eZ>iMQ9lTsaOsG*7bT`KF@~8z#0&Y!;Zae)LTub0tn6 zY$UHItf5+iZBOu2fL~K}({`)y7HksHggxsw^E0&gkqPSwH-B!s^NvkM43q=+-n-rV z#=EzB?!ITcTkGF%%YFSD@3P(K9}uw~Ykl$>rS+INAKEbIyI!o1ta0UbJZrGl%)SAI zAGumJ93ZuA!%dyl7W0~`dQe~Vb>~y=-0rxO`rckQY;z-*e163jb`JIpH-=;j@iqZm z>>=M6(wfTp3LpPCMd|%RI1rlH57>0*58R^Rs+=r~FF1WT=gz1afqvfzEGScQ8UeMlcfBVqH<8s>Pw5{g^%SH2GXO}H1mwwc*$D-Y^ zTj_J?o;IvP*69t|)!WlHGen)ffAnK2AN2gymAlwyQztVZYmjHV13sgB-|wzwF)E`46gY=p84A za*d4j?2FPTJif$>J_tY09YV%$^p*Xd52IAi`+2Lv|^&&iLo)Zu3dH8ZLMLm5;RrQ5`^dLQV#Rd3MZ^_ntkkN5L z`loy)19i{65}p+6anc8&?t7Bi<*KdC+wClr+hYiqv7WZdnD4RV_0RwvoY1mN9kx4i z2--)B?xD-XfmY?6h27P;^cXglz8iJf(tY3OdNA&;*R>)q9MX|m+OEf zxRfY&!Y<}4680n%mb&=+=yu!FpPDD7B3NfCVTE0>em5aXq*7B6gWbYF58YY9v~-y?+i#JHDHxtA-z3Zn2RGOqu{%Pr^TaJrQf{JMQZJrAeJ4c?2WF+Lvkq^q5@6W{p)Q#x92({i}enC`i7>ITonMUS?Oto}@V$ zmroM72;iw}$>kffax<4OVfI9s$#Td}T|9rtofRECr)-SdU<`RM3r7%tZAO_-$ZG-f zL;?C}dGafT$OyJ!GT5t_OYZnD^GAhs2@V|`z^u+=z!~Mwgj4zV%p_rQt@2eByQ+se zcQ#?FRb>Pz)9pI-2O;alZK8#njlEn5JfbuoJXO1+V<@6Sj9PA@+c;m)8!{QdzDb}w zKFEmP=;D(`-kL#Emg zB9~-DCJ45U=ro?=O?%OV~ z7Mh_Dt2p@yI>zxQz|ozS%xl(oyLPzsQ3Q3ME{xf0k6M&7bDwQXuYK)1j#|=Et-#TwA*>aDzH|;#(Rn1Ui zpnFiYjN)fKsp+h%`<&}E>*csqDd-XtLLsz|U_`C_il?tvgGssF_$u7dhDx#1x%8o) z(#NBZqML`MXMS;d{=ZV^a1Q72mcauLJ(>ds%`0OvTIM9rv+ZBeb8n}W-vQ<-+!c-@ z44FGmrx{KpnN!EtMDF#b9)I&%nCHlx=0whQ_OJn=AX-Ps<- zE!!vc3R#1haA`aMMJ3CLw8sm@6#n6sI-)ad5I=dNw(}2KY{TVH!nT>}NIpuTueRxD z8$cYrMGuGenEgWA>+pM|Z-Vy?@Oc0NCFpZwJ%_FNp&9z4ZC#E&cI6=-NX7no&@y*h zq#p#u(iosEql#|x5q3z>5#`12C|%O`y5FSkv~kao;o1=)}6H z@i6)h{RDdvK0-7|y{IDxD}K61bEU1ZdiQmzDpzSR9lj>~qGQjEXt7+R& zrVD~sg-LJdUheD>6ZWAA$lUrxch;};$w6m+#K(``^5eZeOr{KdBzqD1M;%6$t=^e@xYmZo@H-nFrv9$C+tGa^MF)o`P!R zNq>H5#%st)_tQs=FUFRXgq~Vg&`#Ysj8mVHOJC@=K>yUW^Lo5UHt)qHXU0wSJ@3`% zPriLDCK?ywnY~T+%09%G+8pY>UJtK1$+h$nHsB>AdZ2?)E+YtkV^ zwgU+d1>V7=veFxVc~0?JVUU*xe=vF)X*;O8;H!FN{y}G1$2{lTO<(GJY6H{`^pT!- zGY@zF$-3YlvN&JbZr^gPPhq#y3C#V&xvsAFoAQH>y2m4Vz3--6&ttBkxWe0V_4+Mj zEvL^5M_jEX?sFCq*`gQgqh45KU^ zxbpx2|MW>jK~#x6@~@RWWRPn*eVR}oDR>%(G;mzPZlq*RVarmuo5=Vi{S=5w=$Aiq zPvelK%D4nYT&}4VB-t@bY{|NyE|FWlmiYjat1pvgK19g1JouBo1IPUjgk$mp9*fVl z$}#g@Yl+7PJE{bPd!C7yai@VM8ExaXN+adE0x?B*p$!gj+%D#cgV-UCde9gPxcQY~ z41F*7V?3siWkL>N7Sp*%WWr8g8~vYR-S-wqDwob*v6a=bwOo?J^MfWtj1iDWwwY6G zoHUO4BbBp5MGMSRjHWRFW?{%=UK6@;hFv$ySi%Y50WWyA4AEI|9h(NKCd?cWe(NH! z;(#i?P#JJKZdcmid=-71y26@zg1VEDweyv$tX-^LQI84X}5B+A7NIPQm_u&rc!--+^$`BZB8iOIm0`1B#R+hE}3-B51 zGTLd)Q~nVpx9i>Y>(^fJni)!3;)>+3U+!z=n ziynafd^5*wS{ok@_-;sBQ%`h6J^V{(Hhj{34%m3#8i!KcWXKq&k9qS}hdJ0Q^f`3U zdPgrF_tHcEBHOxB|K&fdT6dlwgl+?5o?=_BBcvaWxs>qgpJhA_B**U8g2fNnb3igT zx%}EwmEY8X=FK^^d(V9~=!2HtFf$0c%?4JDr=ra zBdk*{2()_&ls2*-=l8aA2KWA}K>8k@_!6q?!pGy!>m*O=J&;%UNzLbQ4(D(VPZX{^ z_<+`(2eZ~HJZ2&0o}M3>o7S)qh(Dpab`s&?4|LEo@G=b>NL=wGB%zZj+v}9q2EH}l z+omh*wXNs>X=}s`-T+uQlr-U-0V@B=}=T_8SA{(NyCGAf|?%?a1)lB zv)rido9})~0)MTk)^V7cUdtV9K84gruG6?-+jWraUY}`C)3hW4@{0y?`vz5G>!`~| z1xDL)Py{F6ncoRzs|d?(NnNTc(D%M(>dob$*(l9zo((sw;H7-Kwhr za{Abo@*&Yf^&_F>MwxA4p>4UUpJ?RfWOPV7xQ{^+TW#o~Mpf})EeF|y^nEJqbl(6M z`hZ2{0NTj5CO6pqM9huWA@q47EeBFO>652@xTy}E-s22=inbI6X5HQ9eC<^H2p5b1 z7yQa;C-v>|<{xYEmH;0HaB$LxCpYwU@7c8=l~#I zh^+Lp-gj8PH}gMIJdsrgFSY-StfE7ImcH0q_)QC!?lWHIKuE~kL>fAW8+xYgc(Tt2 zQ@7+`il?OdNyeLQYJJW?#n^cg-hGL7xLOZUg4M5_2spg zeqA5)M~xSQ=(?Sx|I8z4hahRFhEGIbev6JYW_*y#m`NW*H|*K{>ayxjmRCG|C?!4g zlb#&#<_X4rT1v91UY=J~hH@5h*F-G`awU|Q1(ct1D+0)F3ijXZ*YZ-kM?_guD06=QuMlw zW}gRp%`W~WOE7aQW7Ffa=2;(JsmyiM!?cSJJl(!66YE9py-Zx!_4GeLRNR*=L-PUU z_q<*5l84+=C+HyS9#5e5ffVyaW3RX5X?w!=gsm&N5!`a%pZz|uNdXd{L#A9zU%o&V82L)3{pbvEeAcB8Z#uiP*=ThS3i8bo&(S zJR>lO?r6i$edy}2P+8zaY_VMXd{1^gW*HwjNbWo-I%E-MvZl{x;v0)?%SwMUy5xXe z(8-j|8lco#ePZ&kfmH2|tZ5bTb8F7IS@O7p2L}Abstp3M>&Km(O$`HqT2%aUROBD_B z(N=8u_;n#hXdk)^-HDHNF7&~a%JIZhbRB(5*(QlDc~ysBzvhD}Z46}i@@Kw2pgMfr zh8Al6lZu7A7wz=bd}d|iGgiH48+F_aCnq5JENl8H3pxi&7}dQ=?Hd*3r4J4R24g}< zy#dpxK8{gh1ET+5NNKxu%OwE(GkyhR9sQnwlF0*~*1ct?eawpkklX-B`^+{vq-|Pp z;6D6)wL2pNL_sUw3YSZ`p#sSnK-<&CAlq{n18l{BC!F)8jP_+OKpwbzC<&l+p8y-JTOFJdD;k zFCfBfmzwuXj}0gXpm|@sUpsRpuaS93F8>#(LUk~eia-y#=5HOyMF-*sNL}d%Re$TC zmC)6-Tqd-0$9!fC5B$x`>EsKE6CggtYE>=J(Q)fqk;66Wn(xj(UB}vu6Bii1PMa}K zj6Twi9Q<9r@ACG*#Rn_HIh?~eoWtXThc!iWV8ScTnBRNOW#%#Vyi+kFu0ptmitFn> zq&P_IXN5Yay=><2Ap$j+O%{+hSTbgV0h z!Fr;+ZL_|CWV@-?eN*+<5XA-tnb2&V_06|&qY30Dm)zYACl@Y~O_n&)xH-j5a_Q`P zZen8x?{1naABC2}zBZj5RGzal2ow*=!s+F+hmfIPrp??I*qmw`ZqoZdEVRbvNr~Jf zv;8C8woZ{BefS4Pa7y;{uaGcBg?G3 zl(Ve3l5^fb$4*oGyALdKSU{6^Go=ltp8SoISUig8ys@k3CkgY~z8pl&gA$(P8n5c( z0HY40ywB)9RQ4V^8GEai4-jqhQ6B{8+vtb(h^?9bL9(rhU2VPfA#BkA?B&upYL@zR zy+A(gC*FC4uCL;7tKW`1WIBLRF2Tj~*x;-#8=me1;yDjla`-e3p%g~v*6SEk=#?j= z=v(R+(iwf(uP9^=a=%tzkv{21)C<~uz~$Fz-j@H3x`kKv`Or&ksWD7DBa7z-@k7Su z*W<|TB))!1igwc6=O?8u-{&Vrd2L^QyfzO{(@)iQX)uFzSUMo!Fdw^}C+WU1uKBq8 zp$|p7&#Aw8pFu&8-X=!+W1@u9FE-ZQqz@NVYe;1B$w7~UtKcJ9xB34Cqks%71ID|gmjp(NDgdL zMJqawLnGL!1{n>LwC=fsmT8x=n_s83jK!wmh)=s<@f$bi?N1yT+m$DDPJz=G91uY) zQ>1f%aLXDuNb(`9ToGl*Kwj6Of^QL=B2saulZb76VlYZxmBh;D5Rc&~wjyS{d<3R* zCXD95L;JY33KA?Bo3QQJ>k~G z%YxxJ$%mXfSsNG|xl6lV&f#$)wCxZa{|*bE0gst>>C!33VB^U2Mk^?-M)~I4rwzC zfgDC@v#_8Kr))ToCyRL-^=<65-cMFj;P!Fa%+q0iD=rY(=zH`@7WzD~lS2`2e&v5u zMhL}_h5rx0sOew3a6o{r=*MlSK!;z9<`7SP3oJbhFkeeBWN;oON%Hh@8!oI-^G1sM ztokd3aEbHd+ZZY~VuBbow2=vd1&en91mN5584y98uRs1)8OMykh_UXNYp4(HOFMg0 zr;Qw&a&Erx8c0g8sqr8Es;^NG=%E7+G~F-QJgNMX7EzLPZ=L_R=Z@LiG!yaQ5!kj*Odow|7mos($}4Xe3D&-BqxV^+!O-4K6`XL zb)}r@WkMzxXN8WkJJQ4d5U3Vuk<(F@mGJbht|RSs9$@tYoo^2=rSF^7mvSg|;nKw< zox?et!#O-oxc9F6w~H6<-R{49+3Ri1FrJTkZ|ZrgFsXZ3W8$yt$^1(g+U2dx1^8z; zbK{JAlsQ?r&eL8uYHpu?=Kic}16e<{&Q3RLUWgR65FB(^Ggzy!lepRbq?`C(qZ|B0 zy$Yw8WMkigje;Jz3H;^DyCZA)T(;7jUhW|duQKe=V-DTm zj~k31s6#1WkXdlTVJN~BqXj2}8vJWIRPvqR22TfV*ORD&OPZnRz{3?QgstO^4jCFE%7Gk~^Z!~{=^yhV z=CLKXI*n6Mxtjb@q0>}OW)qgPkEqaDkk%`(hgLd>8-ao*KUoAEpSIe?Un7h@qECHi zA6mv=FyS^h$<3jG`4`R!ycZ27#!iRhV@OFR^4gzEg(4dt%NLpYbqE+#-t^gcNt`EB zO{y^*Qcf4s0-)M*hi%J0v2QT@hOx01M3oUXK-F_c>kQu3S?kS`Co^DvD}Q)HbJ16_ zjq6omA$@IdO!E|p!?;LwTsqwW9aDyjJ;9U2MR_~Gc7wuLPDu2&n4g9%}~W7B}Kz=95zaWRaB)_sJHsO1u8teE7P zc}ZyR0INamFZp~Zr9PSi5sZNt1suZBPpKaNiIB}k44!e5M&SOwZlHzUm@BSz5WPp= zxzaC*qHTgs7fIHr)O*tJe(Ujq-0)P~c&uC4e=^+0ifNt!q?i(HJHk+&mPcs$MpT9I zC${n|>CVelpt{h0j8oq{u%VXCA}%`R<_x-Ee!vjpl@Qbef~^bp3G}b>{mbjws_7e9 z+tpXpHf*qBbf)j@khW7^WTfpqC%L_X{PIYOcK_+V2h!f|^To@3AD(0sQe*VaeAfqE z-bgAT@j3WpQ^&6{`N3kBWHiIrBJIL#4 zYUetbFXHfNHsR%XrzhNNuy68cy#;eqHTIIB^1@@>JcFlWLod3THb!iddDCm$__Cd- zwHuvMX2_NDwo{bHzog~~shfndg>x`fhZE?Xa@4sn^^Zhr@4)LUe4M2s&VWc{>IL$R zKGsae^t}GewzA6i(|6-$C_-3fupv5e zo6Vrmp^?_^M1>h=HKyn*`C(ffnt&Xb6!w!m-UIRDBidbM%|_y+@0MM!k9j~_ZtS-@ zlu}*qyL|t4>EdO-3eJa9vN35d_6M$K={R-8&1DxHAU=Ftzom}iH|rY2rnF4V8>(~U zrvl(H9H%V#hwFqhVDfGK_Ko0vLdyTqwHr^^z*IebFa1mQIEUi&Wg&Js_Bnk8oud!N zF*JNAb@Rxd8SLtEwl^yF%dnfrp= zgr>i*<6cxn8#Uo7)a}sGA_>u{C2c}=N z7a1D|f9ExY_~jPG83mBH^8>SRxeF{&$Jth1!0JB@F3P273{nQwr+k4mbe4aCj+Tq8 zjr5lt(%*&IXNsrGMMtdpX`jruz3=Y(8lHo_9u&IWD4z00(}V8B=Iwid;_dagQR5xV znmpI$@n3x!3%Ng#xe9tPqMvx4L`IEO`nP|?BFy1b_4$(B!j(P_?^TQ+s+24}FN7D> z3omH(!Iux9YQ4*oiTS~YyFbW6om`*M4LyIpqRAl)9R*Q-;F~D#M4eCK#f95Y=I0#Tg+8Dd?>v=}2~T#eW!f zCLlDL-r?cY;3L$f^6$ZP`81yc920qU#^5Xi0sQqH6g%|Tu$Oa{H~x`}OR&hBZ-GVO zvI4S}_X)!xO~b;saE*IBP}MsFO8z1}I2^J%Vto{0Aa+GV1x9!Y%(&AqWVa`cv#ZZ8 z0JTECaVC_WcX12sNSy)cBVhbK_)(bgz%TsbGCndU-M{pKCz^JDS3mcSNKfLcanN{L z#?V;O$ZOZ32^KN@El({v7+{{%nM5l*Hv-Cwtd{oxED*f*vf}TB$f!x4i9U-rZXu zUuu=1zy_^1V|^&aEBNoMrU-2^Ea*@NY=zwrRe08INYdnF<=lqWEgf)v3M%XD8h|}rBcS6-iI=BAR zme~lgS-d$n5pKBAH)d4X(dT+IMxALcZ>C&62A-dkLN_69`W~Z-TaDA!p^-eydk&`z z;I?Bv0%P;F%P4G$Zwnd6{EKk6r?<}W0(8NG7lfpMdy z+ZQ#Ozv%Y3@eM4-0=U$H>~(S2vARSrAoF3N$_~FUb0zhf==@bym(V){n%7AfiRl;p zsc-ytbD3_%cbsAI?cz@X8N!QA|E0^9wtMcq_xyjQ&fy%+;R(TAZ+QK7@7;HA_uqHf zYx@A^nZ_~i5by7VT|m(3%HPmyY=8_!3m|O-jv+~0S6t_3F3jn=3VY2mu4`Jk`H@%I zUdH5c;ZTZOG@2gyvG0`G>Eu+T%kiO(w#Bv9`A~{=kXKdtVCn%MO7TPsYoz}>Mftpr zdM&N>v?%ZDfH)Q@a!r5I0k6%nmvVziw4v|)NZstOwn%$6KjIy29#4#E zP?F--9rmqs$39>^kn#`L>L$KzU)!H$$57X{$5bcZkTXrmTRLya0rL~mM`-)LF`h%I zxc*@r^hGE-j2^Kq;emc>KWsVd$nIzS*ops^B91l>Dj)mMveT^J5(hcaiQB9q;&aH* z;cizWP3Cc=@lbswwy$JluOqt`^wT``bd3*A`ev!>Tc`UbOuk@mm%LIq&lCt}<%VGn}7 zq$}#fn5NBmZJ!UM>VKqobsurGqxmwH#V6yw^%Jd{r`nJ`S`MV(oBxrzbP2pXki7w8 zioD)S*H~$VxbDk*(t2<_m~5q|v{}zLeRJNrvA_D9=j`k?J$5wq80R^Z;x&-c5eHMA zqj^$F<6ZeYKQmvheMTrdw4#mG=DKM!`Xu6LJT%;&fJ>u$v z>-J?6H_u~Pb3OmA>wC>j)^q9F^|cU)Q8EbW?_<7M z>=k2;_dX2DI-4~$hf%^RYkJK9){Cr+1J*O!3fx;^?Qox`9>^N`B9G{q_rkLo@?>D92vJy*lsnmycgJb%V7R*u zCYO%idDc$Tv<4D?K>H9(ZR5232?IN&N0{0|Url8}$Ubk2lPhov%0^Bj0OP!td8yOGj*BW{5m zT|x34=B%(XI`8Q38Go7hGxjpkFPkl6`j}};7EkEvd^izwiYpI{8LV!+@zg0RCOcJ(28Co(UBgakEh77u9A*M;V=WN5rs?iJoXD zZVvFcK1L}w5d6zX8LDqm9E&vV>-O-13zaA3|D`~;Js91pDby}`%?pb)uX@STQuPZ$ zHf!Egw9(8#RW0f=CUQf5_S1a}6qJH+oF0#Jf&&~#YrCIPU&-o?gBJe$5-gi+kUm0@ zenm};jmQHr)zE#}uV<(-F(P8Xpo4A-i&XV!{_g*kq4kLo<(n2l>P|toOX(D39MKMJ z5`1vTjRA!@DC-UAA+X#)ZNnBiyeV7YeKZ~TkD%bKT zpN&S*4Bt1)0Bpmrj83=7tcTl0XxDRS`R!IF0VUiv-3OTKXb<%g{1}?qkme7J8DHJk zy4`@LjZM@Cx_y|7+!+(hN${`n=Wpb)&os~ndg(C)O{KXG(NoWpapxG*B?#ebgpt$5 zL}&TbH~z4*bgVGJjfq@qtP?hjTcGbGUZku*5fZHJ3B5 zKlaR(nP{v#V{-DUxrF85&BQk5w+cE>ueWDH&8LPX|fELU5g_f{+R-h{H_=DBa2 z2Xi1L9?H|Up=)f}vR9h+NMqN*zWB>cx;+PQ7&rA84GQ&zMs!j3u+#0L2Di+} z8{3_?9{6S+_Aqu8{mE_02YJkX?-n?yOZBKDzr)l?XvgEAG3}in)zQv*atFKDuj{+z zW*=l6{*RRHSCxYY3*E<4f$Etnzc*zX)R^T>I?L0+Mr!>G*FJ#KPfpTl3qUY^j%Q>*;2n7MiMN6NW0fD9$) zAyy7DQ_AJ&TrM06k)HBFLiXb|xK#KKj2yYwEI)FUY|cAqFR~2GI9pK{zExs{(1NTf zhHQY0jRmGx{t3kGGwwpi1s)?$N(DQ=>(YJA<9_ZT+jV~=UkX>- z2-Oz!6_7pcxYwsKnDssFmbE=Pp1KXa;3`MvptduZFZ-Y=Zm-Q5`@xcb&=VGy$eHyu z_6F+&FPdx}eS)*U$xh00pwPv5tD|2^Zehm(aW$ zBY4~sK#Fl>A29@UjT@)q=5+%eZ2Xz^=JnPu�ogMKNWRV zP`)A#_Kole5O)}M3nu@uQt1@#7~M&#)421v7q z3(bQLRQwtb^YJmrQ81o?R|DjWWG!c8#v8YC#RC-Ca4q_TS2+xhbj%w2gun#eGD@jK zV_>P(*>E+MZ%M_f?5x!iF`bhzMtFi%j$vVH-4kQ_}H_J6z|zilie?utEdq2kG0)6`31+vsrS1dl?IB zo@hMOZ_Cy4gUIb0(?%@_b>p?s^N8{^K+-amo>(XcfAqNUO(}B}jqp8lZV*0Ecp@dc zjNOw2Qi)a})tVrA%MH{P&GoP|^hAH+M!^FQJSgK?#`LB9@{Xsc9+F}I2(PkH9;w+z z3;l_*Ql<2n`5E2PM@fM5b?R@|s{k7c4C9-8lf4e5gx+8%Zv4vShB7#4Q`(b$+kzr^ zqp1hd9>RPpKGz9Iih~rU7%0Ht_d62UHt3Lc1l#Qcbf#FRZhf+K>#i7*o-gr|V8*OZcWUO(+17a9^ z{$-zy!5Z(1n^=7W^jwgB@+9P0bPrhnD$_TZ6cIg|CvB#=iS}6Aj1TlETkat?cId{6xt?Q)*F4D09exdY@e+qp_p8nLRb>8%+Vhg~GLoQ4Kk53pF9GaSsOcf3=4`cFn=1~TinJRvG+Bp+?Oa$U5%IKK% zwWI)v1!zlt-O1~0E?&6gxN|s%b2x{`1ODZ@*VQ*^uE`U;PkPe0wJn&l&xosOXs(3> z9KqM&j(4;J9XG=b(^mi^I2e6Ue2Uko>ME^;>R0+5Y2cdBuC}k*f*{H z!yDT!Ik2S;ZO7xcT>(o8iOge9s?6zQ{NU=z{$UqwP5Zl#Xv~cR18iE=-DT7##}Bre zz67_tvbC@qIjq4JwtS0Sg-O@2(*lAEqj$Gx!WA+cgvr}&%@}cA3aK~!gtqp92?rS1 zy}KuCmP4sg57bn87y1Wi=xPD@Q&+|d!@@QS|MZ@*2L!zunp%0H%C+R<$&Ua`f|25rb8*duPE zK3wH#P}=9>#Y?jNFWD~T$tXY3CK(u0JqPyu10cS2>OK~6Y+ zgr-jyCv{*+Zx|ta*0g;-V6Xuc2;aCvcOG{oqw3e=aI|NIAupd@Acs;;AciC|+Ph%l z9~&qOwjdEEsE%eO9-#9Vn#ANo!*%EP4e{=D9S4q{PEqKTTNQ4)q0Qwqn2P9u?Qj16 z*pCb~Syh@Y7?5@URfsySR^RGaWDi)_Ac5$a6t+TrrhHpk0igMA^m3G5%STHpJ>|2c&5`+KBg9-&mW9JpEBvVQ=VcXw1dlZOR;t-jUUxg7S12 zUQV^gf%4L3l}XTQp}!(0ebP^d$w0th@Szl&JEc*~vU5{{!?Nmk5Trc)iPyCCfmAl@ zF~E8Z0?6RLDO~{8m-Fn?eIo>3*JpFD0?tE!MZX#2@XN-B!^0fp9J~(hOi)@XKfLkV zcL0f7uM=D-UO?U+N3M$yyCHiL8hNjTlAjs zp1xs&)czX()lVW%jca|0@#dQR7?bc7Fdxy^IgsodESVQlru&WOpGM{Z&jp=skIT3( zVO%!7YG39)+KSEa1;!JbQ6AMN-y{4IpLRocCN;*3jcv)wVN^E4$euA_8R$#JALJv? zsIU8zHx%)AyLc`ozvDlsXPXJ@-7-&N`rTyc&%Swjw z3V2=5{g?0GF5Y{2yKwKtiam#OIEQn%HhB2KZM*C_`ikf1wgWtO^#oS#em<*-s3$2; zUhOZhu05Zvb6d~1=;_ z{arLveyB#a>;+h#@pJRl9H}FE(AwKKyW7o8-O>dgxz?j~gWk{&wlTU?ewR)8*t|K6 z@+-zTpiq3T>7raesEJ(pkx^`j=pr@gI@m9#{q39t@{xG}GMCq|*f8RYed&W9wV9tT z;ZRC`>=?hcPwmz}ynzsX!8ht{yQ)4-9KPta>LISm9^Ig`@Z?acpJ)j`TzEjAGU&&) zP}@LiA2>i08_GA;OUBs4xyeWQS?^ti3y3RS2qRzRb9j2(#*OrmvYFE6^q0eS=4mOO z=E?sL`3W1^ogWv`pX%l*Jdx3TaL8=N0p!sa+Z3H#x&O*`Np|Eo;NbOd8c)=nv8K}g z?1|tfE};RBig$SjbX~_-ul_9T`tS9z#5*Bm4na9axr7`@*^!P9G2;S6p<`rlKSReJ zKj=BG`vv2zlE{vWA&Z6E0z zEYg0=5AMg}iIB7*^A~$S4tRLl7F++)rAynr7cSTyy-+Ug=l_vn|0WsSZfm=X9(Cxt zbblLj&4TWKN^_qooutp@zkE5Iy78v;xjAms*J@t&j~ji6rv0SQ|FO*LPL$-GbXpGv``i9x;R>w#*BL* zV@ep8xb%1Hv*@@TYQ3fp`>;prxz}>~4`KX@b2|OJ^1I0z4rRb?ISP>>c8B`9T(8MK zkcyna%HQ)6gsUxkEkYh2NNMfxy;J(S#;w;h%c^|U`0h8*gExFU_QQ)fqtaV$pmjmr z8ArZ1q_O5Fc{SE>SxdR+L0Ug~Jm>%DXQ!a9L%D%nV8DnqjgARS-zgw})g0I5< zC(U$yHPC$SGS!#^-QmHcH$Y+V0i~GmpN*Igd4mH?D+poJ-nqd{@H31oE-SwQr@V{j zDu7(WXFqbzc*hR^A*>uMPx+l<8Lv$W;Ep@*+&>BbNYixlkKgq~Ppq$5qgEraoN@$JS^I6);{4zrE{BYus$;Z`01 z-lI;bUW09@;SN zEjrS1uMH3*vJs7JUT$m!@dbvHM;Z~Ml8nEG_0zxDJ5-_ zjgNVmUyPdOhjG<6Gg7J{Z4juoPQan~M}~q~IPjG+_O+J>q(=*`%eqoS8?i^d#Ib(>_Pjo#fZHP=`sp#(GfBQMP}=AiDc5nxd^w@9G7&5%by6`LrB?^ zu*RP(zK`*A4SGUOD0BJ2yk4pH8)y3Adf;p$-0M!(rmR1#x4z{tp~|5P*1p(N5BWz; ztbg2$<;OJqAjE5Bt-HfEubUqYP=n;aiYzt)$yPqAq|o}$CH~SEal_`rRw7Y={oz+& zX`RJ3X}i=nz5SH(-hKmn4Ygf@TROoVdlT#vt=sE5o|{{u#aeD!79kQ^n zk}rWk@>~U=Ka|v7^fnXPR%tu7pIpz)Gk&m@F=4x&Qs}Ggkw(=Su#Oj7iVE5eq3={) z*s_Q454xZ)JJvr;l1sZ#DeJLx)i)lI89UBr6Fls!dQkcVmda{PLcOQ}-L>bxK->dKE72VMp_9P5Z->9eR zMjf#m{iLB>#+3Ieyn+fFSN+C?O6^Ag-2uCR!EE`sj&ggfSKC^TmB-)8(|R!`^$e); zhxz4_L~4uP@#wns+=qRA5JlW77y9G|FE@ReYtRw4U|x5KyyQgeaFu4FM!2uRnA2im9i%h>kt@)KDcfcgKU@=mig1>qumk9m&!O{?S`Z#OY*tQA*g^k@`eAvjAzTVpe~b#>9tHHAMJG1Kb`Fa zG^Im#fl;nJa_}Y8fsQ!~9|tVOb|nwPKt-I2cifVW9vojf@fdG(|JFQJ;p9nK>x3_T z*7H^N`oZjX$2tzuZat@!I_;Q!=$gLTsQEKLGR89IGw$8TJ>Kg;zvBC8r7r7wxrW8F z<B@^_$b&2`13Vf_rlFVWSJp@I>bcse`Jl(zbhAcfo#I|- zY|Pj|z2;jI%VS+7(9Vpg5HMq^6v_q-ag*M@3CWt&_79{j_iZ!VQK;|{Q3@%qPCGrbN@DA^wyzon2 z3YKqz`%)S?UV2A0dYDcc!gTn>?X*&oE(aJIfXNk~xZz`5I@{r*(C*_0WU@2-Yh6x< z$s2&<;5y`Kkk4@k`2>IVV?TEL!Y}%w?e;tFaQJ!8d*1ff|N39wKKf%m#$m(4X9tJG zR#@VvBnJgfdmlfz@LNEkisHmuzI-e07=~KxnMxYL8MS)%u(p$#t?d>_*)u7ehRUv< zVsH8j{f3ESj*A|T9uq0zzJrnH4WsV`#1)VD{ybNyvuB6EG7u zevBU$1aOsCI4apN_>?P*(M%n*IjBW63tDj0P5BH(8y+kI{zFmZuzXVn=Zj{AsZ)4{ zX7e|i@Pr}t;<0yxpD4KI;z|93iQ*EkU3)5@T|5aN!zvWH?Kl>xWiI>4DU5B5c^maI z^dI6Ed0Je3BVFn6z_@4Lgb|x7jBUzx`_d0MDxl41CmR*Dkvqb^5r7dgHXfp{HgzAR zGOkd`Tvx!+2J)15z|6hsh6px+uCrwBn|vfc}|rBwms&PicWMbjFP|`f*AW z0&?4Hf8im7)fu4tqzi{ku6ycWLrXG54%J5=x4G)}jR-ng&79`$Qk(5c#+zQmX= z@&#$H7_aF=J>K(+sg!Gj5n!BKM{W;{M2t~xs^kC&^4_$;2vw;4JO`9cdY&A7<${DQ zpJj{-xGk&idwwZ;Z1BdW)nh5Y2Ilo0ewClX=&LtgfT)^38wBcx?!fUBq54nv4a;O% z(k5r2F2)&`{OM=2ZjQHhK-DV_Dw2G`ldy|-?j=fo6?FNEX^(Twj3*%Fb}$H0n)KwM z@8?(E+#Gm7oxr)z;T+E49IgrOyL?IOM1GLcHle2*WBj&Lm3Pc(c8Mdv*S_*!4KmYS z6BMn1gz*5GwBtf^%;nJE`x_EUjFRTsX1CHSF6(ZsH^N?j@vA@xHc(h)=lYwQwf<2C zYoCx;Blh6~dPC1yE3<}PXOZX^(j`EeBCPj8dF$R?}YkP(R!6(i1NAG)ZJDJhY|8jlRO>*MIKL4Lj^- zZXSk*@|VrLzC8g*xw@gg%J?Q^jNJ)Xrqc1~*SMXECxHO6m$!l3qa3KZ2h+C5=R>KR zWZSCGaFEfjdK)$_eW>K`_C`PSzqS{nue7MeQa(LHU5nkghowSm zWsn^wp#csr`4BIdKAnD?{hehuCOvsNz6cn?>l8R|JjqqA{R1SB;+y}_vGkbzVeJd0 zPxyC#qwm_j=9PID@Z`g^AM3OZrzR+`vnSnp zxj}mIK~x<^-JH1#9l5VjN;39gl%Ix@oc&ZG=rQE+tv%A{i*p=6qvh*9&~2`--91!U z(r`MD^BsWnb1Gj7+FQv1NR>Bq z%Mai`L>#*vh_iV1py*u+Vb`N#9bPc8I~Z|coUm!&L@(}h(pScLkh!|!ov(C68~9!s z{e(W;NFUR@Yq$GPwN-RBHD}&@ihti%Gyhs6Jm-6ib8mnHsoW#zagX2oeb1>W8(QH3 z-^e66>DSyJQU8CF=gfMNR`IR_vg+gVuDfWONAX%P2X^5#_^7Y?U|O_Hx5gTCO|MM@ zN%J_1Z1A=%xuSI?+&cq0JW*Gb3sz*oLam^Da#AFz?FTInZD>E^Fk=yYTHXrx{fd@( z^*0YiY=7jB{J!N$sft|v3AdNXfRj&PL$&o1nH)_g6(C%KBD92Wn2JyS6jwOO0(A(k z?pPCvOiIAKT1glm_Gd6m9);vju>~Ml6bB!?`w1-m?&uaGa=73MM67rgAP;ikE%`t< zZY0IA*HSBtccH!GV7ba~P@uvc{U`tApWHs`qdsbT%2S@=@VCGE)!WNH;S;vM_s!qz z@R_jE#lMCdc5(X*r?La7K>*(=JPBtcL_gO63-L)xzN5H34Em!@0yF+tf+JeAQ+W-C zVF6TV?QZe2V3(RgLi$axCk)0#CXkF18?7ZPyy-W+2uCV$HwH=3SnG_RW|8`H;_LDm(&*u!`%1P-{*DJ4Yv}@t-jYcpVIO6iF7z_m3 zvfCBi3y&%@g?HP?>yZ7ORzRAu^Cq39L+%{HVMN4G?{$oPHkdG0$|&l-fUc-#4nQ~{ zN_mVE^Z^SWo^X)my0A>A-=sgLA0h-PBxj5Y>Bp8a>uBpYzqDzdIrLytW__z4XI(jj z9$Ou%%Pwp5VJDXzRI2NQyVl(s8&H1Ypi5o*;D|P6lYTQBDI;y2e$YlB`oTCfb(Np| z7=syq#vbz+uwdzyd>lEs|Em9WoA`!@WCOcR(JymE_je3#&jAAB!*BKz%dp0eG>Pi+ z>GSUY@^{!9SINN!uzyMESBYp$@m_iK#a!8q)Ah|*$$?(#SaY1}?!I>*n?LPGM{8Q+ zn&%9R0BGSfjP63-9Yp?is880X3RjxF(<7)n1v`Ja&chXY6(XO}VQv!84SffyN}VpVvcwrUuq@ue$PN-AG=r*E`l6-?UoL8F!HIvo9wx%mK$rfpZqjV>`LV3#vS^J%@!RX%Fu7+HNVzL)g3>;IAed&dy2Lj z!{bS&9gLkvIpoErvW&i|o*U~v*kat8ugSa8-A)rbuJQooKdDO6x~ybsI@Uq8srM{p z*K*L2r=)Tql|GbuxXodVj<6F|U)zep9tW18c%TQj2T$PK--jMK_~4L*AK*}?<)0o4 z9j{^a3l4$}u3LXx$Jn55yXxo8*Lhd0GWzc5CvW-7v&-+81*#FqnuEU@Khj_C5%YTU zj3tjp>yx?WVe5qhp$m84>sRGnym--J^yYo5^iDgPo)4w+l+;B2L+aLp{{mw#nuAgH zp9n`wxee6@hju-$y}xilyf0ib@|s0IAxfKh-b;H^+2~A2-!f9C#K8xLd^9uQX!N4F zD1Gnx97Oprs@*=6(mu#C(uZsqyIn+|5^>0CNAw*?@KG|m#B>Zh ze*#<^Pr?~xPJ$9T4Y$#-q@4*%=JuLp=>T`GRbY*Ak2U(%I_4=LU7&mBR`n0tMKy2H zUqG+jh0fa02wR`-YG-)W~_T`t8df4jq2lGgBgc47W;<^^S)3Kr47CQL_Yey z#~ka0#()o|dacp=kGyI0=%MGD`BYZWat*zd9;u_sv+sl!>txo>@bbEb>o}z;ecUIZ zT6uiZZ213qB;-tYZh<$QGegMaW{wy*#CuiqZhME!F=_jB7Pf6Aw9 zZ+QLd{R@Q8f8{H;-}8Ha@AgAK^h2(!JEuzhvG;l}Khg6{(RuwFUcY_uU;pcF^q>Fv zpEvz?dDnN@{`epNMe!uHCe$zL(lRW3cK5Y9dpYa*nty+YB z>Zg9nbdibI+kNV%eVXWh*7m`F>QA{7{=zT(f>oT3AHcif)1Usd?XzF;itV#h?t`SK z-}AWd+l$$c6;4#z0UEUDgA%^OJ3s6{J(zLm-*E+zxa#4xP8LQUbg+!&-=XX zB`pI?1^*_VIW_WT#Tz{b!oqiX@b3i=B# zfBE*gfAw=s=dIrAt+t>1$)EIOwRX7HnOnQ!<6p1<-wdsA9D6$7RiFw?9=cW_;nT27 zJCM~d>KDMBxdckzkHCHC5Kg7iulK->%gT2aG2)HM8LzG-4}O9`awHF6e!4i^nHmTd zjug{80gK?CqzDV`R$|JvEOAW@nj>S*V_hsV0$qFD*TsZXxoR7mF z)-nNgrtb6w9~{`A%jN}_4SAR8#a&b^7&M&6dPN?1pabU!ntluz#LWdL8f2Eg+ZEZp zN%6r?ZK|r>kS_-vAni#%fL3&2V-D1MlN%9Wc=Tad`bi9{j`I^!6-OOHKR0#EK(y(% zsSAc2^dF9l!W(X^1FEXSFaqVz4UMW>rA7bq2AB_iWc;zdqbuUxB&2FKei0!Rbv@md zrN`DE@lJC*t>;9$t%IH4NG5UtV%-0rILyW z+2gEINhCsci0r-Hk*w^kj58za?9Jiqz1QLFeYP9U-F@%#{eJ&}`|0kU@7L?~cs`%0 zxBUCjQQTge0j2aOWyS7ZEm8XW&j23DW(bMs4)$J#T(kO_dd|zJsm#bvb#q@inROi! z@&@&94vD!1N*1&=dh)pKKT%x0?jh|k&$ENI@=L1tbnLvKDgu+W@!dxu4^VXm2hDT( z|39-`(-o9$By2ifcBA*^RfT&*G-~DG&qgLUr6wH&GY{N&7F>BNP^V(U%oI1!>2;xb z*X@O_x3pRk&4vl=?Pq&&6IR&OOUAF&bKJ92YCr5}YI@D1Dts6nUB8Am^+6X>1K@#N7lY714va-}; z8u3hNT+-WJ(r*djp|!GX((TvA;x#Y7J{13RE<5HyEiqsrx^WcCF(s?hF;vZo%`@CB z_*HLWR$vC*P4n~s2{3j|sn(hi^^<1&1QJ85=pg|zTTZ}S2 zOo6e=%ovu90%Gkex)qx5U#>`J>kwLc6l;{x>FkbYW{d}3BH?`bC-bd?a+9pa)w~-7 zm9$vgeyxGnH2Y!wJv<1t8g-tRhqt}R#OaV;OMsr4tOfA_y>I{WXn3?l9D%A0`VxSj zF-=@@TVtyKUW~fK^BuLlOkHSK$%pI%c8)oTu~ui%$D*(-?HJ|bGreia;|W8|me?it zUZnjd<({0mIHWC3cB%}e!RIthS@vh`Szt5jGx42pY8vd~e$__w9yZ$bn+acPvivxm zh<;2Ii-d(y1NS)Xu_V=l!gq(i=IMsmQ$cJjz@F!t0?~Rj@l6}_>#xyiPOeG1m~IzIK*vk~(RWYBo>`c72lepRTGS>4lD&QGU^ zF0@dmn+xp#qB*$U1Zk!9xWn?M+q*z!00;44hO~%!CFC>aF=OOue|REzSm{sJz*976 z#1G60hGKl6sF=$*;HuwG2>PtR{*W|iby4H5&FBTgfWd$_!rxtZEr=lz^mZ?J zLlo%hdrlTEw8J6Vq5BXUeR!tWE7Ni+GXeHzd-Yp`bf)3ilN;pM&g$2W?k8=L9}hSg z@Rr9`hNlA(8-?7a0(i2uL$J)?ufE-)nP$K*%bF+xFB}0QAnZZ$sCA3LT%)+zBhk`oj(}_9plmlctvCV4=bLvGj7O ze|at+37MMU-A(GquLDpK>&x8}4dM=Pkr;xd@ccC*n$PNnS=f^I^xgdgv0-mTO_7@L zl8yLyA;N9In+j{v0g}*EcSt-qOa{rr-LcBxZ6^B>?_pQJumYQ~!C#1?hC?MJSb ziogy-bTnpuxln<6os&1?hvgu>Ntd&|?2jDB7Z1<=p0sjL3D&^;EK;2-ccM`(OM zMn(ZQ^s2eT;lyA-D(?|Taz=_?zQkwRf+%rE?Tl^)k!(kb)acRt0QAb|Kcj04jg#LR zF9y_`m0qPe-6VX~i=UB>cQ`3hX6$g8Ey~N`))%$r;s7#H`-Zs56@whr4 z@c@LC&pYlAdQ2Gh<}nXX1KK(DU~h80Q9|ln=2Y!^Cg+~z$GnOvGUDr(DOGXh)Wap* zT{wOUjuFyKxg-DMYkfzbUm$Y>^h!PnTU74vLB(RC=oXXt+cCAEUig0l!nTf5v!vFS zd=ID6{pMWDnet>)tUg$YR2ux28^Tq-SCGTFi?=^{phuvJUJbLp`WdWz%xWD{CdG-r ze~=32kJ^!Ol}vTPZe1EX>x>-bzMNB0b8C^u_)x`z@5ug~6)v&zKUz*xGDiAjl!{eO zHye9W7UHP@yN_hDaa)`6dnVHg5lqR^I9nPQ%R) zZ2dips^y1{9-%`btv$!z+QBYY7+T|ZLV+KXQ;+sVP`A+$5RHW zoMMNdR_84a>$S&U=6mgQthWJpxcD2!-K>f8BlS`X4bc8f#`DiWyB{gB@GFe{%Rx@+ zFvrHK6a_=OZRe^OQrp-P)eJg)yyTN`=DnO8zg+x{-5;Tnp`6}%I$P@87@+xXbW1;J zbzI_d62_F{a$k%nRI{&v)O)3RbUUF|{VUx&%tyQ9UTlqE#U~xulJ6NPA}APiZ_vD# zMKI4~icjpe2a6Hp2d_f_!NR3FcfR*8S;;k{j~}-r&}erG4tX<61h|=xhsb?t zq%9ixZb-2mKn#*Y0koGO1-(uc&3mZ~2L$_m&&;IOVL8p|VrWk&~OV^tlVx8JrR=Ms)jSS$Oyc0^SDl9?v?CC-7 z2~bZ$2)g?)(~kll71}iuI%p1Eg6@4vU&S3~ZsY)X((oF`s@yzh(|BO=C+quQ_X=%0 zuSA-LCVA9t-7GuyVtx_UetUt76PBqTk43EM)MEI{aF6fsVxvZ*8!LqX3BRjvtIk%i zz%UxI5Yr8f-<`M8%jxz_-*}|v!yiww>3tv3_D{T-men&Rl4)(#ojR_Hvq|9Go%Gwl zap@jMWrqG--%1W1U|~==TX?J-a&-P>*PFX}$+hUv72Eo5anD)OcX0``NmRXfR2_Jr z+-ru@V`}#3GGZd~Dp)HpbGz$h0_1*LP1hcc@%mF=bo_<5ld}$WK_W#jfMwh6|GZTp z)z1JF-u-p(l?m$oF!Tq#uk(oDpNn_Hqc5Xf(^gF8V`}o8*Xil z(EMz{DV(63-kAseSFmEggto86fDSQB#4Z??6%heE=tAIONW3Ey7e0f1n<;lEBlr#r z6=+QYWR9PWAR^I#H`xj*^dzgM7VOH5Gc6U@MLXcK z9i&weB>U3TtV{_s@Z{C-SV z9-ndp;5;X0Ngf*noIL@j_WQMoLUsyzBu(L(-|ruQlS{xSR6r-Z6=@&xTkrv~40+Bd ze;vO?x`a3i`L)wMLR8g4u;Gwdc}?8C?PZhGDHD_rDGf}3))mMRb`Y@36p{;mJLa+% z#cX^rj%sS%Yr52)6kQ@L*6I48tsH@;RX|(iYZ;^}@P$UiyMSPv5NN)@9_o*>t;2wz zm?ilWIPfdJQ{z%)kAfh6q%y50Ph6`5;)MZK{!_aCNs!tLg&OvV<gA*^WWhMJG@Cps$$b=5N?LU6o*IFM3-5zh@K`I{V7S z4L!x6E8%-gPL*eKS9H+a$Sl6Q>f$W{79W2n#PYq_2+YHAOu;t3xf&3fFC+)y#Oj+| zGP|Cs>LQGDvF#!iroH8zVb9JwbVa^>O`mn657JmD3)+A>?@~iATd+T<-aX#y6R8MD z%>MS%QGGC|BFn3$^&-o3l5j8Ubm;OVs_p5sywKM48-lna-C5nOQn&OEfA5t^&?eQn zeAdd}iD#v4;u>YrNI&C=VgU2NLeBZuswvdWHqBaWoeK`j=?kKQ#!mNrt2K-_X0Ca1 zsW>Mnl!0-_~`PB&GZr$BYj7Fh4wdkZpNz>@XLu zG2icc5u6EN-dboZEJVOhZ`ePuDBEitNiej#EmRRVb+MbqFs0G!*!oSzd*5wSSOHSZ zP<8UY{0{n7OaGT!AC!O7{z7-~m3=LG&ZpIx@?(|x^FyCU0HuwzOnr}YggznULjjBP zxBmFP*ovDiq7Fia&DC;l8lV>6|E<|{x@+hKr$WeiA~yhM!l4#&PqZ&mZ9H6;Z_v&N zSW4O!HQoio(?7kn7tqYv`nKsX9c~P<5lmTgBO?WLT@bfR;w@L*l^Da z+QO6VQHeMHRD2TO?Wk;`&N+=v-4HSrm`@7} z7XTTcl%2A*To>p_F;N(5WXsV3R46*1u1q!Y%8|)}hYNK6>n+L7P)n~6^@9tAahF6S ztNAG~Kl#Gk=&(#lZf<7ReR1}cXahi6=a$x2B&XR-eAeQ~BI&T+c=&K<*dSEz$PHAd zZ};--J*PxhZyH?aI8N>-#6{b4*3EL})@zzXMe##L3xh2d1QUZ4T*9Z~uYr3$u)~nk z5o~7prZ2?e9_lsF6%dJHNfXszJozO7+hxc*#43pTvZ4hV+S$2qBy)DE_^H}fwm(); zLfF_C;k7#5$t5^+e=SSdSLkfxLzc^eqx{y=8|5@ctN3Hj6xSosWWB zWm>HLDLXMgrP{_7FLH@OPy=fB%)=95^p|Bkk$g$W|LCN~yB4MMOq ze`-ZPhyVE7HzRvPSBh4qik5S<)GLgyK`>jY9NwrP1d^)f0$5>DT;)ve^5#-8wFYlJ z7#?vdt$IA=tJH5jNgvmU`NyrBIk6M;OG1%0Fta%n)$ptJEAUi+@OQNj{^u2O{O}9i zwto7nXlHWDmV7B_XkGz1ESFhQNA`Ykw;ooorQLOl;9I+W*7zfXG8t*xI=(=z#<_O( zOBe742YP>6d|q1ro_qAprb5y5gQ%)Yg$H{dr}K!nh7HrQ#{N|N$+5=k8p?JP-Mp5M zhGo@u12%1Sw>^qHY+D#OO3DwxCD*687B%JCf>KqcvbD)2Qk`t+zco@`JVM{}KIevZ z7R;b)4&2w5O&Vdx^*-!s}+kw(KdiB@VDc@(T9jDm4OW&2=aJD_@u z)JoPxM%bE6Yc7+xwZb}1s9*Kry~)4;5_!9gKm%YT{Nh_ldh=tF2JndL;Dy^T7@L9I zsRZES0O1r=jRytGGa%GVwbP;_kVcbp+;5Zyd_+#=cy;QFu7b1>V6bz8I#$>^FQNyA z?Lx3N@9MZJoPSlKp*dgq_dt3Lqc;48tFigU(TE>5RCn;3(fD#N?vS_$J0;1rY?~Xe z>#msv95=SWNY9a#{sW{v1A&SE)USx~>42}Zw-#-{7C(@5TIYE2*KuARb1LtMXWR)0 zlIm|>M6_g_zO0tzZ=R5WRh<(4!*Tq_=|;sl@1N|phcq3w%0OS=apO3t<_wCilx;tU z5{F<*@>@{Q0r`c2!6ACU3lSN0+1?Ki-zt8f#O2L|%z-JXO1m)0wgDlItpuqv<(V0dgYCYJ<>!kQkzYG+E?x*cq=k?D zcs-k3gLmLiY#$ZQ4k>#0!aTb&U^`TKulQHg`##{0b(&B+_xi_!vu{7d7us^);Kd1RN;S9ra zI<_yEK?#gW!z$^H*BZy+TrJJB|Bn}?^;&c!ZhFjcAh-RK!RhM_iQcNGk$l-w$U!Lw z0MdzV&FOJvn{!;-AFE)|oZs-@dA=EJ5*cB|HSG{#FrQta%frV?uR~TIaog({{N*pHr~ zF?dSiK{S>s3!Ds04>154Q!>}jaJH^q8{IATI_(0?P0=n#3L0k*E!M7?0&y$K_wwDF zI2s#QLnUk?iMbOjfiAi&Qolvy{uZ8sV5M_fKxUuV{RTEfZ~>80L_OU=32nTP%)hOGwEl);fH1I z;h=iBz)-az-^c((yv)4MSPvnr)(x_>Hrw+IcCN@#_+s5n8I} z+7;+9N+|UD2H@Bin+{dRPC@Vi)(7s2{h`UhEcE*NzAqiW41Y24VK_BZGwxT|^OqcTPU92| zi^(wyLZf+;uhvQqub<>2uGe zIXF@eAC;1NV@A1}6v$uS8v3%ciV{!_Lg|91xEf5Lk<6@J{VtN}V{?>cWD4ltL$e(TemW_St*Sry1G5c;VgErI z2~n_P2?1v4^UF8@Ar4gq++s<(5w^`^zO23Q)QKw|R~VM??czp{ur8(L(kGsr2LQT- zq;ZG~OFb-VbX7)ioUl$Ym^RC6so`>HV%Dch$};jOq_z4W|J%Yo_I{PHW$ur;o{6pB zc~wq4$iSaeuVv?4oM)Y{IMWrc4y?~ylQu602geP4d3^dls}JaOonOzggT=?r=N5Bi zVIPXRY^`VYaRB$JhQ^?=o1Ab}>f{eCh{9AM|F`n1G|pD=Kd4^dgVk&gqBemu2pqnL zdr~OF&-SL?ZR(6~DG@JFc`izK^&Ix`VBc!&MT*~_krpx$`HMOYG@Y2UbrGD` zk&9%Jdk{yV@BQ@|hdld|5)k);(xDT`-~`YzlP!$gV*>2@+@3`T$4pV)HX<568xB}1 z^c0Vi>g|Po{pu!b9TY@QHVX5v#WKvrE9qV#XWL0@=*{0_v7aMc@ z8Fhu+5RHE&%olz(reQ147+Jw(r1iC%Jz-|;i}cxR21AG2*v){BJY~Qz=jW_6`NG3i zNxeklI2l6KM#hPyp}XO(Bg|DR{_D=x(YE)H`5VCe8c$6W7*$NXy1K;zI=i*^&Xv*h z`!a8X->07!UtpH91BdbzM~-tGG}YMnR2Xy3T3u0TNjev=hKJ zjMVMa@3s$&iyDVlB6S?J7JLtbQA-k=G;jW|<)C1BiH`L$sE?n?clLioyvsz0^y~EA#;qo7(!q-|3(m@bS3RsaeVmcd zahTk?3N!x@aGl=6iueHR>jXBXrmR8 zPVXhe^a8G{a_4gfRd(JroPUE3jpO(}Tcpfw>CsXlxBEBSe)_;9HUiXg5O1=4X`kDB zSbK}UX;z|Z0`P=p_OT!bBX=UzC@l>oh^rROL9}e_C-eB;WW%2p%l?U|XSe)01G#Ks^YSFm(q4BfN+ za`W$jWBdRiQcnH6f%DDV&ll{~`Dxp{8qVkBL!VY8tFh~j+!I&z?tl1LTiQ0?T|^-h z(&#V;zR?uGpHlL8+?lQ|!o_2Si(OiF)O)Y;Ma4^A)(NGUjDy^7wvpfwZlzathj&ta za8&Tui!#hyKZQ_ z5^5l#V#SV8^~XPY6l}XD|7c-s8%r&1vkvDCQ)|*`CPYEL)P9v z*B@*J7}@#+=NZxY^@f7B(NYk610n^~mtr^KN5UX8@#$MwaH>2m0XP@c+tEj<%5f7_ zrT{m%UEx1M+5w+TC;%^Vfx92BsWzl%Kr--^UH2x$D^s%x*6({PyxBpm~ZJ%~DgM4k5-{=eA{ zH8Mg#R{?T;Lx`mPRWu;-m(S zww4HdvCM_s$*4u*eaW;El7t6CQ>y}%yM(YELT5Y+rW5C!)h;zB^<9>K*v4W*+j?;6Mg^cln1$lUaq-iHz^qWR#o`Z~JmnW~=U4e~w zG4rgb*vy{_8uc5FRFn$$w3pWSLw30N;nB!jlHQx)qI73i16E~6EwRU!77WfOR)Ogv zUp}aGJrA(ibLNwvW87AM>~J^7?xm!_m|uSI$es61;vaZ5%*CCDboJ&LVAQJnH~N}F zgJeTrMUQ_f2vQ78EULG)%+8hPq>;{D$GWT zpB>{;o>U~YVZVs(x~X}<(wOK=?pq9XmNhc6kN)i`a+tkb*R^o}#t)K)I`o*?w*2ttNo|`}tK?kvpP$au0-1TRyRy zlNR%et1?2}78q8?rE2{>&&#Ut0VuaO(#JT?da)eGdeL^I-uu~qbS=dB4!3pNmnU=E z5anUGWK7NdSP(oUxq<#+%|-KsNFB~S)rJFUXGZl;KG-=TQ)DK!Nx==*21h}pB0-~| zP0?|Ix`^?=I}&ms2J&ZI4WIrJ>bECOLmds1k$RTuZ|`d;b?Q#Ok)`aj+5SQ~_Qv%5 z6D>K3?b#ziceUHXvmN#<;GN2T|(t5zNKa#u89FL_g+jt7#U*TGR&-j6XO`a`&S>Z zMS!5;%n^*X(c8J~5~jj>guvX*JUlws|IQkx>bf#{w<)BR9l8wE8Pdo;zpq zM!le5ndxG%`Fk%qBXK{&9X2pBYRpR4$b5%;o`EgYFWBtQ@4j9E*&PV~n?zna&fG-Jj@q+;(sK zoBP!)0UfTEF60q2)^PH#S#WM=GN@6PP+?ys)R(PbeEu2go1kWL0!1aI$Wyt`G+11v z&DFuy7zb_#D<_jH(`W#gDNIwbjgANt}Y#}^P=qG;Q9!<714_hHJ32AU$# zM4_4LjF<;EWMC0|cAQd2SLgR4Q8X##Sp(8O;RPX)$ib(Vk}se-80L?lD{r>pS(l-A z6nraKmv#I-m%Mw>cX%8C9~Qf_C)MZqMnN_*7aj+679VUu+nGU%u*EiwR~rm8tU`uG ze7Ogw6K{H#o%4h{qmGT0>5JaG!7rO3a4k}q>qR<7bMUPgx8KpfMl8h;;rZ!Zdn&rU zSLQzeU(cDf55q?Gg|7_UT2fq%f2pvs;MpD|Pm>qf|5!4D%n)_ADRfiY zDvtNkod{QRDs8Ju^y;3zAa)01e&%>n^a%gRe^I^q*6l|&?2*pk^*f!)^0~mkcRP;O zye^G&FQIw7uAeyOnCM507#Bbfh?dZ~a8qlVU ztoydX7AjONpKF5MA>T4Ao97_TTkgSBD_=zI1{8A35tTi8Oj+AAyQiEVFfBWaB{(hRR@uJq?c_8xmUy+zbjnMi>F0zTO3lYfv(!U6KMsSb)mfPKkFx>d;G*$e5k9KEaFK11%8sjN7K>-sq8)AsDEG4Ms@T1XmYvrK?b0+C>%aJrtY+U2N z47qGWdA7o3yvlQXLW~-2q`%q|4*W6h3cqEqNzEH;?o(PMA zx+y0-dsP)oo*h!WvbX~PeQ3MYuJCA7wesw0XWn{vAM+jN@8jwXVGkJyh}Uh6M^HT} zjrOpl&}Lil)2G#je`o|IDsRq;C6`3;@@?FEKQ9tIGb{OSbCQSdRN?MB$HHRJQ||D0 z+SfvxJx4MaPyeJ??~c zmn=8qJ{U#Zya4O@f8%`Dhu!Et+=xi2qp1$>cmk>tXVCiNWwq9|N%2tmN24sE6pf%XF(1+x2oCINvS0J#=}+$b!yYy}X@DMe7!P{KMgeQ{Ku30z}yb#re<&Rcsg9)loqvDeId1l#Z@Hp@}&v0p$$+VfyO_H+2w4 zJl?5|{1}lYb?y8681W||xkx$z^BBHZrnzS&-zp3mjP4rCQ&cU9f+-eTfJJZZRB`xCbQV~KoIEa?QoGe zr2YL`MXqs)#@#j?-%x{~DNtE?uW8|-DRJsvH^YNze^O<=Lv-zPDyGeC=UgM}jN~WJ zF@`}M65?gCXB}cu8l6yS&8QPNAD;Gs&IjpUGi^EEV54x97CNbQqlh@|h^2n@ApU~f z$*3q#y?Pi#i*Vn5?GZTbgW&n8f2u~KupO##|JX`-7dxwfyqyzK8MJ-4(7TyQA%O^s zkR4UNwxcXvQTj}-D^rQ@x+=Jg$?a*j*M%wqKFpUll&uDQ1CA{Xk}G67>vP1T{`xd# zi16}x$2rS}pO`XAjai*Wm7qJPsI8F6%zcMfn;Rm_i_-n3FojUm;I7bYaxFRJmW7-e zJl!}qcmDTnVbDiPK)a&J_x0@I@;vV58^0QzdslVGjQVZ2XFX-}dKdx-6rD1W#-oe{8q470@yo9=6j|6TQCHK}p=h&GzPm*ma&!tD*{w|_4 zlhS&h%i@E#Q&UdE1#zIc%#r)Ov)9i|3iN%D4)Q~yHG?tOB?LeQ^&=V<@!R$K{|C)v|$~D*eKhOKF?CL+jv0MNZ_#Zx^xTpZ%kaVIV!$zd#)BZ7IT<-zm33C!R9Y-cBnA4wIXq zVyIfTkc1|(?_h+15xK(L9{=>LMIKqD>(~9jA^CKr$y%-?C5E%|Mpw@5|@kZp8YifEYDE z>#CqmXe6GDcphTGzsVopYKlJw;GI3oz+~()+%b{Z0>fZxg>dmgbD+~Y5WmV*^*DGI z6X}Qv*?Lh7n~#{&_21C7sc0xC95E4&(4J%f^Bk<=KSJhg&_l?}AyCWmj6Z5dkm?0j zy^&X?0$Kn3<4kff8~-g+`Au-|#+vH=r^)UOQcCdvaTiJ~Vv3RU^IhXW*8Ag9z_^xt^|z{m zg7am6bLtjmdQcR6H)7ZpdN0{Rd39|huPaBTr~i7@ljLBH$d7lcj7RV85Ap@fesCW4 z5wATb1QwdsxszElF`Lj$Z`x!%&be500(ToHDyM>mJ}R zFyYhxgB~{cb|^)|rb21QqRs4rnW|6=S_UCRKP=k2C&6JoH9%Y!F3B+r(i>j&sf93n zp8w}qh|k(=rD+cAxCzqi#>2m~3E#8kPYeVtdaj2?v&x_GdoKAweF_=b=NZ>(K;I-wu&# zv({5tSibkGwK~%qzC1*kmSp#*cP3OyJb7I!RIm;WJPX#FW;)wQ4p0U&EVvzKhl)Gm z#6B9Xw~d})|G7Nnn9-iM4`7J);H&r59KH>AcP6kKumy;fB~oWzHVVJyK$U_%`YHRi zGqOrthNJkMzc-z+x?P7I2uY=WiL2ihm!N&oeIW+oWV>C zC%4U3Eh{P$HjWI+++tIfi?-0-Vy&>VHrI|3r~D;OZo6ttka}*89HdUnlWoqiZO~!Q;z;rcUHkkxljT3AbOy4A$G1wck{v#2&o{DFeg#Q<08%dM5Fkc`Y$K*PyHDMX5kx$dS@8_g$6b4Cv1 z#=s(-2XhOxE}>-XcPB$R;t-uhw{c2~PlOLQH+g4n51 zRzc(Gi}pb6`DZ|Z+|qIJ&_;c7#O?38G1qg6CBcq`L{j808B3IK;g8ig;rO)8BMm^2 zF1ycrTT+kgL#{@qyPC3t9JD=ezhVT9a9SQufhYWqDHTGmK7Gc0qSw8Q%ig@3dzq9y z`ieoF&gIrEBa|?qa)R#D<@NjEtE#`1M`+dQ>IfkxI_dA$mFr)W$D|$q5WR#OaBmNH ztRy%Zu4a^wa}k_?N2$H ziou>p^sn^sG0wD}8^GN0wosp>Q>U9(?M>@DfEPW)RHg;Ng*07Q^My>Sxi1pqZ-Uod zru~QUhGOo8^c-Fo!3K^w{oEhI$gsyLnu^S-Ajr-JU3-(`4U^UuglJ|Q3Gz(gb>3C4 zmX$-_9?u2PzA9*P#pW5Mw4?{*_~6ii>xCXuU>6QdBzr0F!ZjQ;?hFTB$6i0YFc_~G zlo*h>7A`}XgusQuD2Vi{5K;rAm`zQ5eF>)q<1Y~otsc3yVJHGfa_y8yKN*=?euO%a zyUU|%J#B-11}b%g^Yo3$Gu${311oIZ850c<*NYl#xOtYd@g6OqmVPj>Y-54IUg1X? zzaLM2f|vQBcGWS{P9C8!@Z)pDFQZ^zMa5PRqk0NpB@5F+@|4TW)%{>rARSjk>|Gge zEB%B&J~pk%%936KrGsT}i*#O|+530%j?nv>1(0H&n{n9uWx00;8>xXmi|65JQKouG ztnYp01K376G*h_L^SVG#}#atrs*0t41N zwnv1<9L|d;vi#-1Sbp<+(+*TqQK);>+EMB?h3M#oH(`vkcdB|3vw}g(ikJ1T#s5aM zFH8Szl{K=U`is?%QIOicZaFu`XMzs_Qd)f6QL?r!%PqXC4n9kK(w4~fp<$oQY zucQ8XF15!NL>4^l0#~ZyH)5t}i~^NjC%Aoa_z- zxMq1KPCa;}F)ATGzxIR90s=HUcG0XJ3Cvl~0tH$q> z1X~0q;&GY}378{Z!YYI;U|7#bf%^7_{>19GA6cZCFg=}bPFv(q2p;>bvUr}foq;`j z%lOd#rJwU$+bsSs!Bj0ynTMVK+!1eg(CJF0CW~bP5KpdHq4` zZ>BlUIhdW6i1!cuqG^_D&GRv8cGKth8fC!a?uq>9%X7$3`fzH9G<=q)kgjUX0)1R%f8O}q1vGsFv z0uxuieKd?1Qj%>P6j=#uW=t}AVDSO-XVcwo{+HyeK%I7jfu)40#=;KCvcbOTqnWU? zG1Xq59LCNQ={`fYTw%yI>a|6LV=J+a{4H&`|D~c&Mb5)u&gJN*f#jyDyz%MYHLVUM zT#JJi#u#HXFe$e8rS#)u{w{p)r2G^2Ggi5CaGz=DVt3Tu4W!53b5MMT1XQPPp>6mR z%TezIcUl$d;k1TB{3-nQ^he-ahfhgk#p62rdueZK3ZkkXPehkN?4nHx*Xhz!&V#Mi zf7#8_)+Sg_2=d7=8EtIV*m;-Yi?R~EG8X33!k|}|xLGbruT`Go+68*m-GOf}yPFJC zE+sd4Fu~iBweOIFYe4NWaRS_fLYmbdq3AM~>&)hqdqM9%V!zy249*ta=@!rTGPT9g zSSrmJ;41Xz<*T2sib&mA?s1$|aIoSLr_Js14`?+F{p{j&++)8DNuYrG+r!9sSLdV%7DpYelG46;qvA=4On>)gf^I)cGpCo`NbD5-q z{6gFATyS7aYq7w_JoN+cZos96Z-0h)_7KcsG}@9cSf05^Xc=k-NPN3UiO9)WW)CU=bcEYPG6>|r*2_G>cD}e zaH_1)<)=^P0o-yJX>MYC%lVW>qQ2@BpLI~0N?K#N#x}tkpcEzXWWg%|F?$L*dxf}P zd5*ZWfrPh8UarD&z3c%#4+m}W7pn9vH2EKkEa|J&UVvn|YGEgC5OeT4pLjk;_s^c% zOOTQV@hyMlgIuPhe9SBApbijRDO_vbLgzk4x!rdy%M?-$hGv08IZ1%?;7J3b0ZXZv3yPW^-VUK9Fzxq*& z9^CB6lwNZqQ~|in=ahNB4^8M4dCy>8$F`d@fV&N5x_AYC&`qUwmiphc)cH195O5-I zdb047lg5SN{Xsn)eJ1e_$ifm5z8}fIush>X1DZ)4Fq~ui&7qs>lIZt`&&M9Ku#4^8 zmu6+0KdODNyP~)lp@F0`#57%!6W^XX4>henC2CxKSL6SW->J|}@l{k}*Tag?;ys6w z(%L<1+t8#GHL4|>0mCcfpK;b}C!B3cugaNj`OOUg`!Iw3^>_OHs~!`c+}hAptyZA9 z%1zZuA{*?PX|&pM5gLSb4ezh5_bvxDW+=xx&>XNGjylpA6|lVJQmA*G;IHFJ81+>U{$nA01VYKmWsQ= z-*8RodgIUu2yaTyr*gdCg-+kXSC6dmjGp#-cj2~ZHI0*&`=b&qivK)oedOWx`fA3z zx58}SUfSzdNuGP16;Id%Mh@isTPQ-r(s{oV(huwKJd$7MNY2H6-@a_yWlGnXJYV_6 zEu$+lZNH$eiraV`70D*IF26||d*@y|W&eRRqpthGBb@h-uqZm3*LI&@ObgjWE!XJV z-IJX4QF$%nA6Y5R_=xp=-h4h84~?hE8d?x!+fj-<;ed{WEJz%+NcEV;tPEcpe;qUL z%yz#t>jB+HaPa*dTX&|wml%M>-J^G{68{|Kv7_*LSsT~&s4=RgREl2LT*uqaZ67tZ zG(T9LkQ7ZB=Vsk<;d_apQK}EmR~}RNd>v5d-ZKu6oh-H#VZxbx%?&>e{2H$L(r>O1 zdvk|~qiNpel)2yE7A)!TH4oZhO@#G?<1^fd%W2&6++Iih z9=QJh5_U4&{@1`z9;C)9^iBPn{bVyeXJTriu{4j|**cjN=4;KKca5lP!LSts4A+%j!RGB~ee>pc|XJ7*X7>C)eHp{$;_wHqN!GEw^& zu{e^Je~20HjCNn!PHUNs!w%8Sm*RpmE%}i=HxFMLz03Q-7#6ba#l`v!a0_PnIePNx z1)AZdwa%D=wUnLt+~+z(gi#zoc|9~n=Qk*A^jf6jT; zV`QT5NMFh`4#7u%5mm7zNA|L?>W}jmfvtZ5k`R z(8rze0ma7&m|lBU=x|)JA7V2w#eBo7(#mK5NKZ`CI}X?3Ah9W3c?uKXsYE)EGIlFZ z$5*63BhS8HanHkRHFGZW7Gci=^f~nzXbI3b`uEzyyyE>O^J(VPek#iI&92Q--cbB4 zmxC=AZt<5K%(JSyYDeFj{DwB->>MyY2tjZ0N$-5`0tc+?ch@LGQH5Vdw|oCNAmNj zdZaFL-ENUBb(6cI<14SrnN-kp;}VW1PQ!>V*ar_155O}Q3m)jktVWDrvm!H&6K+)Y`sQ;B44sOCMnzF&{i)cNtrrBa#nrA2DW;bHDbOB8*U- z?%XcZKxoC)U_QtinQGF^P2Ak9#klJKe{q<(q2~wWUC<&IGjSQbj5ROh25v2q!tpB( zFP7xBI3~ry=tW4f`2T$*1H5y{l7p7yEEsh6fA%hweF)VzIuW4I@T820QQD#(3vfY-~o~tn^2vGr%%QHUK(# z(jz%qyP3hSu)+pcb=KGUaxqwc z^aPChESE0K z542t6$_)m7#b@MTN_~R&L1SQg9Jz4enGoh9i}R~p3f<>H@y!AAJj|p$ys@h`p&V62 zH?)a2Z_=In9~cf34&)jm3@V7Mi{u!U;VR!`BIwnyA3= zBXhZNhTD$om7rkyXIgG*a3kyfE1H+?zfbehg=9L1b2x`{D0uMlwq4ZR;9r+(?q$9X zcs5}|@to2zhW3=387ICSR5`;+!~TQ34NRytsh*l!d)A%v@iJG}e7;LNaBUi`VaH|8 zw~gQHK&@-Gv2Msy%JV?#hV4m;R{;zU9wFLf5Xlz0PrAIg-TlUUYzK0qlp8(hFZ+$y zC+kg;F_(*!<*Iu4=2p4HWsNF%tpkAyx;$+rS1Y^g2f!l-FnOvTUevE|UbSuDgNLzh zB1T%%bcF#6XU9zaI^)@XEe|g8bp$ye;rdhOM#4>7GNr!tgOWiFSp7&sZsG|XA58Hij{Vctr~d%EevY23dnXkiiLSKt&HS*9 zal+n-r{DPjG53kE1AQOtQS_DeW6;lU-X0YSN=~-I=Uzw@gGi?E{#K&{2=gA zk82K`{Dk3cw`{lflVZ1WAXRUNs3)beA5i-;-XttsX)pIx_;DzucJY4JPZZWug?Ha; z|3%4>!_xktU&f8+3)SUFFUGaTX*|N{i~i<$sOH!{klOdJOl%&*^c(hcbs%-|-b>r1 z3mi=40GGbyJ%sz9c$r=Cw0uIZ17PG^o5S^R{sl>2@~^0i5(A^J3K`3dV}Io|GKMnX z($KO$^hTWNCqDd~P}m@vjqbBqGeEDAM*4I537C9Pfk0H!^OBz~qnw4R)p@B$tt*igs6K)mk_+^nSmWG#b<4Z_pcOjMWt^~< z_#O&aV_a#hGv;IJ!m4NLAKJM$lON48|N4o;<-eo&TMF;r;cp#9&Kw3%%S>kW+VW}DVH#O<>6fwr&l+HY{a@(Vi85+&G&c2K(T%+-^? z5NH*)apmp}Lix3Bx#Upr)3(Lgc#XZQ#W;ck)mIM{OT{mFR@gC~Ft{uG#( zE{!v#;4s{?#@yDRI!jM=4K3six9x#P%_bgMo3I)gLr z6s)xNZIy8Ad8QPlvj_qU8TAfbx}Ri{%A}=vq1_r=J?5G?WDvpeEjRp`bvN#Unal z4rNhV7xC5L3hzY#b1f)(hSmQVKu%}pVoYSb3Gm5=pz9YunaHsN#6;GTvnix}4CN}S z*qN{E*SZGLJ+Er(|C!>~V;IK%b)#Jj_Z&iuKGhT@Bino0*#}hU)p|5D%a1G=F*a&s zc=W+k+&Xl?=qV#)cSx|bV=#51!sQs+ZLowEFyQp2VVNWEgECUqU&~Ry$gQeLhBm?| z2jl>PjaFU_(MC>}qpq%}>XOZU?HnXQsl+HLl(!D;FuJOq2;Aff4oBmJ-FP6bYU2aztngxq@ zpGTpjNnXoSH%g?J?l1iW8;HIrlW{!fm5k-|(E)fH1@QIwqK?deg}!O#_Mk0-0LAFv zsk``yejEM%$c1nE#v?wMqCXksw`}~1O=SS{qF=Fz6;hvJd|QSd1Bn~`14QP59gb_G zU9?lDIewtkbsS|6v=1D{DBGD3^_Eie4J`g{g7m<}x3-PoAm}{Ap)cMw&cPI~o#E!f zIh?~eoWlzD+|4iF@6$Y+xprrx=rX651QKT6_B9cSY$tBf7%pC465`r8J$qQAMi$Mh z%&nfYYp(W?fR)*5#=i zfpQG=CVv#hb%LGFP~isY;!)e2{4l6&a@vX~J+RHOU$HaLmC91b?pLFKAs4ii9-Yf? zRm2vtU9LWYOHS-hZsK8k@>G0ol0U3+Z>C7KuWS_hoau`+HWK!a^T!6lZq3a|+R?Q} zh~h0=;2<~v+U-W~lCV)>>>p(4e&)l0-e=U|z}ok_|Jy!q*-J0IcOL)A70$w>*#=a7 zXbV3jRReO%kx|U7u>Js!MFS*`L730WMI%U8?ir%(B5fz3dT% zE;8d+Z2FH0RY&)m>I3Xi&<*`G$g2ad_`rxfD&cmbZ8&h^X((P%o~NSn`c1#8)Q3>D zM-=7;e(eA3JHrqC!;c>u`pGrsvdi}w@%sR`+RFV>?c}+IcG73o`A`pLm3Z1nm@%Wd zwfYlv7_Z!>Z}ps{GU%oIO?zJsjjg8iqT=8!x&h)Q2`muYZQM3x~}q6PG;iG23p zLX|%y26UN1_kR!>teeC;kLfP7A)Cvf9|7Yy&XGhVeg}}9-hg?Pw{QnVx4lgheGo`o z{Dw6zAW!9*SvrQa_H?WwOgmBb+iRCJXEDsULXT(s8Qa8lpXl(I$2u(3MF_>UUDOty z-?JXha{$s0=b47k3Q1&PPKgY2kD2=}UQ2xtME^}6o`*s;x5}MzMnrK8U(2Mv3t}J8 zmp$(XeUGpDlO?k($S~~$%L5-H)Isqu{q#rM1dJuHXnViS|6qc(?j$Yyb?XXjJ<+T9 z?sOGsuHk$dQf`(#L1sX5>PjX%@wXgy&q9Avrd^1X6DYsqMx zx`$+YRnUae7}X%v5pj+355nzC?-PJ0UUG#{i1J*AwH`gF8kN@AH8)TPRt1w|P?e9TXHFnQQ@wzzXJU)@J~eBowpSlujY$qTskC6jN?vIz9T z+35D3OexjaG)NxxI+RaY$ToffAl|*8_-1=8EIp5nMHcym9t#VY3%nX7}%Z~gx7X+-QnT>UG$oEg4>c+7S@#Ja#K$Ky(PW`tdn1Q;4Y$ znaUk`3-EQDw136)HO(pXd5txm-rOE=_H#Igb2x|XfeYJq&pr3}FePhYuSEj6HM1BA zl14C2`QuBZ9U?2Pq0-1buL3;6a+ZV%BW*zD;2?8)&C|8!;7?j*r!?|3RxIFrUDoSX z))nlE+?>xE!&4&tA3FTE5$RfJLPTrUqncya8@l|TBYtk)vc3XdYn9h)KkKvNkp&t0 zKq@z~vX;fxi|xcKj`hvjBD7AbKFsqVG@0*dN4u=Gq)|qNn|^MfWzEF(dfoIv`_W~s zmz98=fb+B@NwRB@8~wJfSA8dQ^sRcSZpakOx=s6qmP)C-VG>9j;UzBK;?re10}eI? zYoKiLK8)fpBu_}y4R+;0hrsAhi<|lu`68Wm!`6^qxw(E*p6Vc(Hnz=g8y6cUb|ygG zhi$8RV{1?d``&x*Vw-z zpAS3Tw-Ux~M^^H~BYT3pQqZrAmtJx}mjfw}6Xo-4YJqaFTwl>?-qUSJ?i6cu#DEa# zvhnLukh17O$eyOL;~ zBskSv|;|lEi zWOjF$e5f%1k)k_z$8Q;!CylJ9%$3?_s^9uR74*JSs4>^$&TE_Kwafsm)$$<|Ymg6K zjf}G#4&^?9GEc{Ta9OWE z(GTubfn?`94Tt5Y&W}qZM%R3+jTV$_M{&tnF3|2ZfU~Y;;aMI*(;nE~Ls+r^KmOyt zoI|M=(l?q>?7CnsIGHX~$~%(&Z-p=p@6^Hduq&BepQTcDzAn%-7hZ&$e-BPKRK>*`*WhwhCZ!X91jwQfGD(n@ePZ&| z_XM1v8^-24QqN%DxeoKiS+4>6tDbB8gyY!L`OX66F8Tnz5wd5dodlHBcnAxz#Vc4M z7ub=qXN2l1SmMrtL6$aIdEnpiygHpzVD6)@^jHddt{CA;FCN`5yFWX;_;s1GQO6Yp zJJBgaXcrl|=+E%>=7dEH1I>#@7AwDZTkbICjYtWf@G?N&IA!%BE0=|~Z|2%xi*7Ie zu8+X-pckQUF0*hU2a6Y|^w#B$_X4isQpaOBD-k(--F*_@dS|1O%}Upexa6BK#KW_Q zNHK>ht6Q)&|qKl%@K*--cpL*3)#_lB!BGU}T)xJtJEFRN_6va!LR_K>lsGD}q@Wod)BB_1H3qj9So_2Sde{l~gS z@2*2Oo^MkBi1EkR=|eGOsIunWIHGspl$phOkR@|P<}3v3HlWY=1_OOXE=DYJIluZF zxEi88*HJg9yPo=(r7qnUdXDHmmU*Pwf{dVOc-}bTQFI4#|0tVe`bRdvLg@)zQ3n2O z3Ub3v{k_IP`dQ{L*V#80NoQQ#kg>q#g7GbxR!kdZoH$(m^`Oo5sq`Rv@nr|QXlH5+ zxSjlopUd#K+W|p`M^t&@ELUA{CstpO-uMG;Zu;;)C%^rM^H=zt!#SLT;Nfk1-RpjP zyZf#;Zuj4RS#xjJ?p`~3UICe#bHS+@%<=M7no}#!fR4voWd>suyzOux4q@3MQ*;y2UDz{T2w59K{~^>T4OAf7UXuUAS$_Q5R#17+jsy#p^35S+K!kTZ9K6);H+K zcI2j6*^uy2yw^AAmtMWrt$eGlJN|->AG$aMs)&sEl_j)kqq5n=f7q#IS5BI);Beh$ zJ0N~KGq6Nbf9aa`!3Ot}IXv~jp<&tO*c#ZRwn_9shV-MX`K<%M`W8(~S^4NM)}?Q# z=YPhqIr%>*A9~sD$G+9L5*galH!Ib?t#|m^_ECS~;DIu>r&})gLb!EkG^;%*Q%UM; z1?+S=`uk|#1#P>jE!wuCPIXLBVmmR-AwcvnVvinuWFMYs!6z?7LpH3VU@%ldYl-R^jqOES`|Gju!UmZ+=x74fp zZn^1ZANcUYte($?j>H_DnxEQ)CpsT^;6XnT#VZsq`ah)PUd(?~k=uRW^X{NU#wt)?D#+oGAmr;p zQhVa;5>G&JL7%jwyKrGsm~a?AhVTha8qg!b z-cB|SSR;1{1^519A4-8GBuX_(!BRBJjseHQS9$FN7Rsg(@Bh6383@BIe+Hg>U_nYuo;x%jVW*et{Ju6o(_dleH?N5`sgS4QTypUb z2#I)BumH%={&*cA8Fh6y z(BTN#Gt(Y7U4TMWa8F4%BX%&x_DVSc$&h|{4k^{<0nXaQT(T-zX;c)38E$ke)_fpU$mBXyg&Lf2_P$&vmN;|kg60~}2C z!2yO1eF9xDw|k>o$c+eSP&bU9@r%$xZqz^&eYY{^%?SnAetG>+`d!9z4vi>Z^?+Wv_E+Unx9GpeTG~Kp zL%Hhhj3Gte&U_*ILeCYYGowDF`^LqC98B>mV>X>-m~)t`Jmw=Fv;WsdK>ir$wHc_f zV87~0xt3WMYC%=(PZV`Y9?M?+>JXcT%Y;JQ4xECcG%MBD2KOD@x^(KuwWf2;w?gd` zkWXLw05_Wb`XCOaE?n5&@Y`?PF2DXwmGKfof z=494FGSr-$S*B)~nu7>*YVswrQWZsp<1)i3{~??a9q4(NkLKpa*fKpoSA5UUo_8e& zvea7B@+e1Wc3IXgxn947ZAJx7kgd*PV_Db_7yTI%Bx0_YXS9*-QcdE&y$JX8=KroY;WMc+D$qQb+xYMtBZ zQf_^agt#4yaJM3E>DomoBdq#9xo@UbIDE)hZB^~%IuuEJxezg;%aedW9+h=w>6!J< z2O&HerMAhF7V+l^oy8s({~-_VkSSn!@F)sfVc4kH7xI%AS`R<;u;|x;l<3nI$YuJ< z-}ZTvcHdB&Ve23hKcK=6vMnZ_pkaz~0MxhK1;`H1fojjTYuxs32kOyvNgt$-~G4(Oz9M(U2NX|o^_5l6rG6Kkqp+u-Y4B(U-P>~c^=UqUC;??%l>96oU%#gkFD@`RN9x7>Vlo@(Qe2fMi1wZ}AVVw=BgZT3Xa z_JR84I+VJ2*@t15E?&yhQkvJu6I;CIFNB7k;IRIUy4Xc$ZkL+l+`p<%u_y5pCN=kw zPo>=M)`{wjPUt5YyE*LTNh$wuP4#e}i$p`YVGpGQI=ly!p?^`{SkZ!+lY)HXHkh9Z zz&rP7mVR5P3*VM3#p@C$ayim2QSIRyf5ikKRTvgrD$X!G`*I-|U-4+({@`!kg6R{i zf#H`2zr#xgqD99%>=Ku7qS_zq^{3bF_3$Uy={=4?lQc6s1=cfP>z)3SKIHv2>mjZ& z=|(;&CYZD*uk_zBZ`OG78#t=ZvaT8(uC-g(br`fPZSjLQ@-cU+4>Lbz?bf_sNWAyS zEnUfwIlJWWJO>ZP4dkJpdN*y`1ER+qyE~Bbc&_+D<^}K57d^o~GTRFiU(FC*mA7H_ zH&C`=(g;r9gz(}L87zn8IUHv*yi!n1`Az_N8=Wt>mT{6{AKx%b1y4c8vr;Y~a!$bw zdII_XvG*Tv*JVYSH@utFnT(io91{i*1qUUmLlhOkQH-x)6bBRs2StQ&95DwJ17R4$ zD27o{M3SHw5J3R}6+{V5XgWQi`{_u3*Z;bz)?WME=YINW;QM~>H+G#=f^wfGAhtE`KiDICXCYx!P|q!%fVl+=$UK5+?Cv zcnDBN$%7C>DlUx^j5Iq_Vk)r4%t?g-`L+@78I-&Y!)M;x_)XgEo(zg+lOJD2Y~v@e zs-y3r`MQkoNg-{q;qCgYap7do^@j8rx$Za9__E7nUl^jwg$N~=(1xq~_}Zs+fojEF z*`TxA6h$||Dvyjx$;4(Li^%LM9jtl4{eF_Q%SZ{4ZEVbiz1U&UA6)p9QC~QrCmUoX z%l2ulE|+pr?gqjp(6WpOLHV~)gtq8U$z=V371#biyMag)`S2%I)U;XU?RhDFv){5& z{nAG&7{_6ZjjN>(AHJ|5)EBS80MRz@ES66P5n@cw0)Rj#Fh!I4b3VP)54Hwm)N9z5mK;%*xp^YKdZ86@B zuCNU=$({`{+zV!9Z2YAtTtVw+o+r}(x#5%pwmvxV=1a&gsk*J5A3o+*VAh+M9k2k# z_8A<|r4W7L*_vPb*0_h z$Etq`=J}#;oVCuOAwCC|881FWl%E4C8|k8DT;4?emcBuq{42|W84K?JO3zI<`rE1# z;|5fE*F#C!dzg;>mA;JYI_>2b=vaDe^FIhA%!yNYPabQEhRC=4_9deI36#F2 zYqja+m-8P1N0%cO92>{JhEi_5<7yM*z14Le2oU0!Q(*k zQob<#s%#@6uF?C2>>c_LYJNHjo2uThY8k_^O{ggQ~dCc%Z0ct%mT*GHQ2rx z%&2_j_qWOz@s1wXc1{K+8RX56OZ=%9<0AcmdC_B}`vm^U9q2xkwSfMzohQ@3GJaE< zYp!aeYx+voKGwX(>0rff=B9mz7v^a>GOujp5)wD|fAnJ`b4%t*&zq9d^Qq{-QQzX9 zvY0DNUQlxJmZKF*-k$RX){FRioxvv1_@TV$C*y^3%$<6PZ`a%Fh{h~)$H1CPSf_fe zY5H}~-}(U}cOxZLX~NPSt^-`&GYB_Kg6R-6UU1D^KE8=4sQ7}p(*NIusgx=+F;UED zJQ~Lhm%6oaFtF$T7{`fSWQ4GhtPk z6?R;s^Ya~2Ouh+rBQ1a>LdfB~#O;-5jx8_w?AE1uI=*Ot)jLHN z9E>?dn99Hzd`hN9j-&=ObgE}ljpEnR9J{A z+=oNpkkf*;v~WZwii=#9vBo|aBfk-&erwnR@NIqI&YP1_u9aCCCZNn2x}NpjLd8Yi z8Ox*8)MJhBdbK|g*ScsN!f|*q52wPr+!zBO`~p+VGJd!vZ?e=}(bI#+1q>k}WJ7aBq@MWYYl;47TKB zQlK4+A0w0FCF23YV$`X)EmaszBNp67W(xoJ0*6Qg|6IUpM5 z4%&&f$wo8>hy7F=dW9GLf--U=FB{uFn7RoElP+6*8)UpF5BV&{j(&iy6*SOh>}MVf zkK$AAILO_B=3{VR6SE&?x?OVXIPR8Ao=Qu6+6KQ}C*x*5W=%M8fMdY{v zZh#x$-!L3Kc62$;p;RBFXrAbef98;~cBLie5$r#-;S08~fOnyq@gx%Vqy5>?e`XQ`sKb z@5CNPM#9uL?5fO=TF%g+*ZOZx(fAe7*e%#Alp*Ah0f@a`c8QQW!-u;02BGr$W+5(e zOE&sTx3dd(`CD0>WKX#hxV2P>^;`1h|2UB)uH!6Q#h0C$gQRIE+nQK{z8pImx;@mtngu|i?I0a0MR7l@bT9cdH) zue=OQ{0Rog<9svH5(3eNe02vnA7vJLUaC3WXoAAflN}D` zV%~*(%5NIS9S*PxG~+d97s9f*&3^kFkr)Ua+pzk!u&1mrNLk2U=y|-x#_F$rK_=WO z&)k1{F3J-~_!VD$*t}Axfj0M=Q2U$Q&(A#K`(Vr|o>!6{%osaRpOVGs+Xi#E`Y=C2 zVD6MWJx>}d+-rvWyz`pvfb<#3(DG_sv#iYZC9CZNp`~-#)Sr2cxr}=Txo=?osNDQd zwF*%Nx*U0_Z?7l((1`ifW2WYj;d)F{e(8vEEo0%XjwYa5Q`J^DG-1mv{`o49$5edX zu3&UEw_j|yjJ1|km*SJ_fRg1%(`1DAy6YxF0gJwG>5LvYVMJs1A{v zUzKiRAU@OD*FdOZNIQ`JzJ~{W$%AyQa;w0ugX;mRF86!D{g+$c`qrcISbV^qG)DuP ztDr^APyJ+fiMo)doYx0mifsq}Y(j z(C6mxRXq8j%^DfRI(n9IbDEF{HSyYx(j`FomPxd24CSyRhfz6@is8}+Q!(_?hHTnm z?8Uf>p_RH)rt3E6h=@sF?BE_k|DFuu-d zEQd_Vhwd#a8%*R@9_nuqMAJqcYaH>`2S(fUq!e;yjA)Z1Lo7EZ>V4$3*@=OcL&Daz zikQ2_3gx%9TzN$8aWEl53L&8m$kmwoOowd#>_W%PDU?~o#%n+pM>(QmE28_ZWl))H z0?$0_j5?Is{X^kwT6^vw&Ta1YPCXF^X9EhugE^0WHtNVtCxvNmmm$)gFDPr?G{6)Z z!u_O_Zz9R{23_ODZ7n+Yi@Lc&f6p&@*PACE17)~VcFK9wT zR({cIc<#Vpyj#L`!Au{2$8-qS*LsgJSOW@FaTv4+cq zhP5npiS6O1h~-*uh|s)*mWv`Nuk?UE{891UF43#XRGn-SP`5_c zQ3y41*fy9BW>l|HXMb7;aVsJY2t9eZDJNSrH?;F~Jcovaui!~34ho@F9zXenuH;6p zwinhi%$rNqrEeCZW7@UL^1+nK!=(;nqRwr5yBvkl4&~D~ut5h(ZtUXNMP-j#hSa%X z_=%6l*8vj0+tKZf43rO+eFVC%(f{Zpa&1%5=Y256K_>k&HXZcT7v`}oup47L;&-ts zlYUKT)Q7Ij-OOMz8NGY5jgvI9X{+G55Hn<;wU8gF;ebkt`Q5WAVZq#02?PF$j z0JhNyxh#-!s=OW#F1zf>11Or|hr`6&wJF9;Gt!la4wxgDPy8dj>Z`oUnl$P_dz`@! z%XnoUPfT$j#s5h0A5y%wkHemtL(m6#CmHi)_3B&Ir+9sM4y5wKIR8jb>FBlfz&_B2 za~iY#s=o2VMU4^mD5H-clh9@InaT$nLbmp;6_ZXT$er8%xJJ|HQh9nF$L(=oUfIeF zAS>6%5}pU$P+b>9{uK=rRt3T(yow*ZH-22cD^16Bc)$pbk_I%-CZR~z!|k7Ok^5h3 zIGDVA{H=KU&X}JVzw}Su@Ye>^{dv~pdTyx)* zzx#B?ePn2v`rb>=F&*D~#mo=FDudFxuOWliw~{TgX05aAC8PBNFL+luo_}$@ukJZ5 zrL4TK(b&00V?=y4etZbU9D-}?b*9R#vYof}gLYa$*9(vZ!PlA(*zikWM7cO$COtKl zT;;|e8Xry#c?lOk05R;eEOED?%7Ej{6Z+fXK+36rU}QN3U=|KkhB(EcIQcWzf~4{e zA92F_-1olAJ@0j|#5rwBUqkOPyy(R*UjECYA6=oq_k8d7E>C#E6WrQRqczW1?@Rjjz=Ykxk=@}VF0VatQQk&;Fx7 zvYh+ilRx!SmoNFUFH8P{!O#Bou5rNReGdR$D~AoQ9XSa#UT;vQHD3aRXB^NCv;&Tv z()LjeK9YUrnZB_f2A@O9-2#2Xj@%itntm%QF!;)2nUSyLZll1cg*c0^+^H^b5?8TgUdaNI zjdIGGiaH2YzlQd&aq+EfS}AfIhfA>B^~SsuT9`4I|eoz4p2rv8#CGqMMd|Z%9Wlm>X5Y;X?UjWw3BZ*h;Drn zCpR9viL~LAHrpU^q}4A86Q4fXMr5l881({Or?q}vZ}%c zGT%rts8I9r&6LWE(d5G^g(<^_9im09sbKeY`gURK2YNQx&{D2xr9c0#k??BK zqI1eb#_@}x(lO{v)DFi`VPH}bwB{{desutbp5!OY7yx=M7;T0Ak?%%n9`T{?S$!0vi>K!-q zT-MO40C^*`HC*EF#HKECJPw5(V=9aC#-^jf(KB`Th1`Mm5?$Uv6}k^bl05ky1VjB{_sp~nSqpFKI+T6LdW^2b8@jd) z;0;)=J-DsA0Q$;9lIbRzgmKF+>~_Q6iR-KdRj<9MR9w>(`3^YRGKHXNuXXuC>p3^} zcny+Y8^$Y){o@>NjOT`**LX6tWblx#0qTH#^Zhw_O;!KK=_;LMR zFHVaSh$mSlKA=?$|MVNiE(c9K33YD0t`FoXDdISc;!Od5+K4_zU6IfHs;0D+^y|Hj z>}u>legw!5^L$9i{N>m6@q@{JgGAl5Ry&eE0>{SRt^24Cs5tkvsXuX`Z)9$xKPe3v z0o(jNe$*#8%PqP|l5Dev{@BgoV;v`og5M)5PaPmw3z0!wVtV8ZncR9V*H z@JA;`K61lVXgGnN@t zzx9Dr%_-}AD0JUVIEO=-Q%8MU=1ES=;yFI^dLIC>kM+6>mR!oN^0U75oaVKv$8?oN zIl#d(G+yD2J0atpdlKGf7q06|$~Ui)W75*~0Az|>`$1`kCaY{po#Z$Sbs6-029Q+V z12C0q@2)5_PxuT*@i=vjG+iYb>@^V{U`jM3oY`(rHf{mIBHT7@4Xybe|M{^{lK{_gLV7ys#tm+$z_?{K+~`LF-gPfGpxkN>#o!8T~R z^WEjHcU^w+r+&%@J2rHR&I|x4gE~Iud%nj5#DlDmh>gEA_dkBUl& zW&DuBKPefJhgE5eC6`LfBn~&JAd|PFFXeP6N-QEgCD$n+Y2ru1#QYN2l&7b{J`a-KI)??G((8K{oxN^ zzW@8bf1S|anmEQ(*8`D{=NekToB5(=~(3loa z#TBTuh+gt=Qx{qKX0mTAvq*Uns+-Hg$Vu+17al>%O(B+>a%-W^VSpA&o=(86Mbe8p zbj}68b+ghsYMr^P9Y~og%>KR5dNAQc-O-i>nYQ8j)&;jcD-dJ)!n9Qk4kLbtiQmJ5 z`ccnPsDpeQ%~Ml(EtxlIGP?b4b)K3M@~(AWGlBsHU&`@QO)^US#E;P+rT<#_-ui?>>|;wpu@S&NUqe8@y!_p1)m*HIAlWTQ&` zXvay^_=VRf5P8?b5swpZI%Y$*-Q*wPQWJ6gG)mV!Hz#ti2Ep1^*{rQ?vW}$4BRT+u1AVZ_pZxhaJVr>l>g$<3MC(2}ReEoi;kziso?Pr%|h@92x5 z^xii*;7vXIpolE)E9k3l#86(9$$vc{Gdj$nR1T(~g^qHmCw#g2FkV}tdeHx^Q_4bD z5jaBP44EAWDy~U@=u-8mIvG``xQc7PWC$&JDs#b3a6}x47mT72W#g%oS-65EPw~_N z$KGzI_-RvIzqaMF%a@~?b1pt|)Y)!;8{h``w+GseX)d@_w#VhZDeg6oPt6(e69_#{ z;^RUXKWWh70^@kc(EbCw>T@{kq_YW?z!d9Pn}TLFMhbLdqUA6<@Ie8jK3I>zCYY&zB{Zu+z=l+iX(uU%ai+d^8a zvR1}seEHoC3bb!(3;PLRW3h% z;s)PMxiQZF*_?I8a^~52(!@9QWdm{ZzjZ@Ca~pDnW7-QFx(*B#kM3yO{GU{ApmP%t zNo?~_cC{@oI)z``ihbh{#P6Gxv_~PdoA&l6U@;Fs)w%2wGJ(W21Hdi;Ue5 zS4>}&Jw@A9f$oErFS5|4)RVda^hxU1Pe=K{hy4qv94$Nt3n;rwm30==WSFzty z+5LKZAB=&>BUyVdMI9)^V3{IU%eJ-&$;|6(zd}TER&GJjjXY~!z;xubQ+#@!HI^*k zhHH?^kY(_m%D|oIXFLF>)fzRy8itQT6@g>^6%l_VU1dYl{ntlPQ0Wqo7$MT32nYfL zQIS?cBnMI=-OVUL>7Gb8NQ~|hBu69NF>2(3jLA zg5abLEX_hwy{MVmX~x(*+SnjLFqs5KDrM8Ox{ z&}_YQrlXcQA}Yx_(p~P{uIY!S4p1e1-j25r5E*x!i?h)b^g{5LH+_m@;NyKCEySWd z`ilL5V1!ELeaa&6JS1S=HO_CTC-cJjh3ug@mERn4U%%7WXM+m+zE=o#m(jSY4S(p= z>AyzmA4C=Re)7LK7Ma7gvy}$S5EL_noX^J2%M(Qgx3HCnztBV}2b|nvM-3=HTE1A0 zr{jVycM(Fi`-KS&z1f|zo6KXMa2Os@m$a@ZQ$p5R5AdKT;M2uH5E(1qcjaNr9Gb*{AM9aSNNlm@lMG3)qr%)P&&M4EN>)a_e}!Dyu!n+XF=bb z>2^-6aiy-$L=Ted$xMnp$r}EI#ZP9_BakR+u{Q&huL>Eb*=tVzTPCA}N|zU|=jw86 z@9Tm?kcf*2d^X-VSTyxK`yBYz2STwT3G8G_15m{A6pg> z_vzZ140+>y;=7Vc=l4H(g7egeinEgMg3(`))y>Cv>TZtWtS!N`6h>+Ot4AsGeasP{ zCF(q>NqrOWtRO|Nz&oSA6-xXQizQ_)8>j2I^cGGg_@}OG5)MWp<;FN<|8+TN__v7M zO3s$nzs2i|B<;E%TdbS+OzCYKW$%_MI+N6R_yyao8fQONDq?M1d!-ZJ7cRiR3hme@32daPgpO=*_xNM?A1f319g^O{A&{k$h`@rLf!dC zx58R-DnlpPz9KsgY%@?Oeo`N;gbNElh@r6L%-)gjGjFYAaa<@vOnMX1!{<~ZYa>Rf zOUZfkeYEy(lWvrJlD%wQ`AUEaK&Jqp%V%lVgGqRvnxnV&J^^yXRdg?ow(>RfGhm=< zl-kdJSmI8echh(GJ1XA2cV!rV@+dnh!vqqyr>LyU#Uz%^Z(q`!Z2Ry4)wHt_6JX61 zux?~`-9=fObzg-_H|$9GjU6zhH7UEBFv7iRl=3Rmem92Se*HztZzi`Q2|B;p| zyfbHQC|7r$Q6T9;Pdjxu5roJ)l$sZqvu{{z3okX*{maD#=@mptWWz32^6kMhQ`MSD z%DyKf4yPO2znqGu4b{_#&V544_BDPKDsg)>tg0A^$;IFKz!Rr*|7O3bE^Y$THGXS% z;7YJ0>FFi&JAM=GRw5tCDkXP^NE^;L_VEh)JCuS|jH53Q!*KO=W}DCWjH4SG0?ca_ z8Yfost@P^7+{5u*CE3e+GWM-0p+C*@-52w`&P9P9yxE6ri;<|)7^f$ni#>q38edjj zTTQrCw+wFC#*6wEI>q4hE*q}sQhA?Aa2YxN4LM-mH&flMqLx_@iB#EJ{zwFGU?ZDr z(2=Xc&Ld79HOc;P`@C+;-$|+IQB*n4>)A>1k%z*r{o{cP&|Q$Zedo_Z_nR5T80vN% zAnf5OMe8HuKRny{HLTpl@b7Z(0Q^1za<3Ih1v~Q(^dIvxc;4$2u~2VX{$1gi$g+N} zPpKIo<}o@SX!u}@zf^o*A*BWR`cora8xdkEp!Dj=^b-iEnsrAkQa_e`47?s(7e^Jm z3&|@!rqHM_|JM#1m|gRvqt*hS>6o^WOh#x=Q3RmVx58p&quKg2OlJd{(JH1c#~s>G zgpze&$+mQ?Op1EqtD58r5&LGt24AndmyW(j&g==;Av$~ zi9F`>()++8e1#p1N%>ybDkpM~!70?rjF=Tu1(-K_D z>axl`pM6ajYWjz#&*=t1s1(p1G51>$G%EV#iVG^2SKb8${5?g5Sp$RQFDSj<{i5w$ z$D_b)O_;37N&m7LKEy_ph+!b6KhPfaUjqfamcSRM8mR)~F}eF=N#@Bf6);QepE3W6 zWvzVXVWthTyNXmuP8R5&)?Q)9_2P1oPD~i-lYz49Kop_AP)qg{{mN26Bs>fPM|9#4 z8u-Ii+#w9_kDjW(W;Y5vR|dQLoJN7#mM$v2>dPBjrYrsxb+*fX^1z>FU!THN2u~RD zIYiO`P7`2bvU@g5*0QC&f_I3Eg;y_6B1C~#XIiH6o8rK!Lv(U6CK2?qiHv}q0=64o z^@BvxSojd;1Y=y<__^H9_KwIZ_GiD)7^Y&?Gz(c*EDUnbzTuN;A;IM@gyYWBJF$zf zM&Jg`%;3$Pwc}UX?-?5Vv}`FB$2}CRK+X2IZ*p(G{<)`dY82UJ#-B!217sqNqsR_W zzng0zJ2+kFvO+x?wKmM>z+_3|gpaJXyz_v~SSO<{{Puv3t4|UKGx9He)cZ}(_1LA_ z@>?&s#|}sr=T7nlbHkmk`v&4@+Gg$tSy_I?H8FHP^dh4C7jr^yl_ul(msfov_7x>K z(bC!QZlsvMfMdu%uP8~w5-^L|a`-${%0_X2VW8h}@7D5J{}9t0Mmq)%`*YGqw4-ls zZ|!6FK!*OVlO%7la@W!@jJ-8ge{bcEI*FbWUj*oMKKkQJnI#s2po>55l3u&kQ75!b ziKMl8?XO1X?e<%V6DcE-;k144C3F7W$uu+_3}2K24#2Fx%kJ_^sk{EXor51R>GaoK zhPF32QjT|~%u3y%e^_d(b9_`>+9?%iQgnSe_XESst)~VG{_TjemO9;94Ct8K*|95&2xvpe}c`O^M2yHMX^R&Q@@Jiuz{TP$ENS%OrvR@qX~cC}aV>)UuDn8AjkW zcDDKR~^-XDWp_O!uxZ*)^{ctheoDZxqCq|J-#sC z0dw-oLxma9v-`rX(s6lhsm%F{$<~e?N1;V&wuiI50<~%x@=dYy@2wBB+aiz};+8ao z;-*8%rlV6zMpIO|iwQfT-r=pgpRch^nCJ^o9LIDzuqF=}aK7DLK5YdG9An z5av~8=#z7m&gg(lCARaGw!udBcfB)zocv&yF-`e&`+5$MEi(%hVt<9bQfd)dXy{R7 zyXc9IPMF$?*`Ua5g!43>>+KeEfCfA`QaH23bTcOyAHS~@4s#pm$_i@nWK)zk|C&YQ zLFUw_Hf{6o2R77Fw-s&6TJL}>d-sC!>)cHVWlgBsTtK7FQ?1Sq=}gR^ zx%t_qv5Q5%u0XCIo}3giQ>}V;Hf*HRScF7r{B_>$?7y+mA%#E4E0l$NBS>aMom!ybQuS~mK@J#WionlL^$c4 zcAj^hWcZ9#W-#14SPSM8Aug)_P5W}*ondaqw!It|G7nOqPd0Hx?y4GmGi?ns;0-%} z31wl<*MKZ3r4dIgJaLx(OZABJtfy;OxSVd$JNFA|%xrRVOr*;X?!n)b3DfVgtqy$T z%bTI-@|}2GIZLb;W34z)2#w^GVfLbg|nOQf& zw?^x-MZ7|8xY$dGSB)ZBIoDnM)GI0iC^p#`#1uu3IR_wa}^;VSWhloMpc!3_roiYU8h|@VmT8eC`#eA16LwS|Kl& z)+u3(;8Vt%pzQ;<(Nl$WF>>Yv{KLk!LQsBxFs+vDGQ#u9$9wgz9-x2o-@JY&iX6M*-gB*K)xbP`0(bJ$7*NTO=I1?Y}?sK)iwHoKyx!K5>38 z{8JzLEg*kr`ER&p-z@bf4Mc$dDhqX@?K)cs#A4tPa@4D1iqy94*El|<&==DRzB*35 zSs{>{;@PNSxaWV2q_kHv4Ef(bhRymf&r-tA5DzIwuA!nh7$O3E9bp0p!t4?mN|FOn zNx*vlDXD-Y?Cwt-vVscS#HVG{fn(|Ta{-s@R(}@f{g>PS(?G$sJaO#zM}N6qr+Hr28l2g3blYcb0syJ5ac#oz^KpVa(rmB16G1f=8J3%Ln>`m9 z`|Oa0^i8#_jcL6!yRfe@$SDyf@<7)Ip~ijJFLEYj15-A{WHJ0P*0pIe;8&oYT`|YOimuTyGAx5hA9wjy<->l(AF3@;k+4~PBY*1 zReXoS1drwq&*{&X{}=2;1Ptz8C*2S*IuB8Hu3(MTVA8S}HYk$GSvdKb!(b$)!?T!d zGpgAk0M6N8J|kD~J1Hx@m~F#6!n~d7^&T3KvdJTL|4E$x3r2s0-26**BSNRx@?aTy z4GqUE*tef5Jvw&|;&robtT!3*z#B+a@XqCY&8BUUuyXlS z(xr|kGkF9*TuC5V$fT;Pgvx7h>HT6~J`XSN8D|X=q5O$m>>_C}p#9~+KEL7VF>@YC z6fWidhBM%DBoBto);{*%LBSIJCJ1BEnHa+ol+TEqiSZIvj$@zj_e*j5WEpE&X;!m4 zpC|H~C%F!JJ6)R8xyp9a{uw>Bc)mGTdK;Z3m-Q&y0ITttn3Kh+dGda@uT{ou<;KR~ zg3tN~A4U0sw6OFt1D)$!V!P;vg;oo_v|+?`S}eq=UUV5c}bhGT`rb zQ&~!QOP1b5s>W=chP%v-at2(toh*Pj1;-&ro-D7>Z>RNVrTa7!x{;avRJ+h<@%vT{ zAMfeBQIR8pP86Z|?}c$r3_{u3$hhp6s+p6y(VJrn=o^!zR95%#)ao7VmQiQT*d-_{ zK5w}nL$rF!WR8=X#*p4`I8^IKd5Jvp)?CpY+TusW9Xurd;}Xb8-Z7c8Xt;3^G%Ei||X`cF&pE%l_!y+>0I6hClHU*3A+Q!jl32#im~zTeK8F*pY- z7gw53yz&~ET0N6F$?BJFcz#U%WW`VasY2!4`5s>O=`5+XJUw^pDd)=+K&h3GdIg+I z=NYmNr{V>C765CaCR)b+eA zUF0#S@(-tVK(J{aH>tb~6+!JB1Gk9zF6K6uQt!51+R92UV zCeva>C5=BYk~=R%rN%P&wY}K+x7$JuA+HZY>;2fmdUS5P|FQwz%)=N1CK!dU+WO@J z#N7G`*PqXdTXw7ew0=MeRlX@6Ju8aeGr}dkS*v$>-i18|;|YaP=V}2jPEP^&Qy3fJ z`i$Is!0Kz_f7-h(O&oOsM$b4&$fI_^fv5&Tt2~_T*t;K@Tr6A2mkKvou#e{bGOp|& zQo+8loNo;Mmpm+diAXZJQQc z!KtdO*4sxXgiItn#p($K4p0KZrGQM2{C8f8f(Z>zw)q7wcWE5HzwlQ1VKi@c_FvBv zDh7o20Nimoxd`hAu3-MSjmrC>Qk_r_5j#g-oi|7xD&P2Q&DN>Rqk_Bf`nMeyQ?HJ3 zgcH>6TPoZ_{?a1gngl%vy>EK&IVSH zSEwOWYR1KBWRWy+#z)}J8_}zHq1_0j;z}BaQSc|Z*FFPr9i8tn5tU@}__O09C14iq z%?7Fbd(07HmeuvkcX{tuWCk7kf}O5DCEnQoZ?NiK*->c!%@#b(OVFWPykK&p<=c>m z^XIh)%6g)=?lFF9*ZsQ2N;&(qxGSsK%bwHU+L82+4h<*;@buYfNB~Fg>qQpwhlaaE zFs`(PfzcvE7m2`YChfmn6yFQ_Lpu6KAAW*&ANG_C{18ez_Y;ylBCch(x^IwSpc`X2 z-0o8Pt@Yu}Q%3hp$(phNOylBhLBH^k_ZLM4Dxt=>EiQP08v(ufpyN zGTp%x&~p_{cn{y((Sh8+m=8KwHiP&@D{tcT9zLgETQx;*rI^A>j}3cMlubyeAsSye^8qDJ^{ycmRn^dOF-HxlxBfD-)P_ z_b9o#!8m|`Fkeq>!X~(tv2}Y17X1)4iEL99gP~bcs(@XE7iVXNk_ct{%8TumWlB4W z%?sB6J`Z%7P2_eotEzdr3j1}l;lF9SFEi4Aff{xIJ7KABZlTxitE|=(J(rQfuY}F> zU(7sR_YHko^J|`jl1%F~Q{yLk!NNJcC7_O15v0E}$OTwgpIM;NKne)fe9B@|!1dkY zhgs!u?+IKAOFogj4Sd}_!=GIF!v>&~N%4Iy;4Kk!Mh&J&i`;Skp;Ri%rp*2&{AfBB zGZ!gY?2vOBY&GBQaO}*TAbG&j=m1T=cWbunsZ8nq6Z>C(``_lu!TOvY8>7FRcTl*? zxa1zb@b)kerYgMX=*kJ3A-&`Q#&4$t1X85;^`VF44R)LTReYM4P`kv$6qK2-~qi^VOZzB zJ5uF)yosHQD&Gul8*in*amk$ZEe5Yc&l6eLq=D-jQ)1`37F6sOadPG#q+lC*{%+Ny zk{Fi-aO7H|XKk1G=*eHeSRW9w*V=tz$oZL09N&kux&$nQy=I`$MmH zx%v`Xcb)=R4LXL8V|sAZ0?J#8BskHGXU|vqf|I*VZFI9>F3hD;c47U|GmrB7Vm<_3 zy3NzgP8EMsS}mkA89DpM0UxXCTO%0uG^F)r7DNU(A)XbU=L@T@AJYw2nl++GWr@9M zzpTp3@+S`@_DSxYk z?&rH70EVZJX)|#hSIuL{{xP3eM5L^f56q~%=k1z~JX}YJSL@9A%U>N$(AhNTP;;K` zh?e=RvGveFqjTtE3|H7JX}}5T%T%QjD%3cY3u_d-m~aitQ*#sGTsa4yg{kbL;&|_r zbu7Cx*fl-IR5mCRZv| zbJT6Kj2dOn5q+Ol7))qkM5RLr<<7GF7eo3Tlc>NQJSH3N$9^*hX9TY^`Zr0#aPoH> ztEu<>+j*}$SUOH$}KT2VJG(gQsCuGzh8BKV| zbDi9hNy&s|k`!__a-n<*k;%{J=TA~(=8n(ph5WtsEirv|FGy%=Y`Z#;@$QpiaB!A4tf6R&Qw-XX`xK^#&jQQ?%5WedZFiqoiLa4_x11$fg zDq0Dj;IsdVWrrx5tCE4O1>|9B_O$u6>2C(9!&Fa^2T%8_v#PVJ#R=VT=EXmW*G+y- zosvbA446ZlZoEuHC?(&JKKKYEZ56 z0HEja%|?`{bd9m|dM~V3ITj5@Uw`YY+Xbjy>yH?gxz2H6wf!H6Ih{e4niF1HP^zH! z;>5>fU;~6oddpg8zTWKIvfGuEE%}b7+0(J5Hr{8Ow5yM|-?oomrVL$-c$Ny5N0wCC2c5i-%2$%6v=C zB~Dy!DQY(UMBbc&l7cnjmt*n>_0HdMHz3yc7lJqb9EHBEfpz&8wDUnHmF@9p7fR0u zukQ%^iQrxi<3ToTB>kVLHMVE!D^HG}eI(?ae7Nudvgwg_Cg@;Qo+UzLrf@j(D7y2 z)(KMG6(>*7hmU2n%hkAND~GCaO~c2#|5_YhU$|E66e#Unq&Fuh0j?tY3a0u8?I3aK z)?WdG`B6JZ!`$LCvi<(AI`x4RUmYhfi9TWhQuhG+pN8)ar~_ikhu1mm0?nsi-rxtO z+?Ytsnw689hp#xxr?Q;%0EqXp2I6%ew-h3aHGOL0r6SJbp4+XGYvXRQ<(h59EESUs zDEMt#$UQpLK$7!!voq?$i%huH8uMofeyQDI8EqN&Wn4>`?K*3Ve0=Lu9&yKvX`M=+ zV~Zo1WDxc75d+Ga*@K79hm&URGeY6RVb||9lQ}vMxitmFtPGzR7P{wJq!Ks+c-sDQ zLWH{i&*ZkVN|)kvM8Ix@2ms9qsF6MP89Q6o`!e&)AbI%~l-N4*>HTq<2fG~L(;e^; z-okfJ4YXgJkJ}N?s>WP8;Tx!;u6f9xv)*Y=63`-<+`z}9n+UCu@Fe-YB>(q4V1Zl9 z?+5x7&hz#(OPaFI$xv~8S*#(9bVC;GXL7cJCf>AUCV5jcJiZFqOiyfQ?|p-e0^(L*cD7iqOZ+Mp=uh>l$DH3 z`pRUCpNo*YhDuRh$o z(p47ggXHhiiEfbjLYplsQ9s1LJqk1b79)~-mP@v$KA7s%#`OyM``L_h+)F2CU{bGuyF9C!Y}U8b&G)l7?IA5CiCj`zN9ivJksfA&0K zP41vPbmJ$a+Y8H~^`04uCPe(Ld&3+$@Ltx|3;%Tzy8K<92tivXKY?9{;}_hN#rB4G za7~vDQ?*}~xf0@(sWXj$c^>s^!an11R)VN!B9-Sq*Yw2NXt(E1)gTtiZatx(PtW8l z8RP_lj?-*T_Ve4EfESvbD~?r)TPuAMIkaVk?n;p#AQG#8Fxyo%{K67H&D&m96|rWm zsAx>MIL1hD9LT@%pIkVmNIs8@oml2s`qaJ_0%vmq_aiH%OztERWU2(Mk7yY84yeQU zr1A{1*&q8=WP}RjN|&?qV~r>RhCt;9+!e1dr@pc>E-txW^nJNbTK0*aR!5qRflV5i zOwNl;%AVHBxW^zn#*pzdu_AScfoQ#>?ea6;z2ZkY99erTwqK(bd z!jP-h!wLu$VK;;&^jn!{xWnz@5SD7e*c#gq~Wcr3)uCl9}0i{6N zgc`z1_AE2`8TlKQpJ?JLr4X`T`QAI*x_d)1QF1mJ=+8S8N-7_4T@K5TCvP+@N3T264(8et$MFGu z!Rh;u;?7COWy;-2Pd4nZBU<~wV>|C$-_dku?{VM~PeEqo+I^m^&zo<(F7@klh+UiI zTb{RQsgR6;>h7LFwrVrFaf&~8^xVrz4OS$*w`Ytg>;ae0*+l{bQ5wqgbtFD!>6Shy)-oJ#i3aIyv zo5(xmSe(r6Y`FNEU=mj0&qN0<(As{j7vte#r@6MKGu+r%NlQ(HkLh~Ws_U*ud#vPW zM)r-x4@5^sAP3qJl2E)b@@;!kn;i8mRm)XN!oto{~bE)??BXv!Xj??bm|ZIsTHaO7dfd0@k1n)Ejc>w@qM*_vEhWRa^0iX&vX+fl4HWuf=}4!8UUw|b|;c$mR1?h22qhbIxZI&Aj4AXOKGV?6G-GXso6 zT`o3Nlc)WBvFm%?2yV^ZKp!aseNh#w=nD805IY5N^1E(qO*2L7gW<)oE7yHeB2-Oy zEQECFI1q;YcD0GmeDOuVAR67aPT?drCW@^Q<+k-XOJCU{enQf)9^Zg(f#O@>O2Bg^ zNWc7LHB4W2D;JjEarP4H{wOGeVkR?QCB6p=#_fPt>TLJPP2Sx5Zr3$d#*tQabQMIZ zdr;#)2npOnV%r+tXCHrj36OLB3q87t*1|srPW0N{F`yTcXCOX(*rCNStnbuA-%0h( zR+gXsk7`*NVxvp3iq>XXV@=_6gRY?cT*Hy(ZO2wLYY_hsJ>_Pf(=1u9th7D|I&3Q` zY}_Z5XG(4NK_~=~tn(UvdrM`QiFBKWF_7ZjQ|=Ju8tk}1dNKTRW0fW1l_c1q>0Si+ zLjhUxS}<{DRU8McjMe8afOVSbi{9h-b_RP%Ti)h^B6cP+D+T%mZ$9EYM4e(v(g+ow z>0xDIg>SvG)li{jkqP+=M@Qqz`fl*`kj6C~Zz>(cw81#{zbi}DeO0~d+7R#lUm^?d9&ZO#ndHu2yw6jQ0cwbjcs_=klh$zGw4GYP}wdcrL8 z&{`=U%@36FMA~sMe@-FIr@s3NHj>ss6EEv=MkEv(Q~&J_g!|FHb-u2TJ*y-6-+gGl zXVA~l6nk(pmseJpSi@v$P+MR!2Qys>h*)5XrO{l;}SDS``?Fv;_s))Xtx zrG{mTeuMvjFkgx#Ym6k_`=&m~@_3ZmQ&Q%~n*L{O-r$BM5a2-f^%p<20~e;ilmzc6 z3U0s3aa&j)eYATMDL%RVAcVH%t{;9Lm2b~+&=^A59 ztQzEO)URxNB+8%r8PI)j(dql#ySs|~X4 z4ogt2Q#o!8MHN2H6P{{ta}ACFqP=npsli}MsR*Yhoeu?h&$u7i*W@e53HOV({?lr@ zSsO%f+4(w`SG-yctL5SX4~_Wl^>EK9S46x`*$aJ~;?u`0b(Tci)__QDY(lxF8?AOn zXak&Y7DXCcM=DZ{_;BPE#j>{SH!;3po!R%0h}ydCve z2fOXnBzT!Uk8l6cqO>Cfh)(ec1sXf#zuU$ICm%{rljZ;~%sCwF(|V~EawxZ^whith`!1>wU9D^_t*4sqT;*H$RCHtJlm%(k zQAxUGvFU+KXkxmBbx1WwgAx-YZUSJ_{PmWz14 zwymK0&iB&3l-p-Hypp((HSAN9pBa~@CR|FU?c1L}_s73IcPE8;!16*WffvcK328nY z5!%I~HXr2z_~0!Qj`KS|h`U~i zMSlTMuGauAGH86y&6b8Rq zHL}1Dx)6(YtC#_~M2^icUX7i4i+oYz^B&UX1mX-VsTl~f?y|mI!esF+Ad1JMilSO^ zBeL>oG{sc)=XJ`0>(DiCvT6Wl7r)rf=P|z1h>p#7xtUyZq#L#~+)X;guf_&1Uq*2X z7AxL|6M}H3f(7%|9&Z?nuKi%nvTVT~u8v7~ualYX5@%$i_p(jpadh(Ls9#X*FF2Gi zl3>K0*THraWY@ul?aNAp;~u~GFb8ebA?>{`&x8(gX` z_<=24B{x`e6l^MoOo%haopH0T7lXY`0K_3mj7$XBlaNk=l#y{yD-X|NB@l~&)&%Mh z#RznHkD~`^0zw4UF$NeR2dpcV6B){g*#zS{wq z5r5;|(N+c%Gc-tz`cYZeMJ)0^bk4M-rk_w11bEDozC;@bgd{YOMy`gJZBX*ViNumv z*JR(kFfm_f=+`CheH8=E`uP%2*g6=WPaSGTX(Zpw8=624y`(jeRW}3P*L*%g67Scf z)B8Xs0(IYT)bv`Pm?)nooXC8NUf3AJ+I!3+uAO?nt8IA zR&2Bg(YADxEi3$<;qjSuErJPL1s{Fl@ruP|JQX)wTeTUj`D^G&p5-5N)~croG;i3j zQF4;plDo)9lCIa=_hle3Xs!Eccb0#BZa+tod0!00PbEYsC9_W*%|-4fq>s*%6$1F~h#s`_pl`DzPuy`cMMiZC#*ZvON+e&}>Y;s5zqm^*Uyx8N z#X4g%D84TgKpASfi&P_M`<&XW+%iK=Enf#h6#dNOsgh|blt3Gi`*_QDs7wjSav`>nGAdX}x^BzcDBkOy9?yazbrbRigBT6osoNKMl( zF&p~0EwzXW<=AXo%WUYQ=#<9~`Q$>Eg1Y*eck{>~XRD}K zc=qbGC~UR!(gOy>9;QRcRta%?+q70D8r_7DGiqa(>h0MjJ8UD_Zy>(j^WxO1`S*$H z`VX~AlUR`VwCfgz4)$UF)XR3H<76NH_sy&jqyzCAKKD`-B;s>p1?zQcCF}uho3CQ3 z8$*-{f1R)vtO+YeGahZEi%vAPS~e3U1N>*xMNLQXvr(yMcz6gr)5(7pc{$tZCVdQ8 zX=pH4A!*`430X z?XB9~do;K7dJ=8^{t<%aaxNiX`7Bu^!jI5D&=0LlcZ?~A>Q|dSo7FB(?6)d2UIesD z{wAM!ou%1#oZfEO>TNRHY?Ix-^vZA8$QB=vZnR@tYP5>nhK-C#+ewE?EVmV*t7jNM z*C)@L4rwNJ)h;bN@ zQ?^=Ue^wvlKkE0XIUnmigY?+CsD)t6+(8GwlTo<7D149;XCV>5Bl%SfNsiSq)34sABZahhJkLsn_Zfa8Qem{ zKJIaA$Mu_VD`5+0aCk@g>cyRj7tEyEyQTs14LnYo9H4#4t%sV0S+q?JTX9s!l*6nq zY4m>{ljLYFs#3jH;4T{2o84Cl-2eN>9dLcl9?`CfwrI*zx-vNzkn&@{D05{~Ue(D) zDDmT0Aqx){<7Rtz2FKKvt;fZH1rFMxkv0C&nfrnV4)JuERLr(y&CB=%I`(sip&jLY z*0;f|yJrthqe(B_naSjEvq)}Yg&3)pu#ba#`qd7Hydz}yN6Qr0h>R`W=?e5 zI8jrK-HeP>tAldn!%h+WW?6T=e@9csxzf&|H6C;9j@^0F5L0f}`{y@m5qddd{NAlU zSekq8Ecg%lxBNSNnS}Mh)3nuhf{zePjb|wxJC}2Xd)K?rr$O`&3-=RH+-HPPG1sy& z>wsC-_2!W4Eb-l1J?JlS<4CIn*5Cbekl$ly+?(Ydj>_5Uff7tItYgl~+GMOnXzPbE zWZBQmeA;bSOV*8v;T%gmj^4`buOD5RtbZ07XGuhN(4q1*s zJhEx|9BYS{-~F3rMmcsG@h4}Vw#;Ke936e5*3bCvFN5G{hcu3;()XPS1OGs7GWWr$ z1Uv;C#MS7ULGS4XAQAPQh{CMC0$)r*1l$ji1U8X?>=Vo<@K~?5O=K|`T?``_^{IsZ zi+hJK%HmO)PF+_^JMyz+%7AyQB&;=6RVS+&6m3ICgAdB*uSkzUp_UA z0P>#FEq}5o3lFa8mczg5vOQ8$`R+1lTp!Z!#%s3lBn1 ziee|>P&^tMuzQgRY5R*Og;VzRvpE>xP}!?h{)>%)WmH9L*qTmMI($$ou!c9%34Gl{ z<$(K7Eo5Sf*ZbJ_CEd!-2=q*zzGuG9Z76KEBuM$ zq=YGXGXS*?I8|JYBP24k2DW|N=?uK`Y6ath79NGJ6V7LTSayk1z)6Jv05aM}wkM-v z&p+mVxyvz~7&oNu?vTsDpqh-q{yV|3aHDAp(NCpG>|_*wwQO6fteT61tUZI?G;ZIa zGts{IFdu{A-;{Z3H}U&E%*VL>d|L93{WX6^jPB&GC)qloZ2VBVUn?RToZtvA4hzBW z(Qmg_qBPVx+80*HTHWnNV59_9BJja(7;+K8L35Lfm?R-T>rr49@!4@=rgg&(%)ZUQ zEP5^qyG=h?s1MxZ-UrxwO4qFq8gA254jYjAPi2B61=)JOcWmLfaqD#KN7^0ltvPEX zG_hq?(QP^=E$rPL3E3=C$#>qTqxl?@i76l5des~_P#7qh_O!vjmLyd3=d9$`H2XNY zL8~#RdVdT(d5z?Fa<ZL2v@7QEy&Bk7bD>~@9Z3cyw+J+Kv zF9x%~Xj#Q8hP@vwEIx+&NLF<==>}tMEmaq{@pRi58g~_N*Y0X^an%i}hrT&W#^%07 z{3c7r-Ro`tUCZd~LAdxcV@B4Qc)Ekc7sl8otQ+esf{1(S_=g()SZwq5a@IJeTQC?E z3`<`*--dhO2~?&N2z3R6nc}=)pw5Hi)LG000wUFpZi62}!LzRGMuo>@nnJuH&92SJ z-4xh#f2HwCz@8q=_1gn9c=J~HzE5w0m*}xStQXdanOcOLt+-FP;uhmX8A0A~*Y(y) z!%=X53T$z#^|Md^Vx^%-vzdV=Ym%u5D78!^3cuTXY)ZKPM_f0w^|Yq#v;JKevj2Nb zxfRjNLF)a1@N>AuoRA`!{^8Jj!bdq3?B9!kbYjjRLeOiNzR2-4gbd;}*17okw{|A< z;bBT7(8b?CL=JrfpX1RNRW6u~TJ`8Ya@k{RH{tvxSTF`t{02=7T#cz*ne|OH)-NrG z#g;6KH2F5~vOz?5Ixps2S6D<1SI*;(jaN>toJ??gdQPUMqE{49IVaT4Y`X)hSijT9 z=WrDZL;mjc!6Jt`asBWVQKM5h`1A^<20}){MRAzI1$|8Or9HTzZCA^*t?lG4xTfvo z+Zcq!|Ib-=W80d~m`N7Cf6O2Yp8-s~QHGs`_28Fw_%QvjvurYX{qr#s(-K*5Ucf5Z zH!v6bT2vQfhndf5HsN@fVr+LtO#x{-cC}K0Nfr1+#?CJUr(M7)l6E~CKc6iE#w7xc za2ujHe-y0qn(Xio?la166C@1I4_JkPE^FWnP;V3=bj7=u@b;TUxZZ#9D_;+nP4Qaz zEx=25Y2N}zdYk1!wm#Ndswqt#KP;qv+!ttbX)Zq+&Zl@O0ZwN9QQHn;{5#B3GJ<$$?>ai+Q>7rG%WRok%`hYr(O0i0Os3;hEqd4z3cj?f2N%wJWyxAD``9NF$M0hfhP5=J@l|X90QbxX# z*I50%#|C{UKNt$y29$iZU#fm#S$yYcsJ<_5O?>_MwOEswzMP;GJGU`mMoc^ZlK=qIE&Jj8B3 zcJ!F+>EjO5ZrNvQT~J>ga@EysbK~rp&~G0|S@$hx>7IH3mc{$F{0Nag(oahnwGMeq zY;Lfp|8HS8yh76s*Vh1I82yRgAR?8bJjv&|XCKyw?*Q{FR}61*qp!JZm_Xbq)5uPg3z)$gi3Grwo8$$BH& zfr{G+&i4#Z$%RSd2^OWlVG9Me)`OiN-^&DM(W6@JL*DocsT$ZQCQ=VX= zj+xWFkEf0$FMurY>K|&lESITysb!-aeW;nsvHqGr$U3neNU>K7Mvjb?npc&7tFMyX zT05w@>w@GDCVzMhzJ+eTns?j>+@{Diq2r*3rt?-Ihe3)w6gqp8?ZYh`j4`;KtFfyO zrSTKDidbCv#u4T-LP}V1OG6kq?cQJbMc2OZHLpuZKnG|c=(sLl9fR=OE8!KPUOxRZKHc@8W2N0rTW)u|+xb72p7pF} zE&uHaPdHGT^+qE(gqH@ocKO3+KYO`c#yzsGkh1K}5u)?SYC+o2FfH=Z$a{ zcP&m%Z0Zy__uK8ltyF)x__~1}BXG(B8~!#f@YmwIE{rj7oey{7(csWd+%8zIa_y^d zgUhONg2d{guZ?LhVAL-Q*lL*KLK`_A{hg1JA;IBLeuW59PxQ(rK6Oht^*c?Ae4eto zLWt`H9Ya|za{;es@vGlNziZ52#s39>GN>^WVj$!fF>Q>1ZIoqWFxMy8i+5==5G5!k zdLZOF$;E)-283UYAfx}xv(NIAv%Hp!Kah>F6+^KU?tD>J6s8g_?-i1ldPm+e93|h{ z7G)Ssva@OSQ%RumWfLAAGpzNC-$h%73gh<0bxcL`e@<*{via&wC3QoVycazOb?CPZ zLiFNa6q8r34>yE25mGN3ZTtdNt_`-zn@v>8M29!`|23ZN|7_sZI5*dPdamMl{e+(^ zq94iirb*~Oq^AC5kfH}Rc|JH~Bd0XV&ue_r7xJrD#i@MXgn&l*Rs;3MPPd+Ylkn=i z{=$bir<pxq=8M+x*|fqhzlP-IKwhE3O#=Ti(e+`2m|rJ)ZZN7@ zRA>0heS;_%jm`gv0@!p@W8!qVx4!i)m-oN@?Uvi}>+xINGJgusIFU>J z_(eMhSNsvZ#{zZNc%}`hliNo~`Lrd!_Jywd8|nSu3&=sAPM_?j_j0h>H(we(PDI<| zV}{*vu}uY}umbx~p#3P)Uc)vGZ~3h&dk(|{S$9b5`EtdZ-{Mvp2RGN)ICnYw?6a41 zq(@#M#f{LLo}uu<-}K}LGQS8ue(7DP9=``Xh)M|&c|jdF8~IYl(@zE*8` z4P*MM<=WG(b$|4e6LMPykixyLNk*=btNN>SbfX*5e=~j+Rp})!^TgH2{q8}-yq@~P zU+M5yC4)Cge^uWCnM~+F(spiFOrt<>;e^iEZIgj+GKFuq+FTVKV z<-!XtT+Y^(kIf9DmNuCVpiK)&S7+)lac|8>;b zA{SZEerg#xhdz{MuFaaV4VFB<5jQ0Roq&sMC11~x^mq89>$dZp?lO}G_^LkaK2YO= z_Os4}tQY2~!Do4m&bsA6{N#2T0mtmPa^&$f{)W!C7uGb_Mwc<3R7c;KW=)XGcBaaNxi{@tJkwKU5EmPVHG)?IuYFhOep4X* zrJk{&Xxp?$+65cbzXlo@j62vkxQA>19>QrJg1XKE`5(D_2`Gx>ss-*XeuvuPWlmbvi^L?p>@`F zRDMv0o;)Ab+?{Ha+*Jp!pU5pYPyD6dvPQI>Jf6Z)Js6{TOhF33e{ zPVoH4T+jR`TNF9V{$y;>xBHcOyb*}~0CNPoQdZCXp!;*7`vC1Hgtzxfj6-y)_INQhjA;;gJ=V{Vol1X}+}1&1uOB_1xKA)YTaQBOqYHlT*J>?jf9LJ_owmx8N!hDV zU$@t>OP8avi^q5sjpr-mP&=Tr?oVzT#u@r8H~lH-bzI1tV|`iALgppr9QA{q3)r7g z&XrfD-&tQjE&xz|~@x5s1I6_tj&^1xNQF~`w% zHP_9UISv10S2$O4TjY`#s_$lO zRXx*Ji7!`aE?c~;zZwIS*9Utpqtb`G!sbgE{7}TY)0kOxSpF)@y2OtzjU}t0-5p3( zQs7}QvaUSrtKcX@PV2^FhNzvmfje;y6#ek~T5xz^1Ba(r^sZI~yQ_-~_c`?hae{_Mps?u;9* zqgmC;kA2+ZmWMv{q06Hl^{C}pzyJFcy;YjaAk4S)=rOn&Du~9y#1E_P`vukCC8S>L4hZG(_68t8gruWehZuXYI4`XA#Qc z)p2|X?8+=I<}n+(UL?Ix_J$B{KD-gOVV#?PEd03HT(n^y z<35KTxlzT$>Fs46@l~Q_kl63~glCpmB56 zhcS}d(7FQb&yBk)OI}>xIB5e%b?l8SNS4SSxm{P)nN6qlFx7u81BZIAE*ICb;zDC5 z;0mUGpl^n#Epsy{HybRE2(J|_c_}~0Q!v%eLN?g^D%cx6Z6qbS<>#ry+!)g4pSF_h zZIJSMKsJK6xcRyMuZDBD`JxS|!+j_<-O$M)PBx(E!Z#`FV2X`gA4)k*Wd?hr6-0J# z%D5pRJ~4=qhyEscizj;Ty7-_;?e4Kz^to}HO}B4csUB>gn>L&Ao4BpG8Sg_WZMsif z5gGB*53aeIO?hsJ_{KQ$%C$^4O?%K}Cr;#-m6r;sgXvUn+Gl?9>D+VA_KmBv&ekUT z^fTNixxvNZMCL4^_|tz{u1kfuY>@pLByEVjvGL8`cUy+km$qobK1iGo=j6ZjEpM^h zopEfhYN8wzS$ZXg^xcHFP~1Jhpl1uwFo@AQ@YNq5?k*JMgZmP05S z>$Td8{>b<-DxHnB=N4hcOvVbY=5gJX!Q%jh>SCv$qBe0`$Y9BwPylzo`=MXD&bKkMb;3*{RSAVMSG1tJ)8(uSZMz|ZyQ%E||9$!KZ;*h)LUy=Fq zn))Sm#wG2}+X>F*|7+yW6W{1qZprAi4WvF5XL%{RY?~FNY3j*%A|0K|wV%2dSTsTB z?Yak5hGWr-ksIVLf#2GAs>`Yh+N`{S$-BZ#05mU`8(9FzsFQU;7P-!;ODF>EPu|SOeW1{7+3jb#x+u0> z!VZPTU&n6z<`YlCz6ugK!j^FZgK_|MYV_P+H=bGhd0L|9C)OeNH}xy#8Q%=mJb&fo zIhZnM>*F-vXy;JsEUo?Ac)yv}nvDA!FK3;3)^gT49DvB);-#=}VWj2Esei^EHdPLV z`r!Q>jd#{kZqDOJzWle%dML#YWkLFo4}|6VX1(Mgp8rP0=4sk1ugHz*9L#V~*mmA& zwikSe;`Y^?pm@d<>(c&1sk$M)Ih3Lf*v9k^^#>nHX}-oq56D|~5Ko9);zOy6b0~#P z!jBhZFQkX6aQch#d4BV_r{2^xb)uaOkRmI#8M3RtkdL{DdiJl~Z38M_A4a*KsK3B> zZaYIc@a6)4`&eg2#z28Fiy^@ zAH$Isk%Bp#>$c8o`?6l}Uu%Tj{?ax5VqVLr09BD=S;syNzsCsej;)ok(uW|ldyj{< zk+C(A%TMc2SN3tD&srTGtwz9yO4QeNmi_L&5F2!RD0PPVef9%6l)AC|07J;HSf}2F zZbPBRy7{t4y2krP$IH*b6m_=EUSr$`QX2Qvi?P18qsD`Hjdcm#Ro_TIj4nr?R1G`q zIJ@k9WYc2n(iT-u_i@#g^&LC>(&K5fvA2`U8oBNZ>C@3AZA4!hy6*8;eT;MinIwbx zF;?JBzn|Rf&C(9Np4xV()@nEXglpXLjyOPB4?yb?vN@kZLdqhaT#)GY@9^quF3(?h zwZGyEHf59_`T>*o04TGQN7i`am)p#T(z>OM&(;_NIrQ&UvBe>oUwho-+FIS^D-FQGwxhJ z*mgf~{X8$PbF=M7hcgBtv@7WK-MlGdlwbF$O4mT=HUHArR)>XEmdovVpz@(2%i8=l zup2%#7`Y~oy}H;pyHz&zu%CeA*6=D(GM>SQ|ag7?v()GYaqj61;zO0EM0r!8Ok zRbT1CNG@V9}brg@Zb_j+%5jdkN&9tOX(qB z{^hhLMByfP1|GS7?(?6wJngAZ^(*~G2jElntLy@JPvsGl2oj5b=mRsagw0-;eQ1iJ zI{+26rtHutNX(Od_DTM~sZaT&PcDHbIGAMvRSjzCK&n=CZR7);{U+{9K?Di!_V0^p z*bToIFqp)%m*);m!(Q5bFcGd^`L2t>0X?nIv3pt2H_AI;AK@O>)j|iMqSKaXPuv&) zERbacut4_WcCHh`Kl#J2!|av$l}Ohy3w|%af#?Aq|MgMd8vJ&m}Roc{-vunpgX&i`?M0u`=qBy2PMq zp45%@a*a|oOOI2UtsreiUHl(QLT?c3*R*V8{3}6ia&9IJRK_ZR{-FYEgCu#Wv;6rr zjtn5l(*1&dht4Bw>d~8&)V=+3<#!p#@1jzZ6id1GSbxL=MA@CL)V9}O+VV^vE^+gl zr@%1kWX#bXq8nw)ZDyy*<>txR+B9>sf}2aKjBjwDOX|nXgGTl6(I40xu1`nh7r}if z)i;28Y_jp}O{n{hbxFTqQ_BGd2N}FKUui0HHOkRNZh9hnHi(ydvz|Vh{t1zF?)JL2 z+IBr?^4OF<+!xrS%g+rM^psy)=GUUxya$n`H=&oExNICqu@PlcP0L6;%V_=7%_8{o z3O+Xav<L;w$4JL<@*Q(sy9H7l^af@@8n}giARy*+OqdqkC;jeUdjq*}P|5q9} z)a>%>+GEEppT{M%9+%VnME#AKyVCyjp=G(H+U@r5cbnxlx4xARsCaVw*0;I!a?4xY z(sakQ@jfd4N00HR^eG#ixGk%%dw$^N3iTZaaEy^WJwdsBFh!U)PQG`W4+m2@l!|_$ zt2~vlxgj&b2+1?2G#TAE5Rd220gRjS3hdQ|F%|#pIw4^~_%#N4EVLc${wSBHh;nm{ zgC}nAtnI*fG(Ar|!H`2K4!>1{8(qr?Q<|?%6ixYD=;~gpa`E`4B)Z{a9l=U2owF4& zpcTVqCk^skwjf&ixDOuXdJZnWaa6t;Mppm%JL60){FK92*RyoBZ3mdZK`4f}5is@h z0Tts{DCIf@FbhagT_L>5);c16)X5p;aXj^4ouMvdCc5(2&m{qP&0+Jxxt4GR4Aif@ z@>^$&jcXJyxjbK~?dcz#3;kKXg7_RcVDefO&E6qYsm2s^3^9^$G5^FT(>Zj z4k|oONGey$HpO*%>y`G}_EoxKjdLOSStq=29;XFV-8O{$YgIq1w`W$^v{f@DPpFYrZ^uyTD*w5@E#7k`$Tah`U=dSdZ&8QuLo<9?XKl5ki zO}jNlkjZ^ax)nsrbu zjGfr+!TuovqgncV@B3(vK9u6s&nmv^&Kn4LQi}aRo)X0V_TET*XgBv~*RKzySaZ@p zn0I>5#bFe2K9o}497+i#^BL^rs8980uZieJ=>A}RWlo-Nh6mGM#yUgo#NT0k=$G_0 z>eYQ3mnSOyA5wDNc51iFE8JB#>dkmB9q(2u2Y;d>GGS5Ik4uEnhU)C z);!&F%-{v45Z5hrq>rw3$rulMo~BMNEBU-%n#Kwk9dalI5oD|<;bDErMR!Jx4dQ~( zkWM-HOH)~C5894OBS??9E`u$OG&W$$1akL&9%Zcg~Z*j$F>X2dEifuT|9CVw(HnRKRKiy*ju2$PW?EPy75PS^v8Vv#qYYn5$<;W#`Su&iGf$W>Q(-4rFXpJ?G*#;XUzXoaoFRVckdDGldziu zUB%r8&wK9kme;-Rb<6qheCL4%Ot$31AmIP5_+P3uzIhWY&kV(I@OAIm(@_+WIJffZ zu9pqWlTrK=4PGJQM5HMVsNl1o{p{t*Klfz&LpQ;mEHlpOE@zI|fzVGBGpX~g9#cU( z{Qepf=(P6|27|%*AkPuGO*ue!7)*q#SH4rN0Cd+A4&)$WH}a%xr*>-CL?qDt&zqnL z5@I^s7q2Ux%G=@ix8S%+Xf-rl_xDvj^5i=~LMu(cB~NiX$&Mh8%P4g8aDAHBPHmul~fn{`%E*B%cjiqb?6{b8b_R!gYz!eUiM?68# zC;?;-QZ5mEDW}3FcVxNt6s-QaZgYGmbW9e;#j?$M95Uw}OZBpXCt2f~J(jiEQX^0UcuxrOwXVJxBtbZ0$ggUhBKv`$q{bQ@XH za)^<>mzzrJVXh+y*+_PKAZIq=Y{>IUjMjGIeZTi;@@ z3*#a;D{^DeX;Z<>H`cG}u0kExdFKtx>`xzvo;Q`4ya!-6EzJ;*u!=U=TM;9A4%^=; z074E3V`wV{Sao$dd_vB8@a|N^tYTPb%eRpr5|8 zOSqooyDYDA?H`;f4oZsj1+R;*-I*WM{uwhhm&|mfw-p)R<*zho2VJ>6&Jvz^=7zRq z-_-FS4xQJ5y5p25ZM~uo#68Eh4mz#s2&TVhzMlO#)U3D~_q{$c8F(%zWbB*1 zX^ETCJduhTp0EUh6oRbKUBjlat;z09w=y7E_@`bQ3a;z;T4N8FUA zy^U4hnoBL~O|xz`x;&Rl-BLfd6TCcD#9JotR1wVkfLURUVnz=X!OR;zSq>y>% zRbV$#TcZQ|Y5F~5ksnG#>m=K!h_(Tu>+oZ3&kvwjPj5DKj7uJmqeAPLa=I)b2W1=E zNp$Tuy?C6)6SLg3lP+5J){X5o*N!*vDV_G>Nf@QK9b%+U(B|z{-yqD7C=`KlnYyA^ z%Y1d_(|&?h{xNq+N3>Z=j(yhhd;OrkHD(|ja;PnnPbGP5iN59_=3MjcIfy<|I%nMK zYw;S2&E%n|1aPuOXFQCX%G4VjNEY;$!>Alc;i6Z5%;$X*_8rXiZMzJ)=znfwc*u1= z=T-ZwfB6Sr0+ChwqLb*YZ3Fr%I`cSy2I47S zpW+&O{M#LA`~Z-%(Hi^ZroWe+ukzEDHAj<=kIM|txg3AwrY`6?Hb3hic2d8Az(4vr zdfYz(<7qYy53#X&?WE1%qf#g}GL#(IXQVGM25)vV`b2c@{)F73L3ZSGU#R|WTSen4 z>$v(qeZ_sBenx5e3~(D2xPO!vy;S?*rago9C#f%vOLEeQO(`yP)nDtglDLFgF!JLk zcJ?Q&VffZ4fdUmy4tsV0n&vlRMqGKh_IJJ*R^J2_i36wcfv>RVj4@Z%y_6Z7#(OY? zu)fj1hu1A{bF2AKicUy$T#C1=OFHOuC`GwD z%-$n7gvZHQZ&Q&Ac;`Fb5zk&IB02969i92)WgG_Al`H8j6OewiHq`m&pTB9hxe0;J zdcnKi6~A@ge9I-J2>~$)pSo7(9lY(CGZZwN9sV=SWrw!>g^nl)=^8!&FCLnTol*idB?}lTwX~X0V!%YN`+)1KORGeJTTf!s~|luBn>xl$zzI8Ctzq@kGLt#AOh!Ei;;gNN?PPJpXONgi_FwJ zej9C&330q(X+uXij4z?}=6vJ{_ZG0*qc>JIlB~B$Q2RR{Vzf_?B%@>76?D>wu^bQx zl}N1Rsk%`91}qPKr4D#1jGy|jk-~L8f0G7ZxMM{2&6d-(;l`gssXTFH+QK#<*S;g& zNhjWfDUOYpZ+tqw~0A97Ck|B8RwH#%>Sj*rfK21o|X0Ku?Y~ylHoD6ot58 zHW;+!jV%jCE*nm6vUsC<#TCB!kId-@C6};e+U*NO(?^E!E%c^cGUio_xgo&+Vw`i< znakNcvBpMTaYoUr0N-fKueM`=N0*Vq8vy*Gi+ZAuYOJ*343-tS=<8`~Zb(>90&b-F zhQb*)bDgr8P<>P{_|jH+N=jw6?kz`cvXqWq8m#we1422~l7h-FK5l>EJih3c;Hbma zIW>>~X^jM`ng)C8vPR1{$0W9o%Fq4UMB?*<4b zejk!9m(K_ny?H9!E^)g^8nU_GlB;xLJ#3JA@93R)>j?i=S3M3>PP=P;NBNDTuI&pr zCEp$Y<}w^$<-0~V+a7Z@1h-)NoqRh8*k44JslyFDc7c$sKv+m0h)&TVb0kSq+4A6q zZQ|{#@CviPVVl08D89fx70i%oXk8L!PSQMllXWik8pweKYk)8})M6W|f~lM47y1U! zW>x>_1MKy)ZxZwWdA*+9NbB3XuefqK!P5qp)PE51gw3(zzQOBxtmZHG+v;;FIQb{k$+v|3oz|FdyL-#s_=YU%Aj8`&J->iYTna;Y)QyJ(v<49uy z8O>L6x~$06G#)z~#@rYiAo^nse%|Af9J`lJRybt+|Qk!AJ)JNa37JpxHK%ULv()<(qH|eq}|Gvb3dp9H4da~%k}@KmTgycOP`=#>Jx5* z+<_YJolTiMnU|-Z_zz8YVvck-L4+=Ahs zBMDU4)SEN~i>IT@4c*F{^!Bv3Be{;a21SHtK7+zu#RxOrrZ#4bFefl(`bs=B2X&jWVKwS)_i)t3IN~JpijviF+5e?3eaMzEPh77PjOVUjrrF54cWksKT55aS>&~pbL zm(4Kqg>OOPSJ(t9h#nor!ryYa>yDNeu(w z{qOH`7aJSibRwToMuObg=pw#d4B;3~S@d($QDw=6rVSd?^MPj>6p;~wB<072??@bg z%-xQ@b{&eA{3}r3xc)Y60l`%tYhrI=Z5WmD*hU#f67c}_;M;YMp%uf3%_9CBYLOpZ zR<3e$Qx6?QPosV0?^>)O{K+qCfQH_OyJGi;rB4V~oMv zIMJp_?FqDuZacZqljbLV>hOnm)6cKWGi~8)q`L1+8^jprWmF0y7xD;Qm!O>?NG?bY zy^T=vsy!g`W*oUr=anARK^ZEK`zmrtH*93b%?4pM<1xUIjSVM;FB@)dx^VM_O({lj zj9%ZNSCJBoBHRzEU%Nax3dFFl8$o4+`=$XK)GB*Dlsa=clN$!2<3Q@BlBNH%DjS$= zPGflYW;%V*x|2@O9q|C7b>FiR9f_>zw|(OQae37kI?Nw)pYDT3bOrc^hveZ#8aI}5 z6N>gijw=a`5mNRPr(X1N}iznnGhuVTCctlnavajo@rNf>6?yV=`xskXM~lWO<3}DyvNpr zS682C^k&XCfJ%^-GTC|9{QF{h~04xxYi2U z1=>GoJAhLc=d~wsE6)(@T4EW1yI^AH3BSa|w?cVaiQXuaC^taw%){ zCH09e7dQE8{USeD>n>~9xo4fVoGw)2^6nFtEys@>T`s)f!sQaVyb6}rVDUdv^MU&{ ztUonhpi5-d#V%_k>pJDIZr)@Ztkrr;9<>K^qHpZhVF{R5aH+gYFY^s&o`PWg0$A_4 z(S0+?**A^BF^_4U^_&e~4oQS4*E*Fhk<+p>pUX~&OhV}gTZq>j_CbK{s>04U=6>qV zxU4?qv>mtCh91N81RqGrj^UMy@qc&>JY)Aty+i+a5@}6Nej={Uly0 z1ZK;%wm+l`xq13bx~wOqA{+ANKU1kA?ddq`NxMkz880TWT|s$$DB}5+J|TMSE+Yq1 zSIOQe{$2l`r#hKeulc8N_fTq<(|yY2#T6^1SD8bW%tIVnU#hw3 z$R$UYOD;aL9Q8qZ{2Ww`IYj-Px_4Uv^dZWoO&aM_8GGsrxYqfq_tZJ<+=sB2Uw);^ zV2;Xo@l#SN%Q`Qa$(KTdI4DTp5tsOm!-o?YiF3ST>slAm?Tceb`v^VW62`+P*}~s$ z`PZ@A6^%akA#!QVHx2x9}Y%az^<`D7)A{(rR4Asx}qCX`wKF3dag_-YE z2m9%xiSxL3ovJRN#@X;YU*(5)$=Lf%>d;7CShpC*ww+BGIb4=?#u!K$nS*LBa#`9Z zsw}`=O*~qMo+qWZylJ4GAVi-vXJm}m93`|iEkMcJ!VEYpeul`hLgn!lkT2i(xmIH4 znR0u*avXjq*l!|&?RhWZy12|`?$++Q;AFh3)r?%{aD#oi&>tFRKR_r^YdnWixjQ$5 zs7yQv*9;9KW-G^jXqp*M5l&8ikYV+O>&UY@z~QvJa8M8X0ggd;QhDRnKvJjmpt4)& zbQ(CW^O}{fhO~*0_Ms*DR$DbdzGCqNd!U%6{RxL_UAyg@_Xrowh%>?DpJ{xx-{7>w zOtB}0lhaCmWbeB1p+5SKNy_o5YdpypD9&=?E6-dPK1n2)YgZzn_FjM3WRB+muR+tE zm0}y(13LtpoI4p?BS5Puak|clhk`ZsUc-T+S`fr=o3a@7cgzldbcC z-41RmZUWTn8Dz*{==;BG{5lZ~Lkz-d1?ua%#!db60cPI15TleB=h#u(ddKYpyT%x| zD_?m9_);f_?MeK44fs|Wa+C<3+c3&h5C0-n*f%lgn-IY+atW&tq(?94q+i3rjR5PT z@w>M^2Z6ZYnyQ-l8XvCGEr)bU|G=1)EPi#3J45n%ez!0D(tZj! zviYrHw_(N4(DJyNLg<)ZY!G7%vrz=)=gW<-Z+H zuh$*+jSk7f9KxJ}k&VIaGPD7L0mLy1L8}-G6$W>-E-~PSwqICC|Ir zWC!8pJmnhU+2s{FtTL|V1|4DiqKz!fW#Xl}4VicA>9U9m{XQMIEc1&Sd8kv>4RpBE zpzCntnRPJoG#2fwpV3dOtLSTsiTAj7ziCWd&m*BJ9xA8j`c>Y2^_AYFr*RX6@Z_%T z(DC8p_^PYRc3&=}e7hxE%Q|7g8W&UCKz-yn7+>hTRffJbrWkTbH+l#j7G`^QX$@lR}Drg+nTcr=v)(s0rpcQyc^$ zcM)LHw2OD0ZyoEnLKDU9@H`*e?kanwiMxn)S%a|eyf)ZY=rx%&Gygr~Ck}-CPa|_! z##Z$)>o7VuxU80WwVUAEHlgi6myve!{Hn5YW0%)&@qdf_AL6m)5^gf{%0OiBE4j3` ziH02EcJmX@4nr7bw#dr5Ebo zda+L0UM@Sc+6(nq*1{ne;rI-RPRxeh1-I+u#FyV-c?#vHe8i-LjF*g+o^xqyAnmx? zlYC(9Jf7=W*LqH`If*_)I(i2!Co%=CzEWX1_So;V$SCzhR{PV(Z8vov-nDhgu8Qr9 zyfBO|qL0>Bg9|GkWyTk17m%Otkb^!hIc(obKR4<2Rhg})2`vw@v%ea9Dh{Ob>OKyj z_`#SDr;PoCl#thDRoyt$ao@2HXqRCBcxi55T}pPK{i!=`ei(GNIrna;`4~3(ED(7_ zh-cMB(gq61O`FVff86w)tl2eoWl!;1PhNq1{Md2L6-SnfFT8lU=pz0zO~?V&k)wW< zU%$E!8-sNiXgRunSby{r!t(pSHnGvI+mhMxNe1<8moJ`q^6-k~@;*FgKc;drZZxm7 zeyX0-+ngr9KP8XjhJPF9OnoKJ`5ZM|Kq?FRY)}&PeCWs`Dj=KD+{<4$!3fUr zn)>_J?OJb`jh+F@CQxj@=FfV$%ovHmJxTGHwV#~dprjaw=)lz ze`yH`Q#Zy@&&@*Hn~W*dN8$9@K;#Qw${7bz*bAQTR5r4Uj$)es*jJ!C)`Qybfc`H= zuN9LF&J!H^G`dM9a4q)4?*j;e`6NWf!JioWaT3?_PHctpcl^#cl5hCm8zB32v7oG( z=)Ea2caiL=-T~n2-f{@ciM06y4mv9F4i-u7YPgd)+QAWSsu_xGknbTGaF{0E8x8FR z7&gY4LOZb1yl>#T7$x1;>YzaeRB#Ni>clbNt7BK*xfD-e!c0s|piBa|OhiEse1Qi+ zxs3@|T7?HVTXi;gP>#JTPa;OR{d_9U(G_=nVa;)Bxm_ndxHImVuzWElFOV{-BsrFv zVAUmY!2~KIq4yj{=6LqhT#ug_L!IpA5ATdiFND*^$v1iAI=yIhF`kMpO1uHP z;Vm4_@jyzdp!y82!QqOW>zk-49$IHu!-1`YrfZrp=ft@2roRjlHbdiJN}DJno2B|S zS#OqX_*LH3zzT25SOtl$x424&JELX*x6j;8(C5-@QwGIw8GeE3`OJ9*&~$yX$qV)` z5p9ggjj`8t0G*jjeZnhAm5y(?<#)L1Lw~G3iSZlbE&Y)>j|JWBq4l7RLS#Vney2K{ zGj8Zye%a;D&$`2A%I#Kl81hSiuJ@r}8B53=aJ@j%sSlB@Pmf>LJ$6Gm;KATd|HE}204Z6#u?@NvxTQ1aQ_yXa(F1Tnp|AGtczvz-nmLo@x`PZ74 zYQyT+hg7>Fvd0!A=fLFL(Hb+-obG3h^$>!TP5am)w+MvfS z^~ft}@;@2GTj!;x74?y)(J|09c9;~oZI>7)zb<55-3Ixo*VKF){|36P4Lj{G*zb$q z)NaoT;TCo~gPu#;UPGs)XZ*f-i9IKG&NHyqr_4wASGsj>I^1t8q2vuFw0>rJ*D54U zE-;5TymwHSm2Tuom`jli&zhHD`~=qc37u^>zFDntMA$zU=(s7^A{0*oaTAZnpNco# zgid$7LQPY6Fq}L|s(3!_cerSp4qwN|OJw2*I{@Izr$hc$Cg&rz!ij^b+>YyTe;n_e zl_&m${dIiOCY)rL5y$RC+u?Zl;wR{Y(2XE|;zAcU*f7&{;!4l8XpEKSNPiP<+VTU0 z?N*w{G2^w|70b2*p@UC-wvfrH*bbPfeTBfdrpF&QUd7r|fWNW9b7Y||_SPKQbLy~d zApycX*m9JAuv5O?J0|CvHm_~02i*8({b4>^v_3Ap+D+@4Z}#%U5@E%&W=40VL+R2* z@9_1y=5>tN5|R0_WuPt-t!)ya7?2ody6|5<*UI1j>67@l_KLCFS#veA_5ZZ_$4gpg zkq>_SAcpdoqkTxObknnrx(;qV7o7{xZUJ>LRbcfAHu;)rqE^zVGd5UZ+hazjLrk5( z^x^dPLMP+qQ^8!iu%c~02_5b{tL`HcXiAmK+Sfn8U@b%j09xB9vim_~20Wfh|FK-Aw&@2x1q{U$tpqa7KHt4uAQ5wTnLxh{L5+h~Q(vGU>4K)c$hoP@V} zointr`HU3uE1&y|?c0pa*j3o?W$UVMF&FvAShA~+9zE)Vr(?%?wW#cDUehY{FHd8y5ASF7I9%;Tf1_@_hsa^nRkG)`m#9N1er!aVZO2xK`|4WeJl+l<>2h19 z;M$hDL@b(`xvIjC664-xMIhzZsln&{=yO2K)gTgk87u0KAl$3 zX-;kb6t{)0&kl$T9gKSo)pnjE7;B83n!EZ3lFSc7zv4?j$zDrq1M39siC^=e4{PX4 z*!V4j%S`_mZC2%OaFbSZLB);3WWPdo2dDg=?~1AjB~ z_Sm#%($|1XIBi`x(|~i@z||Wb73_;}EvwM$lVm^^{OH2|Mre?+-aK$@Mu(%kyT|g$Gp6G^KY0zPy-2VX& zsPeZ6aBWNrC!ZK>%CUbA3xN@?iD%H@^Vcz^JsGT26Mgd*e%XT`tk`@8ltMQ3^28wh z$=`;q>q=T>aW*bpjIqjfT$SUmJ}SS0#u13Gz1JT`=5RQa&qx><4BOBiXo4b&qicnM z$-4pT2q@j{g^@Yhd(QyghwyJ`5+Q9#o35`OTCv;REnO_T{cOa6^wSzEM#Wk7u0Ul? z-zmdG3#0$Axr~mnSSrnn5O|o51f4V}_R`LA@3}Syh=sc?m(oG7xZ`)cQ{(31@JWGW z=!9+9+1P2LN0@_!7%R*{!9I}Ubs5<>Ves|F+Xhz|S;P2uagk??V>{@pv0xJ0DWdTl zi|c$7{m53Csy?lQYb%;O^L$J`Na#zm@#m*XsB^ig3w5BMQGz_CQ^6B67={>8!+50*Y8Q;wRW5Q5 zg(Ysr_Kt8x0`j(Aoi6YQwc(br)4Jt$2W@YC(r?sP^2^T{yT~+{h#H?mw95t*R$O8u zyW6f}ngbr#4VD7yxHcR8`WkLvA*)0)t~}*9zhst<^Pc^iEoYs1h7YF3p_Ft=`*09= z(ZxrW^WSyR^3L-wSl;o@cP(#!`}xZ|-*v%qA-~q-m)^%N&4Jl1C%gudh(C*B}s z+_-Js$1wOKdvC}a#|FbsnyA01Uh}U-*`QNT+90|Yf9X1h$T^f^Y*Ov$s@k#9kU-Mc zB>WhdohL4HM*1jz5^$1Zf}J2LPwSloc4+J$?cV)C;J5&>YiLbMe0|jRopTV}H9H-}JtY_{o1=ShOR*i|Y6kP)_3Jj|rOI9=i^cR<6B8-vB2;*uJzQ zJU(l_j<2u7x)=|pL$QKG9Y5%F6}H=PEBb4%vv;&E{y5zf{>CYPf&HO1Y(2|jKM?sa;L3J+F@JxbQ@ll8Fe`Tgb@Q>pkIy z!JoB9>rK|<)4dK!Im@+IUb$SWwTkt`Pc6xw@DD1usU+JZI*g88wgrK4c*B>qzHL7j zJ!CXx$$$+7Y8^sO77zr}*R>!ce&PhF-C*G~fVT8}jUCTn3OB_pEB~L^hf>%}Mazvb z-yBAn&*rVY#cHf%IYRJ2<@Y|9!Ft>3xW7D%a5r!rrnVn-D)j zkz|tMZ}cZ|@q{Q|;&d(Ek)`eI*xJ7NP1xg7I+%4_swYnH9- z9|BgLf+^Fy%~z-~)%9)Jkh3w;&#)Ul99GJe3oV)EbnkOuH1F{2DU}G}oMH~jL!XH) zR&!0+X>9}3zc>tHZaOA=_u`8$vCYft`uIWC@k@^{mz~Jl0=&1U4=Zo?yKb-OO@6}Y z#-bomWTL;NpJ9*37N<{BKKm{B^JLTso{*Yvx~n{FOY@+P@HU_7D=sVLt^UNP9gB8y zjgzjBb9;pcW!dLgG`hlQl%zJTE*s=C!r|4Q>jcvm*wYcpT?yt>e!yiEjZ-_V+{E@} zhP7_(i~gKoAOg1w;2r1~0N)xK zU^?Zm&!8)QzLLI!>-yC4$Sv9E3l0}e@-*jM2OQ|5hePL_a-Rf>auBzt;!hE_lnQO+ zO@G-SgMzOINBR`~qasZcs5D%zO)~skG}m$27Smo`is&#nEcNC$zxndb-~7$Xr+(_E zR?NwPn;+ls4d1Zb`ObH)m@TxDZg;!eEqA}iJt}lvPz{mq;IV-za&P(sce(3bRQBB~ zbnWt=AMuFgdC!0T@^`O%WyPHeAWV1IX;CfAL`TP#f1-2Fa=XEM5+a-lC@x{6Xfa-X z?#WMHzU5oKWy25r)H~c^`G61j0Q201&%D!}mV4dnUMYu&;@R%q?|%2?iy!nLPe@%} z$eM=0vV6oxe8lqPU-Jprx%Ytw5Zuy7bK}VFot`&Z-!F&eS{XoJ<+)^EyXNDq*qia{}B(T-g zj@}vu8*9s-_Y4lXHlsaW{W2KH}TI>=+9) zi=`K6438MY?pw}#ICR>{XigPQMZ=+?eyN^(qrO3sC;!Ni3vr1*2+Co)=GAkLH-2s0 z2{GofF~#kJsTH#sjN#%rtlYt~_(!O28$8uU@|WrdI9VBRG56^c-A0X0EHL{jCuLP( zf;ms}C1d1^tm80es`sW&HAkB-tj(Ud@ZnQ2p(~iYrl4eGpgv>MrGA)S`sPqdoz}G3 zv}8kzPFX9+&zRw608h$X%HhZH97-v{w0^Bk%U?!ui;ofXTi>y~ z=`H6i=e_Nn%RA1`!Bn1>x=b6`%VU_!=tU;?HS|K;)Mkk=@fJ$pz9IVg;Cx2E{!BdC zr2Cae+N2*-d29$Ut~praH6+x7CmQ<(2X))Dp8%hcczKK+>$svH>2agON-uQ61}R|| z(Dg3vg5umRE7(6Gk~RekD;`A~af7nb#;3y@Y@TM3 z%SbEiyiTt0@R!@;Pve%cJi7yH%#VD<=THzGRPl!T0KFncd?z|C?lu8m`D;$xOpOFI zUwR%bJELu;L5CcLmJfDvPs6W(NNp=L5*{GGc`Baun49?g=z_I}HT!CAAn~+;@G`A! ztXp}?mxG?%;0^YCm)f+vu~S&rvc6FGkkx=Nq4kZt8^`UN=stup3HA*h)hJn7Ylcek zJnA25u&&Bx>YK*AW)(Rh#Ktf`>L)tvqj7Ugah6SV6zxlxyTG~35$1w_rf=Lpv~0@(YLP&j(De4`nZfVf{s)f$Wr~IWCIu%^Do}< zTZaPZW5dbzZyQD6zQCH<`Dll+m!gl!l&Nq?DpcJ)+YrS&|96!4;dbK~8w8s$*xvdfn% zwD;;)sHZ&b+fK_o96mcp+>BFv4i?vb1%@titzhPk56WNBGemm=UaLnMw02V71}(#y zR)htcz!r*h@|e%Ytu%thice?`2P(EbU5g2b>AD5G4nub}9*xWgj7iNGV+<+X`c-!K zJ^GvHGS-OftHuu)JQoV-N2A>6X}7%&v|szHyn)sKcH`YAH*wjEu5-+csmDCOxa|t4 zm)C0MW01PJUh_OXjSqN~&U|?1IVU>n{kqF>-Qiz{wdUJTN#%bb{lsG7mHZ&Hek?+{ z?6W+^$J!8CQSI=LHXe~wX#-9rDZvAfbOtb$YCYA{tbk z3fRj;duqS|$UpqUKP=z>{olX*(R2Q2`OV+@&E-??aQNZWQ=ama<#n%n-SW7{J#P8^ z-~avPum0+#}UVimgf7ORs&wbwWmalr)!^{`<_rCYNmp^;SOO~fT?P<%8{n(E!fAjK} zyS<%EndPQ~owod^Z~fNgKKHrL^3s>Sba~+mUs&l}_F+1u(NWva_kPd!ET8`ApT7L? zkNn6=U!Ma|I(5DHSXa?0ey7DLTHywYuUX4iLOFK$6E06X?>U%3xSW*NNZ3*wN`Epq zg-k;=2Xr*S4j=2upCs+a?I*F<0Y$QkLVI?*uKqO~nv=>d7^Umd(&VD7%Bv9<-Wr=e zr1FAY?u28!cKPAp>#huIdVq5F!k&b~NDn3)O1T%@{U%7!x(qfezb5r?(8V|zSpCbK z)nKWOm5i1A$5b|l{VG4gHU>otHlJV_bN-c4%IPwRUyZfoU4Nwml2#ke%4gw?7QO@h zHKhM^o2yOB@E{H~KxYH6>g3IBVahMpWoJ{^2cjvYTpYzkl&)9O;|AIrx#A+%dl_=( ztHCzpG3s17lS*4QX!~Z!d{cxvn2$dVRREy00rt;yE0DvotpW2z79ae{AomRvjAOZ$ z6ZuLWm^Ll-Q3ZU%51vxrkRPC3(0Tqd=*s1UHq8F@B)<^zufw)CYP_+rPQ(|a9{O4@ z^k;ah5Vuo>>Eo`0KJL5XlV6%CM)}dfmEupi9FqB|tSfkj{uSN?AS3e9pM5BGv+2f- z`?JOqSh7FPet)0Q1iwzx0~uZg(;H3W7FZg4%8w9(8bAM zU#iuw3pyGbB()1|)MF99)6Bo5C}z!bq{(SH6B8IQBCy|X(c{PC6e4DCXhI)odgmGF z1TLn*ab7;AP4fP{U}YuE&gufb&gq1Qn>hQN-W0PxPJD{Bf1)Iu!~vnP&jI|AeNEek zrnLv}l#h=B;W44(XEY1@&0{y+vBa-*rnOrs_%}2a?QrOzaK0pk#z0~)G@TZLrmK*z z4THWu@$Jx_w#HSAV5MJU6W8(W4ya(qc1p|JavDq^4jW9o`L;W*!3NWk7EjXPQK9iM zU$Ev!A?{{;J73&nC$t~-#}i9_XT`ThJHEmOQs;03aZ`NHmHPC&X}9K6eaqd4Ni&;9 zu^xVxyrry9{i&buGYc8$@4AXa^vUj}ww^x}q3#1=hXWS8FwMwIbBQJ}k-;R>|pS z$~~8-j;zVd>(rrTciCmj<>Tye8#)Ycf2ULw0RnbAu;#%Hf1uJ^m%AvcX1Q`|f5&W7 zN1E~vcjh@T=}(FtY7JYFBj}AbPG%?nY}B=*qX+(mCxXHED)Qi~jOS;!IGno{KYX-T%~o zd{bNf&Ng@1$a!-_`X;ut-v)8nWqwUHuc-Fx`uuhX-a^0+zA~=K8~FsP)7(Bi#;vQW zcZ+BpxUc0c6!ZtnU^&#+LB@()_Bi9lHT9yr)?ayCS3dMBA-^oLSS_2tW$I#Ckf|dC z=4mXa^Fcqb>Ht(e$J;le1~^y*_;->k-QE>u!e&0ul8<;h!>@};Dx5&jq13m?0))37 z!|KIHK9DwD+m`*f?@=@|2Y7Fjr+NT%NFU*UID7v$_Jif8t+zi;F1@>4U3E#hRW@aK z+zT1+j9a1agLo`w%zIwCY3`e3o(Rr71}g5rcrWyvNd4tbim|NjkD%*V^PK95JCBQT z_#|0+uE{-{ns;i>ihQf?m?yCVypOK2X=GikwKZdzH6e4X?FPoY#z4l3nT4PHx>gv! zOUie~#7^OS97u}&;du~yjP^dxYeBijN;lSrZvz^bR6PYYu@NKAoc$2k$!n;br^euv zU@x0}k@P8?;g>I8gZqBF8w_p6d0f16-tnYVK1pz5QDpf)H8|_6vzEs^<}p5Gy5~Lb zxjg>yk6-S1$2%_H`mNs@A$H&a4}5?REuQ(zXD%YDz za>(^VKlDRhNPh7be{p&6gCFb;a{WPh+>!R_xMFbjq(AV14_t14``h~fijMfuulNcX z12^-*+Y?mB_ZD1iT`~wT1ZzO-;4p>?bukYh8m5s?TR!N6KY01VFZ@Ew{D*(=2TuDx zzx%sGw@ku-C!gZ^u&@5=<%2%xgO-Os{Nc+(zWmEQIT4q(J8k)oU-^}OkMV=P_=}g% z|Gdvz-u13`Ee{sm>1Ui_UhZJ=c%tHP@wKmg?ecAE|6lp#UrzcdquEfb@$dfb@0Mph z>sjtE9IhEUy}>%<%LjkyiH}uP;X)!S%L)@uNio91KgYL!6Miqj?7}nGJunis6nkmA zf}2WiVQ;<(PLg$ny-ZrZeRztnCrf)r)pT*2j&G_`Z1`ay6ki8BJ50 zjDh;3F013Z*2Tt%+7fJosr!|(TLxs_7?K04&Xcib!;n(RTxHtB*D^UZafYNFgcwQ4 z#SIqsMH$YyNpQ7|OtMjK%Mn>_Vtr9PewoTEDKV_`i=``kqk;ONr^p<%ObBWRh}ra6dlPGzk^)0ICo$GD#$z<| zYd8#P^lKTBOS%DU)Ju08V_xW+eI+yW#M?+$UEzsQ&im#$B=QD_HZ*EJv2LQH;I^LD ziImfA6#uFt$|oP2WXr{z=!}c&#One1L;6ckut~31iU?@~vfoS_A`XvzfZToBmo>#r_o~&tTq~()3FYbY0FKqaf_e71a?r@g7Q0V`V{Ft81x%HiJ!4AcJP_0 zJzNS8LUL5TE~DJeO33jA(S?&tRAi=aBqGilz~>nJ<5L37b$ZbPatr^xBmx`{Wl+<*?sS}=`cx~S?x%e;BJjJUGyj$dro!8k6tj0}EVUJR@5<8|b*% z9dz>Sh8sG5pf0#`tjkSg$s_bPX)U8&A))db2RI>4U$+o~HE*}&Ixl#C+hYZPrDc3X zg`;25o_&y_zGal(a~W-`e(64n{;T*dH){s%R%MU!b6=o;^wIYZd@qFY%pA~%%pT8K z8^`*%7dyRCamdh!Y=Z75mYF$?apZY9^>LltUsv7P7C^o_JmYCh|6oyZW8P7j(03Wu zpLq6|1sQwBtMas9KMmz_nOn(M@*zMzFf>!k)Zx@)4ac(!MNny`uc$Q)p9xvBCO7_S z-`PGi?}^#!37`tAV|X^OLmg<+gXB9<aqWkq(mEoCQXRGj7jH)@oil%iMx zcK5i)J(jn;!#83XD(*Ah)nJ4|s^6&5V@0~F7eZT`BxcuwC{_FDm=RHsPAGh3J zJQ;wGe)OZ4|NetNxZFzoIPiJMLm#^Q)KCBP^5mcYdFT6>kNw!?=cNOlIC}d3{GZE% z9`qo`1K;_mM=ei%+EbT1-tjY+r#>)Ra5P@C=pl!5{p=enlWpU)}BVK5u#CcYKF+!(kQrpbnq- ziJzFfLh^o*+Un*_E#886aKhP!ZB z{2b{zrcJY82sK7FHw6oE48wKi_yy)lKXgbyJ3FMjd_?a9soJ3Q4eQ=$VSJ9o+~vzP zqlSb=@r7JEZU)P~7Y>V`ph!}eou3tS_HuW*4O%Bq^AR{eoFSVr0osrUH8;liim}p| zjaCe=VEJX-WFuA^MCdu#vSAg(xZAk3$?9|)V_gk2aVX=;Lc5mus}l0q0jgi% zcwWRXsb7g=eA2%F$SHpqmQ^3$zyVthmtS(Kkd#9|Tg#nwNOa<$bFeNeSV9Sy8$f-t zApMbzkY5QQDs+-f?kWpypizOEw@fqS=U` zGx`K_peOhuv&X-XKBA7}r=^Oc=edgtaH2T^{#i1_*`DH-RUN6<&8HHFlXz5{>8DZeltW3)Q zi5kq;8eX+84{f*U(?rW-f5~Z?9oOSy#qbDjA@u-Kazuy(YZ~5!57A!}5i>%`2vJZAWYhpp%bQ0qU_5zK;FVNe^Lhx zxg2j_J_?7_mqL?P4lbdIOS*IV)3EwTPD--ODL|!EbV3Q6bf!Hq>OO-Cl*h%X$UDcV`&-GxD7 zn&dkOyK%{tx;6&FXO0mRU9Jc5a7Y=AVO-%#pTjc-h|j0Kj%$BVpXMR#n-9Z|90+mz zKshUH<_`r%`YvICqDg^IQ;7Q-C`frbkqs(l%%ix>qcvxq5?bgI+(oQqDY@4F*l&Gv zd>l$Co!7-B2z&-yZKzp9IAHfsWz!vkj0ChThf(~H$My#atYu>jQV>~@|C+qM*K5A&0h`c(Wfygc`PQW&>Hb#y zCfO1V@eIjPSLcgspg&l);I3W+A>PE1FP`CUa^WrB=5O@faqFSW^4f-NVtXhz6!8;h zdI@S<*oRVSN3&O2?4$gsCr?<#PO@#oT4@u{8b9-sQqAAv2%R)eV+w)VHLjXfZK^&~)z%s+U-<%17omx8(MzRRKY>u@ zV**_JoxHjLJkT+1A)WUR?{YJ{_S}qB`WNZg&`QIW^;4Wjk1af~zVe*a^E$B30lvp`1qV_&R2l1n_k)@P zYd(n11yh;XpRMB%e}1f+y}5s=+j}fy#{BjPq~Lsbwih?))K~Q}A`4k|R!~OmHgl0USWkP;Y$W8!Kk< zAqW30^Z_3@{eMvj@Snf^+m}1t@lMN|-t;C%v&g>x`@etktHAr-_r5+T;qc?JkA3X& zo4@&+J`mc4mhVn?x|5#-;>jvH5U+dtiJ$lhA3E`L4TFZKw*K1>{D5`JL7Sa^L#axAq|uhj2g-u!cK`2k*^&*-KyQHU)p} zSATW6Zp9bT}CUi&bd0irhK%e=UpJ|#mzxmCk``C~BI3L7uxP;!` z@Xv2Zf1(^yF8%}cOm&|!fs|L{( zaz5lC5AkaRX*YPk?sc!V4F0dOsVu`BC^cxxus@SWCXkKG2U`2>)CDMpgReXzDt=sv zF(s8}gx^ajUd?SU&k-5oO>hUgX734{a^Af{o&t2fRO6Jva&=qs!QbsUJ|-$$oxPy4 zNAZ&!bPM=q!=lB>8xf`;SR5mX$l5K>7lRPcLT)a~{&a{p;kZ-{3-v_(NE>n@$ zk!_>u>(Kkms!8epJ-_6aV=_+U-y9f~QI&%l88P_fuEtGVZ>%c)#y7!uQ+^pcHYVZU zZu#5DJa8aofAYrIj6AEHliYk-tufr>w$apv6Zw$Gb*l|!e)T)oC-Kmv zE~$?)j&i$r+?LVtbNUWMHa2Tu`e6*pzER*xn3r{A88LDr18wPi%9E#nuC(z;|8?F8 z*7Bp1=G|0V>JD>VQx2~~#>}`$;c=6jB4F}`5B`~tC8#30EeFAtaD~|q%=DdIyx@&~ zTd(|Ldi**LIgw9*lXmDE6=3{M6s&DZ@#3p^=aKR2R|)N)b!_T};^XV&!q;uf7z<|1 zr95(RKmt$8MY)uzI;^h`W#_-aXs##BEYsg~wS&7Ndi7zH5lV87+y_7;T=R{Su+VuE z5x+h1jp*_;8j8|MT%gC-L{6O`VR$x}+J%6LS+4*-(&l*bB+Ai=Y)V7B1_efZWy+`H zCjvoI64mwTxJ1SaB&h31)~9JI4%h@I8SFX%H<%DfQy6zj%LS)soYq)CbR5H!GSapX z>Xa*A(KK#gB42gI|>p$I-jYg@{5f3GVyc6q)jb4_jvbL02Y%P!07`}nVvE3VW! zqjgU_S!;SNBOJY{jON{Tp6wBAqq6n9u54v=PV=CAxuE%lXCQHo@93(#bzyzP?sA;t zDX-Tv_Lt%BUyF02?5eV-+LjWu9tV#Sv!!hyzT-qSuGbr?PUzjbw_TANhUnPwr0;M) zrLtV5NpIpML(KfMhe_Ri!%&zfKkCM!+Il^Zf>HG0zEk=!cgrkZWkYdux9!u|B(Z~9 z2D91=rN6FN>3+0BwX6G&%J=@lH+y+?A5Teb52JF(6C0iWw~<*w*j6$|*T~y_I(lcl zpL9n)U-v_c?Rp?*$xXR^gSy9=?JV{V(+z4ru`YRiAZ59OeM8tb8vPiTK3(n5HF3t=%W-Ke&!3 zoSWYKmmUAz#B2L_W#7?b$NgVY97gdp=LsK3)vNoI*KH@XI_N{KGy6+FM(UjQB8|Cf zed`DR-&K7`A8HV}g*-8eJvjC8KzIg*CtwDALrW@r<0huVhT>`Zcm?fQ>6~X|+6SFy zFgYA1KB@5pr0pP%A&N|R$4^k6j1Va4X720P_vK(c^H}0m0G{*(SvW1Dep%$TO%WK3sVdp;7b2h24uY{qk; z$Ml4X>pG<0B8%Ke*Sk z%bj%tcg5DBkUWxO%mrY5BD_T^ByI{jv4T-V;BX;Uqw|!_cq^90GGeb3c>Ep?uvhhU-p}zG5#>i5@TY0dgocb>xE} z32B7*Hn1Ch!3$nsmnWtgKKYYBS;oNWK6Hr?i6^h~PeuOp&-}CxkACG>e#PMr`H&Cs z0OfGz<38@={N&A3pZZiEG#v&!RrSSR{Kd=tAMgOv-sU#9S^oO3|9bg?d*6Gx_dV~q zyy{i2@+$>_FZq%$aTz?B^c4^NisccHc*OGdx4+#7Oa=)E4NCj1-})^_{m2jh@ba>k zz03pZ2Y%oOmLL4@|K0Hb2RghmkJr_GPIxmiyl4K7Ms!Pja7q=g;<2Rlo5Yzu_l^I9%f(iUT+PbBi+Wf4}=LfBL6?x_r*( ze2#go11xyE+ud$?>s#OIHnrO2_N3PB?6^v&otP*uy70oZjV|Z{c|PpJKb#mJh{5|g z&v}mJKsI!6@kJNIE=^hd9{I>e`cRAm$FKjouUr1=FaMHMPfqCh0r$J#a_@WH%Li6> zzx&-S(nBBmP|F1U*F5ZDZf7RHPyWGk_5&Rl5lHP`7Ps}I&#J4Q zcq$#*@+aWz#ZuSePszv}84{nQT>Tw!?A zE)Rvh0`9`$Oy(v_+m3+UQxV-l&|f+6<<&o6td^1yVm(urRc) z3%VALn-;S{Xbv+ZZ!t8_u<*<+YG zKv%SOXB;0hEzadF=&;DQpzvAL$FF~l0C5MHF*=- zmm%r(4U|8ayiOaJl6``TtvcyK{rs7rGOgE>;SAd_q4_^;_Q9s~;oxapAv_c}qC4*# z7jV^Z6D$}d8ED%XHfe-fH_fa8|BQfX4kqLZY~#0ZvNU2XO-7Zg`Xc5wG$NCFi%A_)_3m=m_%KxCUysHf2{oj)?&RMr@)R)0F<@J3q|efQ=uYz)Or+psbhefmCDUow`9l@gr>lqIRRV zSrwuVt{e5m?$nxn)m3#^hW!!rA)x5OZ@wM^n?%}%q9})(k?1$JU*ALoc}*if!UcK4 zB(JVgdy5WDyKnIWva(Ik35=23y!auHWfpJj4p8-qEr|@N?-s>J63)oQeq7_+2CexvPR<)?!SXz$HFPBPj~B3@{D{n^59cpq4H zusq0}Jq-3$##-#R*y!vXdEmMO#RcZ z6TRf9@hGnql|9SbEX$?sxn*8G4d#GT))M?2SC$(UXOb4xwRx>i;J@W81It zKVQ|i*z>tvrZzyGzL=o^&j_qgbVlb(USINc{YM=LmkW>%?ObMDCl{>w2PS#~W!Qv) zikOIZvmAsJZaD}!STgaccp``+w*3yC3ifs~g6uucCfuzF`qY+>S7sQBFsh2dY(_BYbj@y~7_MtP2cuF{NE zoV;@Kd5@qO?E|7`+y{rn?}J@_GXneKCfE~YU2K4^SKj!$Ps%^~Q}#dx$wdZ zm+$zF@9^v9ZgGoSET8vzpSL{isZTBHWx3bA?zO!9ofr+@mVJ@If@)PP=o{nvlp{(t?KfAP~nDMw{X?$^BLHJ%Lkzby`K zcp{5Q41MtGKMtxmRJ+3+K1~B^9pL2UaPMtzds~`bI_w)iOqerxQday7gHHjI2*LmO zAOE9uL>v6YU;Kq>IJ849F)^*Ll;zMtV1oLCXQ_Qf!=c$f{nJ0We&j&U$j6gZPyCS| zQMNoaC$4p1%yy2vQ&#Dd+goEDi=?nV>Gg&Zl?BeLRlo*kMjAMK3d;$T}N_;2m(gh>zY`!=MB-C-jojwG1bIDnSzW|Pz zn|9ohDi3uK_S_hcT>ORW+*du|0M+$Y?L$(97pypj6jm`IT(|ZP1&E^lcwX)f-~2K6%w?@I>9M zGi{q78{)lDX|w*&NH95E^!*pmA(0|HmFY5jogaw@nb#l=fe+UttIf6{x4aw6T1|VERoi3c!|g6^Pc(?+Nsm*TJ zUFBz)$}FelG{RhzkGw{eWrsKQCmT>D0tAfl;512~lXIEwl0~o&H8kI@x|nKSMw=?G z^AKl$G6gS$qGK1FdLNwQWZfgkco8o2qjS+q>zwD^%v*%5ucTWC)#hT)^B(;q9Al9C zr_kfM^rjD{a;Pbt@tibW^lJPdw`epy+Sa;68?(1U`dM8yh&}IVR(qJwJipVPo|mhA zj0x+KiqQtktJad^KuZ3|4*8}o*MZD5F1dd|(}zzQkBm3&9piB7=~_EeU(c(^aZw%o zGmK|qr=3lo&Joh0@~L3V9b}U5>W3fCEy&Di_!HOnDx+zhQz0d80yG82XaKyjs7hf;Mh5(6Y#7q~7lD$6VW$SeF` zQak~}Ap=hjJ@UvS?$Gn#Q(yV1etPCx|HZd%|J`5s?<#ilff0(#p;7|}B0Mo=xCToo z+6rrS{6Fyh-@kp*Cw`}hC;-`l7=4_fK%FI<7cEhYu%@|r!6$>Jw|;wQFCmoFU#P7abd zn4(X6!pa1&r~J`By8Ye1 z`*(ea_p%TF@Dxy9h_Cd4uHy(q@ibNQ`47MDA8eob8K1HJ^iTiv_VSm%e0$ZazIpqa zul-t0*mn#XOz!Si3ZJdW=)4sLAQ68opDF8UX!b-D07R==*HAp*cl*lE7l?BV zzZGBt*PueOF`m=8t5v(k-2mKRxtSfG@VVAIU?Fhu8qf6QL^9w6S~^ooky@j!t1b-> zw4yWK)Mp{18!d(lH+>&W*$~Lh*Ecxjb-nGb08fn$hVM?WAD5^V!rlQ~+HMF2&y-!D zNzl?WS?@$1zOPw`>T!h3w{o&H5OgEQVQ}u<0?|*T$)N zr9CR+`idt*s#iRy^qC@9{N;5|!=uoLA^g#<-)SDh6NFy?nMB6$qxuzw5N)*HK9$Xm z{Gutwsuzq(cTH7~=S*Z$;iTEwyTlVXfLo=u^#+C;t2h@`!7M~0bGx3Sm+{kJ8KkV` zWLgQ~Z`l$&0|&ND!DT2aSAAm|xB|wp|NqGbb`0yeKtWGJvgZOXVweMz4&PAeHhWx@ zj#|H^6ZA#uuX@Q>dd-;e7$^)++4#XF3@tAp&@G|$%=)i<%5EOfVfvZAm1|uVfBctK zFX;J&>En;_N6>vJl?4~|4*DFZ9mbY<5t^mOl4E}7Cd3yA3H`pyvcb>>}Y_H{(fv^d zn%>SQcjc8JPPHj%O#^I5+$`GZ!Q{UdbpC=utf7pvJ*3RS=mNy8xPXH* zE;%YJC%#Tk>Ig4hJS~k~w7_hY{~YBIoGyC?&XiB?W7!!yo$G@e%50c*omcjPv=Kei z_c_lgr+8AXH}84SH1@q0h$`*lR`Gk@THJG?(kGSkVV(cDcz)!={McdpCwkuQxt00a z_mkXbfZVsZKRLLRVIwW~9^a!VtGc+~aY*s_qr4hR$SbmZFvX$Ne6pzLj=qnZe(~pf z6E+C;(QUWso`Q`Bbw1gQX#TjVQr-sy+)EpM9}`{M1w!t*b#IHkO^;C__e-_G_pR7H zwtv_(R{h=}G{!cAQJ$2def+;B_esxz;-~LXY<332akF|BP9(rmdC#U-*nOc3^}21? zLAp;P1Mb^w>NS>qhHV7EPkjgMYw3_&>Zv+nv&6oICYb-v%*J9LNbv+UE>BDOKY7#_ z$Ol>2PC+in%A`eCd}ot#Zbll*?wD)u457zPG8wlT7$F+?QuhwPBdFXdCA9xLo6*^|`MsF-D^FPO?`GNO^|NO|UT;X`V z7dBq;Pg!Ufi)J?FsUz`oU6y)vF*iG~{Hs-D$;FV^T-TDsbeSqyaTOjO+>)P=!=_#* zpLuFxtChacJA@VYl{uYcW}wl}`vf$f1eJ!qTuu`7>R zXGdMkd&QLJN6TJVGIX3}H(QEKty?vR_+X(9ruwkc2M)Ey>b_~+bbs?P$tl->criPb zu1oqc==la$*kJzow!A{XK(>^@U#=6lxX`X#{Ei;russginb?u#0;WA-@=E360bfN$ z7WPbc@h2>Y;E1chhzzv1iV3I8+-K^8{2(^trPoMaS1ocy1|H{qFX{QpI?6!``o|m) zJ&hz%U)Sle?C~Z)^_E`sd}Lhaq{bXPXPjGima(0&TWuP3qBou4iziAJ9 z1DfBthse#m-0OVWnt6_Lo-ay9<4Gydap8yhsR%J-jOO0a`v*Rd;=WP$iawko4IU!T zx>v&|EDxCm39cAOnsMmr4eli8u|% zEM4qW(KYQ<`gOvQ#2Q|4580173u=G)&H{sP_<|i}eOVxp?b*SneDWvfP>Mv@IFFT6 zY@Y`_D*FYW{{`E}{+^HB{^DQ!i`)CY@B8|7cXa3};OQt{3&&GNf9f;-l$E>To4)az zHeMCL!AHY8zwID{+GNfB*OQ;UZ6Lz1zFJyC+Xx$JaDDXrrzdzvx9FZah=P8Sn%uo3?e#=W=>O(J{5c_=}_i>pFg{1SeRBk9WL4(jW^@55! z3&oxJmrA9 z!3R^qlBZv!sIJqJ}Wie#*H*;+u%r>}^s&2q4eG*ur)7fAEF z{SL)j;M^{UXXelRx}RfPzNtp@nd%-3O0bR3GCJp>P}25fj=7p}7p61Ud11JZD&cY%&*S3JQNJPGqe z)6EFpbs!sK8TQB&p6DCv9qmE)YHk_7*el(n-R6TipmoXZQtZN;+vj?yBlV00Jo&4= zu2&c8Wzm3MW&C6;^};6g*5(s5sGo`y&%DKkA`1t-6hkp%^P4S%EGs1KxlHO^Xp z!Rlx`hKf%)WP^+4e2^X-Q6N>JAQUnco7uYuXB@ z;`ebgWZ$m+OU}jHa3bmgSXx%z)8+RPSlhWmqSw6mXLv0()*Pp39phBtIaOW^F0|+2 zEMy)x$g}f1oXF^Dr2!&&hU$`zcg?%;@%TsNf5nYHBg(zs^66B!(J7@wXa!3qQAKRc3Vc%*e%9HUb3I3jVCpkU(a5@_!d zb@V4VJaXc%U|{sCF4lkaFFH8p%oXbcG`kRR8gnnm{RsO*w#f7RACtL1c|*MH7F_Nh zp65jY+W>pO|34`74>WXd=z|6@_w2ea^HH6uIQ9De#1D0HKmhXEJovI~xhIu8VS2~_ z8lmQNZxq#itnGwsASs)8#g>5gu~96)?}v)-A5_pH-@`!K4S&Rg+`~cgyIkbN{nUQT zMET5D%X#I!A!&i6CAspgSv#GNyl$V{82cb@Y%On!ri5|yEuv0PkQDRPAN)ef`#*Tp zZyY?)mrp$QxLf^MnZ-{EsNiE7AN4w5zfQl%IZX`^K`9Jn&Pe`BtCMFWJyd zJ?q9N?TH;Ne!9;GtbPG)s&>)F)o$p+7V4*8+~?9;o?5wdM_#K2`p}^Cxep5{4^irF zTiJiKFV|}~^W{oi-`H5jhA&zi4uzJ``HEqK!o$}tzU0H7K1*9-2SpF(nkelu%I}Tp z(&4c=j2?ov!wQj&=}r1}3ByA%WL==Y%7!1C?bQX&*1Xz#=$iwaOf7QIXARQ&=J`Xm zC5PM`T4LkQgDGM0W4?rAJ)?a5Z`2bi%f2B0Th7z{ekw|O%YR2bCc72-)J30Co_6t( zAS|Qt^o-;=#xm%Z0tvg{HST?g&U|9L-8w*dmqUl7)95%G?~D=9>nG+%wVO1L-;%L; znbnw1_gnXw6B>{s<&1=@yntNvkZBkbJ4#o5Yx(IfY<^aN1-qdI-F7d6wbDBOc?lhE_) zY+KK@Zbb81eC~A{KDk$d-1EX$)p8@EgKb>uUx=6dOl`?Ejm{}1|&zwtM2AMzm|Vto5|c>C?=fBxsUpZv+6 z^uf_*eCB6tpZ=#lefx-y_z3?m)DD`>>twZor=ULbGe2|tqA&WQ?VaBFo&CfPe8SH= zsDA#hi6?`&8HV(UO3sPGz+=MUK<&d`_Ok7(|JL8~lSjOMZ-qnAbKw_$;TQbG(no#N zM{R%Z&;7aW#qaXs?Js@lU)sLy+rMpl;XA%CH*xXaMmvXTZ~qSOu)X(tzjqD3z9H@b zo$hgTwUqkg-$fBWqPFL=TBhBv&ylXx3m%|EE?0>^LwDVR@P?Bgrh zwCj&$7SzS4>jVfFrjGVBKlUyDgu`ctpCh>7q;Mm&W{K(CC`1lBX6qpuB#z5ei#hf*nAFk+nSTr<*v0D97655r*m32Wb+F%WDp#Q3&O zrDVAUw56i#pm!thd9b9rye|m|7w>?Db%8=_dq%J$_Aug;$zrnv%cx#)ERJl^U<%S0Ih4k;rhZ$AF`C0}MLS&Srx)e}mHmtK`Ze@;* zfz?Kq6P1;=!2^8yDJcw68GQa#l#uY;Ac@iE1ucf3%2F5nt`IJWck&X~Fn#9JJZ6#f zbQz9MJ(bM?FZ>io`{C0@vKH(ZrO1);Ui>LvjLiI67W6Nn3waE`Hyk)HvK(3#hIa9p ztI4#`ePUxb{pJ2d7UF?MLa*t2?6{t}Kmf9Et+7_|v;Kl5=rpIVdCE^xeBn)KT@^pA zXYlGnJM&f>E?#WX{%mG@L4^LfKWF=*pViOqMDbdDdp;0)b4DNf8X(6Og3S8O|$)&gD}& zRLU^s#W7dz^(1^vsv|%5BJN3SJl)I>Qu3PQj?YFF8}HdL<6a15Yzpjx$DiOn@uWA0 z{R5Rc;K+Tk?A+S;alPWt_bA_&>eW}ZLB~A``fWcn9);0OAzJfRS zp+47>@p9{}(kJxxw&;Zqb7SKD%pUiU}`<x6VEYtrtM?u$Um;Hj{$GaYc;K576mw2V~;YY`5j^28m7QaRYR zjv#yK2Xbfb$QmmyHg_LNvEgkyRW#5^4o@F^(}UZa-uS@w-~$hBS9tyOV?62Dub5PS z(k63LvOr$Q&~l9&BbNnKK)IC`o7(!roFX>|3+j87r5rL`$J6?Bh!3aQelHorj~eP~ zB3squIkNhTc?EQuvssRAgxi8l9T^~F$d9dw^BEl-EFhZqAn8VyQv-xaUs z+hdyh#^k>#Fn2P?PRW9+2xrK#tCJHn*siPtaL~ zi#KqHJK}^HyQ6(AIAom^CcQJ6t^=$Ng^Tw9onfEi@l*aYu&|RusYA{Lj|^X515jiR znqK*au0rJ@C36E1)J;7UBQ+KYqpb zJ>T;^mi?E1`ImdL;#G_QG`{MqzRFLjefd{>#rAby_jNvC;{UB)``Xu59Yz08#|mvQ zP>#c`fA-J*+4fbgfiHeLd=@`nAkH!T9Dsr@JUkcBG6$zK zoL}hLu#-msa*aE1L}F8ltMDv|>xZgeJyhiE3k(LrO>`wU*dthbQ;R332*XD(X zYXMn}i%g)k4}-|Pjkp+f7-KB3Y#`a+hLTWj`(wN%J%9&97^5nUq!LNLdNh7P>#baj7fxs&~iM6so|CyAALkS+yR3=|2GD(F+4F5Q=Z%?w5x2M~$_01MC!Z(#Ufr zt0MGm(V{!yjfD+-^^;QkTD+c?dV2E5{T6UPstw+tsJ89eC)e$#I{3#(#$XqkM)3t7 zq|xSnr6RBH$>AI_j|`j_UZp!82cn+^QIDUDEAs^Z%HNCoEDj)mZf{;hJaILzK=CV? z_|Fi29sR^r_cwBeb+mHb=vWtJ#JPRR=)C69SE3aoxmY$XKWRl?@$X#wt86@j-Zi1* zdE>Aaoc{LV?nn=Y;IF&_*geJUInf1cO4=GaiMQ82!SGDjr61wk1i`LBr?OUtL%tIc zhq9z6k9ix3Gu|-bwF>M2>wsR@h}eRq`~Xx?IoVHQ<~qXL&4oI~g`!1eGV>2^FYR;Vw1c1YKO`DX3CP zceZ%#VeW0L6n_k)$9EP+{y6c)GyI3^kQJ(C{X615o(Q`DbRi=gc|6bRUV;6gdqg(X zIgrYID+e)ctK7QX^1RxZ5*hCE*b@A3fv0BJ9DdSv0QbUqKlDbQ?ia4xEb0E>`(NFg zWDDg-Ivf&kf8yRM2~vieQCJR8aie0D!+F}4c750fih~Aiq;0*~MqPa%WgC-wI<^)c zW{n!?*hV(aAf7_AmO@>u<1-&)3O{`_73SL zEGQnjeb4%W9+#}mn|QLG(9WT|sUxq@OPqCEc~<#1&%-s=Rgw9S zp0>38)AC%`LwJAA4?euT;SF!v zUiaG9Z*P>#LF!|VK5Ct2t-{7QYcw-L+mQ)!mW*SQYwmO3<=}k<@($f%9;rF24;3g+ z9?Ma@cl)k#U@u#m|E2T#Nj&$w7`+l!9n|S=fv@JR`l`{qyFhd@<%w9R6pggePF)Hp zb1pW`)PzsRDUT?WK(hxjZv4ec#xYl7-KZ!mHEi}GVVp_Pq62r9@nM?ZO|2U z-_SV@?t!)pAW!DgIeutc#*Oc*-LJ)~?{BVOx-DKjZ*pLU-}^896llp!#hX53s(*06 zlaBu1!8(vicsXeHy}8D9wdaTpcqb~Y8;-x?%47cwcDRraV_tsm5r7_dkpsNgb^hw# z1zl}wJ_%Fu=jB*E$2&G0*%4xWSqVZ#YNIGlO?>tF9qyAj~2 zCJZT_+zFgl{({Dz19x1yWBaYY^|x-{|F6H_hbF*3__}}KU)pe}1h0Fl68tXW{5oIo z!WZ};<)?q@rz_?nxIUTObI-kgs_X~;_aCU5b@jLS(&hR0-@pBfZ~ayu=zZ3o{?k^9 z6|0mX-CL?F_kNXq@w>c>|7R*mO6&eop?7@8ck+K*@dQ)isQKFP|Ng44+Ftgumu>GO z-q8aLg4ezFHA>m@cK`kNZSV1(@45Z(5B?CONp<tI*3cPxj+lKi^UAj;bKTH=VK%Cd@DF1Jcf=>HL~c&3Lv1W7o;W<+Pp27kuHk_Y=F^V?#hs(NmaQL5J)3RxijT{!^$Pc&M z;x@>QT!W-z0QbQqva{qYz{t&C{KGB3xlbI>Iy4reLgeE=dahq=@hf8g%2NnD`6Tic zUs_C|j=hNGAnoou@7(UZtTqboz2|P>UE5uE-RbZhx8LDIEf)PO7;WGewo&TM3EVRL z`PCr$gp4s>lb-{$Y+SgU573^mUZSJOlJ@zJz7>y*yy3uWSvXXqO~_qnx5hkW>2qG? z(wh_642v9m5aT}1K@6_~d6{VQB#ifdWat77^W z=B;;L9Jw7g`!zTZJ#xhxJ&#;@Ol_5Jo9KtP^i~cN9o{i34iVQhCy;b9Zv5f!Td+*-vxjX%Q`)#*dPoAfE?$!MS6~Cy$^ilmoM^&f$S$RDskR$ppzG#0o9j5MQ0g-tEUNcAR=MK+(X(kovGTRN< zJ1_GZkDTF|P6R#tl_Xcb3Uu5qZ>8J6D>ttzRMCPZAn&e>!|mUtlxKtn>|&NYYa0SR zU+f~6JWI-wWzvyviQi=^*rx{u)t#fOJryY!z8NK0(s%KdG9nL&4UWc+uk>BE*ZoEF;A4Yp@_}s@-6xLUb#ZYblTAcrgz2x$LLYJ3fOma;~k!Om+QRv9^g%J zxxE3_7#mV^F0S%0|Bm~C^2WY`;g{* z?1f9ZXW~MOLy3OH8}~+RvBvyhg*lyjdgf>jil`?Wy1~13Pr2)^%iCR-FYA8uJU=nv zA3q3@HSMB|+t}@uoBKfKfB#sc?;olc@M{7E{I~Lb&%amquzR#Vy=%Kq?!9uUm;0OT z4Bvm+4#B2;%J(#0RmcC&@T3sD8?~fo&&ww1~z zgjd?feggI+Pm1`Uh9^d<-XT9Wu(64h&-Yd2iK})pW1$T^s~kIFu`kqaqhv%sV~c=1 z0nZPjVvFPe1wFy`M^v_>$|m#EE8@|%kmSIBg*~eNctrLJ{1cCSka;#_W0zp3^bf{> z@kA7Dq<_!{wF6oA=C3z;C0F<7x@I@DD@b>igC1b(mp)+Iq4&2$_mPS1JK1B{b$)H1 z^!$?S8~V)a7UU*geE?ya)Qjzl{PQ|E4p@5~0MGE{4|=J(+ZLt1%aU)~mDG2~9hV&D zK(}=tcdjYu%f;@OYkQ2bQk%Ba6Wix4^@Fw`b~xkKSnc6Z%&zBvtap&foGZes*8%pY z)vChaab%k-NId)+l@`04ew2;Of5*w*!7Lzb=I zn;>7jVpz(WjA59_zRBMZcI2SuepO{SKn{Dw%#$|Ivr7>$EpN0F3OgcM&di zG)L|U)uFHJpIGsBvW}(*j!UmDWpgo*e+dpCt7Xx&x;%Q;ey7V_3Y#3f5Ql4rNWK++ zo~sL=`c~M9?Mb`{WXCKAODSh^r4ePT$!zY8+Ht%PW{N z)Wya1G;Qjyu2~zulFd#U&dw1_Lyf*^!mtp%p6o}Blz?fh4b(<_=7FE zO#{~%2B;3|p#i1aLr(IqHbl>Fjca|LFKVvHJRTkA@Q%46dlx~MGe0R0gr)NPx`&y( zPhHF5>{EfWG0B(A<^Vp7Ykcz6x>R4qbKKS*Ws2qGdNo z9%(m-R>*NZFVlJP2UxPTH@u!=c!8zd7ku#-Za2Tjd%o8fKk~qX6&1h`It5pBb2%4| zMb6=a8{BgNZaf&o6E14#HH2^g2H4O2+|Su<_{MMeMnB-QEY@ z^rl3;jR0fn)vx|V`6gpKHpDZ=Fo$8E@Cl!=z2=vG$xoQA(0Z`wgMo>XUz)uBwXfYN z)Ou6m*C_n2(}0S<7JTf-eym^D_t*dWU$5EUF8=z#WlC-h=h7j;ZOvy73ZGl>I-uxQV7Uqg z>0^2&;84DT14g@!qone=&H_9YaIsF1+ul8u3YC*Q$=vx*0{KVLc=~2rci!zoU;5Ib zp_!a`F(jKranrEE@W42-(V}n`Q!zgBN)#CxHoV{L^lUoIV8M{Ffm7v+o{?JR0)$4k;)AIL%Ol>h>+zMhqFEku)H-@>( zhz>vv&QoP#oM)rxmRn=+r@w>i;uDueyMJkFRA2SNLH$EL`E?wN$J#8=!jcVyXP$uv z@eP_2c#%(G>g-J++`N(j%;v{jh)h56w3mg()jo_-7y}#{j7dcLeNgeq8I$+hxP)| z1Ntv+_a*-34?-?gB%zt0+?i!~lL?S#`1jErn%*^hHm&265^!jZ5R71XaMm{O{N)Cg zki5rSvsihYXZn}W$g;wY^U6Q3f)izceTxOhvP<$RkJW!ZoaT97w(=};I|WDioXM_; zizx^e8ulK;_=+%Y&>_2(vWx53YeHqaF!xaK6%v^%rB7zMHmKN2={$JuHOx|DSRbJp zKga#`!9M$;Yz4>6cn;}fl@8XWlg4~`$m@PK@9n9EG#r9w;h5o2!GIY*=6CM9%SKk> z${xe@y}A5$m9N5VUg!RlLkjCNA2&#KR70`n#;b;W@R9>5!u)7~wK4Z9o&fLxi^ASO z6Z!`wpzfuf(?t*De&8R0)P^%Zg5Xf+_S& zOxnpkPxshMm+$al6hEi{IV8#tNZ8=kJ#n!$Z0njHwn%RVvZ=A#gcDjVb9&zY<&~LG*F+g9-UF|Al?jK!oP{Kh`^*8-O|HST_ zHVyS+zQFU-EoV{O^LWkXx`$iR3j+((w8N5AtF6#Gc|Ayb~be_XnTe;!KJ+OiH9y=hyq%+$EpB&Ku7NwO5e*1dY1%{V)9) zw9d5tqc`p!>L@}qQP^uw`VDvJf9Xrori>z=t~Z=`U4!ljIh<{M&6*E=$$ggjV11&l z=MB5XqsM9S*nJbe#8ZvCzGwIcv6W|!D(iOb;{o%$$Z25LPkOay_=Og>06OM}x;>_& z=lw%v|L_1Fi4&ihN01{tt1RV>)jogZ5Bc~zZHI|-dq=x6U(0vm({vguzVj1a&;_c@ z`Jlv-!yY7~(+Lv}$)lR#=?u+CoKw(AE^(sTQ=IJULjnb7Jrn5r@}7o;&JikMEq^Aj z60b{cZ2rmppx^Nk>wij}Q}A4lPJoFQ{))h_P({ca2K-XruRt=4lgH7ILEaHaJ4xAL;pXC1TU$Kaa_!3DKgU_oYs5ChALv02_mqiE`K zqE*HAj^XCFc&>TGbg82kNiuwyTgXUh_i6Q~QN|g4ifp0BeAI?3?a1Oen+Cmckubk{ zq)lB<=CHL5aOmLiLt-+-zk>KsZ(IM+J@{j@gu0UnbVV^0CLB@chlD1>q;20H5%}6`IH-V+<*koQu!!3+HE{vJ@nQA?ihL$=HT3 zO_Fx>M&IKSQ@vCxBd8NaqPeq53(8 zbM$%i#INzw{hD78yI=L8E@+JnIDWOwYo6RTwTn6=JMtOzIl@lV0@!zZ$lq-e4bKhu zaiKZrQYh^;^`(m8T@se$(@+_fB6~OyM;qJ@dlGWrIFhLH<4kW|3;D(waVBLbkJG1r z35_g!I7TMS)xP1`!yPuW38^yN ztNI>=eIh$x-PCV;h0SR0A*Ls0kyY%V#@wT`Nd|r9ZJsP)BR`vT*gU@f=|030KAzXZ zi1{&byU^znKLP9tZ+dZWT&mxbwkbI}ab{YQ<5c4lq<~fs#}Q{g zWfPHmJx@mQG}OcXA1VGr>f!ByH$AYuNyz_65zh}sc~v9%*}MgAy1DJ5Ji$Vz<)E-^ z7;Fe(Z>nbFSLxb*&rMc`RO$uO63qVHcHuS zmJ>2!gH!U5-sI2^UZ@7S((j__4dvQcHjU7+9xeJ{`;LB~Ki0A6-VFcUj>Hvo9C;7^ z=pk)vOj}osjWpeiq3B1`=rvc;nh)BVzVg_if9NxhBlS<&?b+xiZLT4ClG5u)<MB!@aoliy`F7L)~C^rVEFT4l+sy$^qUO$e>0wB zeBezF`bX-7p@|L4L3z&yCDh0SIa(HyRsR^vpnT37msoV#Yc=VQ57Mm*LHe4q(6{~| zA8csxM_aL*3Fp8qGL=;P0aC3y!+vNP_u3IzB3tX`f<>n@uk;dE?Q!0sLq8R|&+HaG zvQn(T^x#LsIyy9Qr0+OA0(tG=OAO^55lnm`&Jwf22$PL+!i5RWZ`Q68* z`vC2v-O(u_`u=ou+xo9K>wo3xJ-$G3w(wv+&?oAbu8cW|`Nnfb%{}!)1ta%P<_qQ_ z@#(z;ggkg$4$a`P$D`clefD9Gk#5&A9zl01>$LP?)}iQq*B$=%b5-;p`VzVx$F1Yi zXFs`BV>)Zbr}88(@qPkyne)y6BGVnuLAS;C^EE#C{gmnO2aj|-HX6mZ=k!6Z^B>C4 zzQ)6+Bv80`9YjVgDPa1}z*>AqBjDQ1L(-Fc`qSV%Gk7knS=RC=8QH<<_>@>EHHj4q zPDI@x%#wEr$CFYg2_7OZd{XLH9bBM1*A1tnQgr3)Yd8}z;k@2DFaHYXl|2QO*UD5{ z|F8b#zuJEE$9~jLOZ8wl1(~G6l94d8n|Z83Nq9OF&Y^jPfAKH=h5wgoh0t2+cAdfI z0px$lUiD4iXn$^+DK+yn0pS7;RvtrT9NCh=-W8mqqYv#{gDGD0%AKgI>t6iM#7ao>SLKr^}v`^%{^~n)*Kb!5;;DbJoXS z!b?SKAKT9*SV$5L za@=Xu*?@Cg&<7YcCZ07nSeTgji^=IT_dEpn&R8PCJ0OePH zDqpg&%!{5g8sd_^#(9Ce8iFJ%Og?0rg>V*-Eqlwm!j?47={S4E_w2tKxQa+tV!39ZzM9w_>Ohb0N`1T)rXI~a@OkD;ExqCUS{y-HeaDny}9OYI3OKbcn zw-T7Ohel|%J~NlB#N#7wx`~@`u_TVU&x0eC1u7}dseE0>OxyVgILif`FWyruJ9*?e zr<~vrTwtb5^7s;eEH}dTEUCwgh8ej_R-nV4Q7VwxMeYa#;mX%yFq$GPX9veH&~bK8 zWvB`btG^D7`W|h6=_6t854gHVc<#49c8_fi^2$FBgt$NHzJMOLUISe5bMN34XzLSF zpf_Yy2lp;EqIexwUPYG8XKya+K9YMVPt=&8X>uRpHC%prh1aQx*6rTp*F9V5U5<^f zXG=7iSqD$1BY*B0x_^`{&_AHyRav}V*7sfU%xenox#v#5PLEgi`H+J9v+6`Hz6U|4 zHuAV1d1F#`QE$M5W7D5|y~<{Dk$U9Xj?g_;={~Sv6Ov!YyIo-NalggRd78r@@#;2s z<7~F0?I3TKjvuQf9iGUG4Aj9!H+>gO74{_kNLpr@yD+^FY;kbL>47$$%D#iqiqHFw?5>FbJ`34y*W?a z*@%pOQ*Ho_5SB9_I$ExEy=;2R0h=TnzUZ}|kn(Hv(6t<Gm4n+}?WHaw^|js= zj;H-?)570eFQO}iai^ikh4*8 zfP%-f%3|N<{KGiSIlNZ#k%#M0 z>WvTB4_)Z-zf`o%@}wEG*=;Wwv@DPjAM%vwlN}kUZE6!warPlYx060Ls0@ARab3EO z{m$Izuv~b~ft}E_DVRE~%Z=u*(w!&c!fnWRUb?J3@l((aSr%xWYFgIsu?}$3a48#F zaR-*(thmnu;Z$m4e_O4R;UO$`=m1&2ZZt-s=5Q5j64_IOXpWR;l{`sralHFrSb_*(z_l; z3e5IJ`S%sP?00;`Jd~msg=cY7Ti~Yc%Hz-N)q8I>JO@Fa4;Klj;aU|gG>&2#Ky=+6t_+6y~d_RSEH}=u`)mT*DLS8^LphQT9M%4ZX?#XGT4QlG6TB zJR#Eyl82ar_Sa>`C$xS#zG1|Tv?cKfPCBG3nCN&+Z@8fj8k}bzf5dBoGfl1o$M6bC z;d*|ai^1jK`+DKnZbAII-T;{+xTyIM{h~8kKIks8IP0;GvgGQVONlcuaE72AG) zE6&#j4JePP!@v@B?lpPM<$G*=b@O|6d&viWpbZvqfYKNXG0tP0jBz4A20)CEETa9Z zN4rm(?lfS^6=K|UL(GR6z*Eu6W5-@fmRs=#isS-;5o}s}BZu}-o=l-L4Aa%{s>KN5 zUdVcZz8GrF2S}|kSY(VJYmS)9dCjsBoV z7?u?FqQJ61rj+dkaSlX!qo!r#TBsK+U3J|`Cl8p#Vm1Q0e1~w7~GV!(ZRu>;>zbtJGv9d?=rYm zW{vCEj*Gv;(S5s)o8hxK%X1Co_3FeE{&`)Z3ledRqn^N;UXg3KqJYIKSkq?8iKqfN zHrGkA=a5Okjv4J+H-KlP4tFFEI2+T@`H%RvfJsYN+QrA-c1hvm9D5iMm8#4c^HtWLg38i3 zXrI9(|9B;W z`B;8q&CR6YA9hW-zE8S*Xf|dOnN2h9gE`1Zjo5T`-_ZO|I-C2KFXex7#*+eyrw%r# z`;~8fzsCi0KhOQlJl8#tdU>TD|GkL|J?>4`8{PN4iOW4vuE9K)izDP-;tjG!Y>e7W z#Gc@_Njx0|@_$Nq)qgv=4e-zAp!h%*wp+Lli3azg4r2qPJx_UaxtMSn>^tAbu^-r2 zL|()Tk=bpxaR|c3B%AZdEA??t^rruEzmI)Y^(t;0N;zK+$NHD=+#{iJQ*CB?6Be1; zpZaRe@N^N`K=QcCGC!_LmPRr!>{M#kl+fN`cL>jdN^$Kk7bY#^+UUK z!*;W1y6+UmKJYypInp1tYorsZ1Ap%Av@vem2>Q~aTnz9XNf!{NUB>CBt!=xb!^pZ1 zpuqgEQyoaLIjlZ!8Q8|`Ho=d1f|nfld4rp=;kL=|u|yjOSAz$;O0$hsc9u7ti#OM& z=5uK>JYW5?Ew9$4St z1+@K*U6?!^LixYm{OY~D3E`^os`-PL)^XdsazS+2Pe-Zz<4@#33LWDKCv=Rt13HW` zwbJ~-uggP6?L+>^LF|~PK9J(cK4@?-mDd1r5XE{Ua}RlGZu+U)Kjq+fePt@oBnfy$pFX(cB%Qc2? zquH5dainQnT(2qMcKCo@aL|Sy^Hy?cosA*L@t=Gbgy}h1yA5EDE*7^^1_q^+Q2fg=PFI?7rULTbXlaKLB zxz@n|^<_@zgJEaac`+M_pS^!su4!W%wL(k;0Zui^lb%7|{)w z$vg;_jD(h%`HHSv$4l?w)4Ww)W6=uzu4@m3D@Xa4JTBYb5u3OryaLe!E3Q4J<}Vjr zO=H&KU%%ulDB(D~#-D(T($0ghvcNucW01bT7CfgKs%^WnZw;#}nD%*$ua(5VJCs_X zNk9yLVt`ZOzQFh73!#_WmroEdhE8#_k-W>{c8bMo4x=U zL$)!UdRYBFoxk>E(cOxM5t`bBgU&pxm-9))u|h3j=559XYf+C{DCT@@*#nn zIQt#OK*I<^HYVV7XbeNO_-R9sI%hn_WgCla%qFk-pl<53n?;usKfdNEAz<_Xyj45cEa=zc zQGfrM+`p9TMM2j!7Ag5pDYsj0FP7NILyisZ4}HhNNIEvU&$5Voj7m>Zcq0y(P&a%G zC<5^A#dAFY2S&epoJ3|VL*l*h(CqUB+x_OnVYS-} z_>uz!sjK-c$IxEfaGVS7$fkg}&Bt`~jZ;W`0(KN|y4{K0W;JUcXSq;vDO1D}pR|Tf zi@&ZjYS7w2)AzN&^)$%mZ^i7Kq%|yF1v)fxPS^zqDuQogVg?!<-`)h=o>Q2_eb`t) zba_dSe@Q=r^N?OS4_O^NMjk=QT@#S(`!XGeL)%&}n)b|19N)NsnF{Pf4Tj26bt8KI~^ayv3z6z3D`ElEy9PN^e&vOEbohW*Y>THf*6OS1U$HBbcFchkZ6dEOMr1r+87)8eNO)9oHsc?HyP=3|jzKd9a^sw>w|#4fnO`rmymJJL44|BuCOE0}jc2XyLZXRX(%t zR5sMpE}nStQzyKF`O(MxAKgBX;=tf)`qXvX-j06QegWY@bTe1fL6_R*jmk#x?|!Ow zL5ntEOyl^H;>F*Bitwk?2n(b0^g;1~UPqZ{cXnn;^*(~om(_`tFH+j;x}(vMw7A4s_# z_bZ1a8N)u1syeCfh;0j-s_MWtv%a}+-keQug}a~8GmitsljhfTKatmT^E4fY&{rSl zK|)qOD^)HEtx+!jEW8shm2JqGL)>4 zA9XTMFqX3B$^13P?EDd^iq1L&}*E1y1;lH2m>wX=)5xyU9wgb&=73*pRBo+nG! z!&m%0zt7ig3X2we4ww3_8z^7i0jJ`i@2mX=yekezSAV%QZ%zm}f^YTBkY@sM&w&$Z zYt*R5p47bq*IVFvuNP*{i}*9`UHTag1jo24u$DVy>EH!bpARa8eD`qVVTL)By6NQ~ z{V~t};?MkSMKv6`O8C`pe7)TjzS_Th_%-;|57v(9wszYkZXZ9dCkEF6l4}76D(2J- zv%)&caceo3oCup zQ+!x|+AUOs9YN4+d3Gi?Z_f5Bl; zmZ3=d`Q@w^K^Sey-~48CAoXDw_GSz9wlQlXw-)PM_)Ozji#64Qtozr+F{C30imVGj z{t(x72%oG)&eaOXAdDf{b@#zd>aF?6LLc)m;FoTyj~|8 zC0P*KaF+WP3{m-ArVQUKnsKuU!bZ@YcV5b7kZFY;I>shI4kVuSrUwgG^b0*}0~pu3 zTdMx7+{h|2v+@4aliSnMPZqV_=<>!B@9x$D()`qdy<|@NsK5W{oK1_lpuSc8{Iq`^ z46xfK78ehbwKP|w#dT^1~%eUKRV_^Db+(?XA|tQ+wCzz|7)S11uwb~KZ|Q# zi}T18em$zXv{>hz_zyhvh-Ki7E4lEVjkfSgdwLU+vE*K~j*CBZvSdWQ-k@W{p!FHq zvWe(+YBA5I!7TR{j|JgQ}G4JW;%NydwDyM@QmHVnn$qu1@_O&X>ggDKG|L3mJ3eqU1{H$ej`s8 zIFkmPZFq8>%6ov3rD(`$YAaP|bJ`kvF$~gf6OJz;lJRO|!f{0E zN=|>vmurOqZ!~c)(|pT)hkL+qNmINx-+f;QU-0Qosk&!i^Z8yWN;QGf=b^owkqIl+gT-Jly&uT`VZ{Exw(BROxi>!c=%e012U#tGbOo|(yVr+4lKU-^ z`7Jl!BspE)?snVm+U|r8%TBz=#>UUb}IzIuTdYxWw{+s5iZNT0Pq6{~w%^1;>`x)~LN?VoEZ_$u>3`7ugAA!x_3(rg2T~8q_VE9N5HIA%Xp1I} zjaFVi44p9#3Voo+eH>TxdxO=s4E^hScI>*T|H5?d9-xV1FZAQao{!FtO<(ciewE#N z8QV@e=03)+_M?Bj?tqZ_5>Lq0Jh%?aHtfxl&*;&I!Ax8J5U8}?Vp zqQ`&F6KkKRJ+a|IZ+HuRh@v`~fvA>zj;}Pkh6gwvHr9<>~v5x>uff5zI|J^%jg zKE|}h7dn4gcu6`>+icg$rhT|x*>~m9D}F^JG$YF#Bu$yAeXJE|oBFW(L87y47#B}q zv-%*4|Bga`vnHjlk~cE*yl9P+e~rJ^lXcCRc}G}u8?ry6I^}wfVJwPAbSHWweG-~? z+94M{2TzOcz_&xhx9dD58=(r5VkhgTpxf?m(t*e$uJyQh@Vue>LGJ^sPgA*s^7_-| zeGRPsGq6j%Fz0~DHvvZp;u*mP^JNd-ax2%!EE)NWya>h};fvrzdPh}tzKp7lRj+VC zUHhaZB=4zWJNkO1nTunSU8p{v9#lt1$7Fj4J}%K~d%C+c0|xb9OcZ6A3)toe>*u8YR? zR&)xE?Y$`YGV~F84T{DREB$gO{)7Xp=+JyQEJ0fhA4~UY*N%a>lMauRssL-6~<;iX=d3V3$>jZX=fX_9BlaD z!n9a`w!uq1!Cq*t?Sfwu!?VQA_&6(A{^*9T@PmHz!I&3T@G7*vDHk-mUdmuB^NKq* zP&kC};myO3ZVx{6$o8f;J>);ien^YYhabM;g((l(ALqo@U5mkJX8LEs?>Y%P(fH++J)eAZRn$XGS zm#{WtykTeB)#i{l*sXvy5K=|2`F0#o<$5gd+(p(M?@yI6;5U+z_%1{I&?~$2VY}t8 z^ppKcn(M1d8W!(_ZqNZvfnp@9e92!uz+CCeG;6!JS~m8pnI-Mi<(DAwrDwcy%02AJjq>No>E5BVzC z%|+!=!SI;2&N*q$Q~K6i(%AD!?xC|BJek`?96L6QxW zXD;jc4t{Nu=w6Xc@S59s3WBy&ozP{@UhPFAg!MIFrWqT@_dD)CyylioUu-4OqYU@A zwv7zs3`U_#y6;8GyW9@}4rzLm&?wrzKO^%g6oHvCUm)RAGVJuk2M2UJYyEJh{la27 zd0nP(Kq&5_R&a7S;=^}>O1Tczi!J4+udpvCVryZS=f^WSye984|Cbcy z0Ndio2zjY57=r-)?H_MR7dY(ggXXJGK$9Oi=CGBz#+ZkH$rx8L?i6`_=&0!7t~yH}(wA9Fgm>$a^(VU22b1UmWh}#zUms}p+PBxw zSvO@qN}DVSl?UOa(0o){oPZ0IE*_!EP&WQq(%E%kmKh%rnF#5)m3zB-0a(=PnR@jDXhqI-3#EFS(Xg;QX8HlUJU2vP(r6QrBivwD8aJ} zoCD_(daGe={|ak3=^f*{$_1~4q&c}W zKW1_kYSOHTv)Hiu8z9^~8MATe_z_qB6$btCQ$NP4-I~D5)g5r=r@-(R*js6*;J}AD zX?_lAXTWux@HZ^HaNQ@9R^DsEn( zcREG_P&{-WpeLn2Ql1&F?|DFyyU!QgLIZW-=)!n|2N?!FG?T%QI2lppp5?D0;WgF^ zFww%Oz=)oUgdhtI4ygRgH5nW+*d&*7AvO#t86!={3*A24a``%VfN#q4k&Ds8B4s_S z8nl%sV!-$;BT%?AvRDjam?}MEmAr`$cDQKlA*VjPb}!=fu<)P3Sj3I0RD27j>o>uh6(XAf)M9Vndd z0Lw>cfx&}(LocYm{M2um<1r91;}cUK1(9bCBUun2GhS2#^5_051fd znPUgG+;jM)PlH$R0Pu+J^rC7mpys9vTvD{_%OWYX-O+>ppkKEl8vZ4(7iRp*6n{1x zq3gU*b{S|ighx?#oz45M4arhvQ@-mSSXl-O?)hWbQZn`dD7NiA?$+;KSzD~$o?jkD|UdE_jxc_u*WSQk&d^wDgr8Pg?$ysE`>X5lIt30iJ#y& zM~07lDj}DX;Bw;eyd3EE5L-7zna$?He5ZNN!A+n%!1SN#&+;@)FmuGbH|5?Q^yX5{ zF`jQV&!lV-JONHB+MO2(rO4_l=mt02-9Vr-L1SR?%v>nf^;9NaGs_biJc)vB!=@N% z*aXjVf8ajh%`7&hxJNhbBCbHsuavc2!~G7#rs2LrK9$RTzUTk2o%@c$a!L1nX>tAP zHKk(*aX-WE%Dt`bYibkxV4s*rmC3z@_63G)`IzNye?vkpW?n+4e^}ErjeLr{>v2pK zj&FgPw#!@B$&Z7HcHY$O0|XA8o?$~A%n#16?Xk7`wP}9J<;tUe5~`nC!S$2bJPD?B zo^0mn9c(zbK%PFF8Fpw~`Vm>dFFato^;#f5Xo8>Ku#9ZHZoq~2G{RSd8#rMXthRUm zjQjP_rECn}&XX^bG+4gmiGVJZhgw1iDZp>4A99~L) zu-%3EQQ4Eu4}A`m{dNLBkr?F3OYA@74p>KwE%TOt<_jMPAR*XodAsaQ(k-4UtN)~u zK3kXJ<+1utD<4EjUyFahGH5zN&$p#BLq>(@5ca9coCM10%<6n|Ht{u}&$guiH?#kEs$2g}SwZ?E?N;k%Rvno{6GPl%R zwCcH$IO&Mj+ch7JwUa4`H|nC!zGr*PcO3UX)&`pYq$}u)4=-1&`O!~EiPz>YJPO@K zs!#obj(FY{Pv(~~i0%|m6k*NR_p;mTKh6BM5zled+4r|)?FLeB=}PL0TzYM7*`Tjo zujSEm^nzVhXgbAziOcj|PopDJuifNCvgKRAc@}&tp=Io=E)EVTo(0W*Ac2%Dq-?o( z%Bjm~K!frL{k00mwWl33Q4uF0X`}9f4)0l6=v2Z*lqW1yhPSD&IGu)laPoS?Fc`(N z;`xxfxN4)q;9`Uq0>|*L!T(*M6+*7{qg(_2LQPV8Jfz{eNZbl7AWhMz&xDOP;WI!K zdCD6&P#gT<3S|t0mr66=&N-pz>Q1xxrS7`v*e!m-U;HB<&@SNk_7tr34IYA-+?NdP zU*iUd=BR7+FVMW84u6$zb_8M7lswQ7LQf7E;}Fw z#K7RN0fWSg4jI+F8{CFAMm~$uGW0QEyy;N%Ef+p=y?Bukg)xp%?t`tXl+U3Qiyc34 zg~8*pG$D&p8)SvkKvafeJ0e^bggexjECZ;GMdI0X$igFe&MrWjE)`Tgan68pJ0s&!&wG zUl!c*`)Mf~pL-*ey1kfyFCqN4zk(w-*3p%|z_TN`7E-ANds7lVsE@koJIlR3K*CLl%!!MOqjH)* zSZsN7W#|uwQhA4ceyLe9mJ*>eEG(is#S>s}jnhI$<5U(PI#wHlqTLId9CX$g6K@pq zSXUpQv+jeUxdQD8|19psd*N89L01bb%OXub4e(!e9#I#U1Eq1MWj$<~Mx@E-Y#oVB zyz`LXZpq(zMBCFB8PgS4y4Bb^HgVDn49|d;TIZo$JXJQKb77=kl+p4l&w!->-bRCD zyX3VGSvOR^v@Pumb~tg#6c5gbJAiU?aXE+O;^VseB-?>lgae(|agI-7mrF=c;f`>+VeJb*;U(W0aQv~%3Ts*sQf0F}U{V|QVKzJ(6>P*Q z|JvSN9>Q_})lz|uS$&;eNN7nk{RNU^UfQ4H6$UO6O96RDeu0Z{v0h)BK#zx%YyUb< zaLeD73E^_eU)b{mbB4lWKIyqd@zZ8W9?xg~Trap{*QZMbbpgz#F5T|YmIYE|K*&sf zLMw=^;d_zpOV7hqn3QZxhI`_g$cr9Ma zpLXifGH4N0aGaHhcv9McpQnXrW?bqAZ5)2Pe9#EF{PNSkfb zOIPyB#$fF39O{7h=|9$Seto6%hl5O>O6n()um>d)=|Wz`IbNGx$Xwwc;+4GH&4Cm; zBO7+n2{#vBk+a6C52ZvO-QutZorD)&bIq#*;fI4MbUbZX4yC*#$y@`{kD%8{&R$J^Gcv)d3un-xrq@7D+LRYoo5fUeQCJF6pLDhf6ouC@lJbOT^ z1Ltve0&XDdufeZI7g?(%gQRq=%Jz1hgBzi9^oszMV~0xd>&nZ#Xvs+GYx&-{f~i-!Rs|-?#n;Wv19VPVVbVe%`-yfU(% zs>{Oo&C*N%qRWdrbO-rJKP;!{V(UpCMrDIfd?1VB#n`ZT18L)h&}9}n^ik_6Gyqbk z9;`AajZbANsF-rel(H;%ve>D`v2-VgQt5m004x%`0c1I`h+*s~0p8FfqvGVkqWK+d zjoezthE56jKZY!BqEBR`EMwCf2&#*PT<8_RzA36Nth1%}M)}ni7W>&4s=i5~z~Ze& z*zrQT#>A9&z;e}PC+D5NGx4(f(K~?rc1MOz zISrGJ7?hu&qwo)0>U1KIu;U75ct+xbr+wK#m+f-qnYd%-&M1IZCv~|URB~|roCkDF z|z^qI34g_7FF8laN?bsiuI2ti{gy#E@EpdhZD6aC1It1<8FKB0emv#QeJzOLQ zV3s)Mw6+^gQdXkVnR^_p2s`C5#5IjDoOap`@4hd9J;yMg^qf-jtg-T?T&NbQfjdw{ z=XHKuI~C_pgBdSq{zrtN=^S|`>~wwjg*fy1cI$2IL^I(#&G%?F_~iCx-?-;cyis7A z$@huedu-#hU-8sS(XlB;A1EBeFRXi8+l{s*sh@bIV~_Zd!kfx{@6r8Jbpx?MQ_l1h z2hvoxpVrEC+s%8Jk4%N|;!D*jj12bn2^jiuQ!0Kwe2oo>==**+TlibbHca1@$9$Oj$U|C7Jo2d5 zt6po0AI1X!9nUq^kvbgk{G0x484{dz4P2DeUw!6ah0x5L{l$vuntWZeIB~PJ;w7(<{JOF zPicL>>HB-jvB7-=pXffi!u%+HLHw;Z=7G7b?qz-*shO@`7E?IQ0rS1C~5>(7tLT&5Uc$$QS7F#P;RWaSK#rlN)5g z1st`+yMA0>3l`Ns>#fxFLMnGca#;H>jh6K$*;lx9icjN^yPWs z5pgk~Sv?5ljtOmtNj5BG@kXzzz664r0F4#DPJk1ZI_#IDiFL*S?N-^9j&4E;x0`xK zT`M-=H4cxE`s55CKwQwF<_A|dfKIy@L>ux)D4x3ZT^2{2i4B~|cz_XgPWc^NEB!pU zR-1n%0NIzjIQGyt=n$wySo%unOrYA`^0kgicR+L~zwP(;By>_=aINEn0nu;0Fb^_Z z%!B;R3vn1(&X*#mpeOy_`1POn$`~1kRQ$S9MmENGyH7sxqzx$8!SISf7UReVE@~&N z_%#c?AXZskanOs@^>u)g4~!iQAPk>g6vYrCjV45;8GF;BQFZMQgW5)QehvJz@-;u^ zznas2L&+2;dRJlOUa<}@*9Oz73$mjydg@=|NvB=6#!rN{FMA47-*oM`tlW|*)k#ZA zrnQMzixB#ltR5>KGeXtx%|OMuO+^}_Lj=qX(lPRqOj~@AWcunWkL%jRv!1G?3v-Dd z+SJ>Jsvv#ZuYO^J8U5=g{g@;2tG;kUTgD`E&Nu_}NA>YHzrpBv*M|!&*S$k3X-)#+Kp%L@#T>0oa={fv z!FfRPuVr=x1IepvJkEmG07-MSVr4b$cDs&IQdy7f6Vi6KpJcf!-H_gu%e+@ILw-xy zKzN3^YLFdmc zy33mP4yRy{`l}J0?>roIT(ZgwTwIrsS?a0$Mz=pag|FZM%3Gd7_fzg!Jx@&H0ux6)dPgrL}9@z;nke<4c8Cf9}8SU-P5}gM<>< zM1B(|-!50~bee9+bsb|)yj_W4WhUrlt}3oSCm(2Uk3W(-~3S2i60R zqqHqgMKmf;zD7fNkU{)SS;5|335s#CB}L&KrIAE?aS~yDkXP5Z4i|Phu5$}aH2MMG^hfM8+_Kvg=Qdduv@tdv z{YroO2|H}^#~!my?kA&^hCq-lPek2&@7>#d_uaGIcmKWH{rBIu-F?s9+vQ7_w%c#z z)t-#Inp1?aUwL|QIlL;oeZgDRhc_udY{9D!qh#ar!^!@^qkk-^c-yfkm7r~k){k{= zF|h`Dt=k;1Yksiapff^rr{8D+ARlDLIwY^@#=+L{yN5_fcnQg-W>!g zi}jH)vSN*qm0uUlg^-cdf!xbg&baU}PX!9i5|2j&=)xfe*Uh0IOk(5dF2A%|*2s_#0{6Z9p1 zuE zwNKm^+#BGv(Q!%hTO}6WfbdBDxlS<0^-1UyI8B$vHDmX0$D7s~=P<659a+J^%1-gy zkkBsob;7Ys(LK;VLin?H@NB?;3I$NSnCkg(rtAqquBz(VK^MLjfoAj~==Ly{DLpoKKkc=?z-_>op}mw`>nrqd*SDO-ge6eeNe@`?ck1&`l#*h zPyECR{XYPZ?<(SJg)f4&uh3@hoVYzXdZMg}!|6lws-D8NLI-ux#I4zzov4Xx8J814 zj!V;b9Yh_%11MMJT?fJP8)rT1WZQAZ6%N7hQ{ei`l^k$e9wWWt2Q~rd4%$`ku`XbV z$Ju3ZU>CCo7dVlAL6NgyBJti3EN%Ud31&eT$=-d_{UO~o4xO%eeGSOX`U6&Yjk`!L zl_l8@`Kp@K3l9ot5t*k)nkV8aEty^P6sSeYl7$*Zk)LXjk#to?(_{7I6u*e;Ka=Gt zC>Ap`{DLU)awA@vA~A3_{DgNA;n^; z7fjVA$suwnW7>$}QwC+ZP8at0oc@wz`FC<%;CyLyc}VyIM`vi^>kuyWH@_^9SKcu| zZCoqQv&u}}EsLH9sKcA#GW2?*<%l*cg)t08TiAjkY0X^oKnH+lZ3JzZ9_sEsUwky)F{OV9BUGV}? zddT4>hf*y58oO=|q-@-C;K?Eteipq7RQnnbYa^GEa|2_duMLQ)gciU0jdT_q@G=+3 z(btqc((MawmeUpkTF3;Mw1fz zwb;_7n_^bhayz7W6{ds>R}gAN8h36KBZI>8{DouQvCH`0*RSC4yCP~W7Saa#H+|uL zKENz{D8JAkV82rX_E%)p;xdHmGN8#DphX$e#9vs+@F0%D7i~3Wei?HNw1yi8NOYAI zy3b2)9oJx1Rd4h&^I-IK#_@~k%#V{_Ln?|}X@f@T`~ILy@h+dd4dTddN0|=!6z+Nh z89NQclQgg{INi^aR-w{>c6YGmGZGd5*s$+3eU5Ui??m3JP0x-R=0(`Q}pK&2(F z;yu4HH+S863fVt~VdIt`7Vv`z4HjL>>4{9^mbf`qt(QY3~(cjo#K9K5Hm(>r5WFNS1ETfW5%f+r6VBUct z_9>0{l#rO}E4MHDYWo6v1^a@7Dd{LbezC2=MzL)T<(K|LfjZq^;sw8Drpr!dByT+%QnONk#_cj(Fmd8Vs1j;uYR)zwhid^^Y*JKo|_=97ix&vy4+clnTsje2jS8|UV=`|I{wm8V`q z2Kwm_%`Kq(x88QU=NJto+52zFTJmbWGLHkYdbC@(g}T zclvOufAmEm<}cceZX=JZOL8bSbRAiijx#n!U8>l9WWLIE7nKgxe3z%j%C?byI?Y7L zJcumi3zR2$4=6eVp+j6snBt@*MAwb@yB=p1xNy9uAlVDnvU}Hld;E30uXe~IEVsvB zPFLpC5^n4rG0%r@O{92kprpRhoa+S>~?#o6B^~_UJ#uDsR#Y=4?s&VE?wsJ zlzkw@0cvzN_n4Y%nm1KLea#no;yG7(gYNoZ3iR6XY30k)eU#5w$vV$i>oLoq_7Hqe zKgdoO({R168+}sSO9nZ-6=TW=jb86H`ku*nr5?4#wn()pc1dBUTSgg2IgqOUX*q!K zS9l8O4Y7J%*j=|m(vo05A9)f_WXCy+{d_0v0;tsDB2$OXg+u9U!dNwQU->#KX?o_f zeex1I;u(O3*q`9Yoi19pKkkkljL^Wb{(iSF7@7eHhdx?Eo^>c854-}ija z_FKODySJO(@f|D2#qbV)_RnrF`jRi%G~y~|+ur>vzGAyibQ?b4^FD9;EnoCSYUD+& z{ErRrcD=a3+9v!r(#BI+QJ>a8O_Z7?<;P82C!Yl9H4lxxfl4*&MygqGrU6Z>30FO% z6353ofTAv5bYe+Vc%k2Q$j!|l{*DWcg5j?U%|l{l7+Fj=#zbXueEZ{6g@?M1;f8(u zF8NH}6VUB?s{k5}Hv%+xH2PN?DKSuq0QfmGoEq|8GpW?Ex;Z=bA(vBxuFd8wCx>;(C%f*0e%t4#msdS4$pI(r9 z!6DgZqbK7qn=BQ##K#~K+E0yfbxr&Mkc&lFY0l>Y__}N-`%>42NjZGQs58p%4Q(T| zluNvl+x=lV)cDuen8bEdD^+&vhpYpz^ie4&BAI{bquPi6MBKuZqWyt?S^recm4xb{DcZSz2=Q&EGFLB4tdY|gO#vvV z-~b923~$r48ys@%dB*Vz_+Ag7lZu#qCY+i-UyW^8} zg3H7;v`Za!Q2D1*$!?v(WchezETO+(X#-9#5aJL3%ZZQt7*Tx}a!w zAF&B(nwFKS6b;q^eIP|yo@!s8AmQ~{rX@Xye2`Vi4D#kZ?d#^Y&tlIdF}Q`T$ITnM ziB9IwG&-7B_w`0`Uhf4^_u$3SXu2)4)|14OZdRN3;y;nXf>1Y`1@g>*VD$@@vi2_F z&bupadT|T{;gL`K?UF`pz-(5VZL2PJn*EF&>ucqgC;gg3DQxh5uW{s1 z>CQVZZ+G2wrw^y@5fZ+1#~rct7yo1yYywy{n!Els;%UMukaIW{Xw751F|AJ+H85KUJk}s z2Ybyi4yIyrdp?qEtRr0!{b3fS3!s1gaS7vN#irq0E?#AWgo;G+=;A&GrJu$Qo6bVZ zm(b}s*mFb0%{msKu;>I66;NJNgdfTkePZn?w-W3vkE4=llI}=vnA7uw zFZ0pt*Vb>}2hmlLVfGR)gv^=Kn(Eq-NFt?O$@H$Ekrf@l=^4lLu zU*}v?ya|{usN3si#+7s>bFTEKehAPyWIfgg9%y&wH4db*-Xq+HMdq>j=0kmwmm5UA z^gC@vF0{Y&3V!IvWLG_EU+>{G63*H$az+lG6J2k3v)zJgO#c`=j{HXY5&jlA6Br=> zPNxDyHJ%4fcLudp?#^FnapSQ+S*`&vJA4xyxMtxC00K?1E7;XAp{hxn0D15qs^0~V zq)y~tVdZ8Iof2%D{9@?z^YWc#WVS8fZYsQpZ>o5aoa}yd=3Sw=SbnbXjrZJBzKh{U zfB)}Sz1Lsc`~8>yrSt!DmHXy*c!%w~{+s`1d#Cq%zwHCQ`l~C)#qe$){n6WdQ>P|{ zhWGo#Pu$+|BR)dSj_`l*Pygxd|NH5menWiys)bdb*1>Y#z-b_vJS(27oC1qV7u?oB z+zVw0-NoJ2L0q9G!Qt2UB)F~^G4^|+CX5>=P&8J0T}NZZ<)*Np%iHkC?MsL7 zm#eSAa?+*c*WuRF7GgOQmWX2-w2ip|TX*5%%?gwu5}t&`%c@5qSH^-22Nq=T^Q;oc z5+h26xQr-%F~{Q72LLQImtXT>BzdzLW37Jm76XmfHr#Al7+NeSvS`g8E{_GU7q2;J zu+fBpEJ^NR9v&Q(b?xI&g_6y_>K**lSmw_IH_;R@1=kd_er2j(6q@RurB>&x(!f{!!2yO>qgOt3m@>;G#qal zAf^ za37R=5xVW=Mjnwv=D;}z5ubUc{n2MWlP|%-L{dWE`S{{@J(1h&o5CKSE=T@7jV{l@ z2Aghd!13#F>&GK?F9Q3%-8_e{$s3hhzU1@0^CcOoj+^W|Ru0nOt4%*uKZk>8jkW-~1_vmMg8o}Vvse6?F-@P zf7+gQt4afIqZv>Inae~M+U6w;B%k=C&2lGFl)`(cV#V|ky2RoR5@%^On4Kf4BG)C1 z%VhzPYdL$|32mq~9F710|MW>jK~$$9-R{tF( z^2nX_9&`pA`;}o?yWDasKWfx`thpJz zdsf+Vcp9E^@R^N#czIfDgJEYjW!2;UMqjP7d+a%OZ1!V|8lyLDlX^Z9KpWYxPR)~J z^Y9j)ye>h{Tb9_|yRKBdTy>#-2L#rD?xWJJ?yph4+!UE*2as+oon(B0gANrq=YXRu z;&*~whm-Yu=nP%f{uv!Y_9VS%a)myD=Aq+2k3Gp9^nH+ia9=D~y5wdfH?Y<`aY5-f zeNqQeV9NyvYpS!yJ9T>AOr7Xu^rO~Ur5irvW`1GaC4cK}Yzg>_-nj<)L%OhzFLX8I z4_?Esc~W-T#avCj{1>l(5NHf-u;*{Lo&Gi5qCK$MPCWH=`^m45xpUiF&f$CTZ~3O4 zfs~SKtn%%yc?NC!0@DZVL+64T=&nk485ZvV$Mt2+dnPk2U$eCD=oakQRG`Cd_r86B zrhdc~j^XoR=4>CjtE+<*H#05VgufB9woH}%6A&R>(BrB@?9U?{F{E$_QLmhpY0dE@B6kV|JA>8 z+#A2*E4H8guJ76|zunt84BYXb{b$>If9=<9Fa4MQa(ls_`*Vu8!|@;Rs#k5#e~XUH{5o**@euzQeC4yy@P1owwn|U;3rnhkX0DZ}+|OmD`K{!e7{a{onrE zm9lM@e)sR*e#6&%ji2ax_rLbnwp&%#TMyUU3k^#jdN7^#2ZlHSl?QQ6}327TP`IsqGK83;-EWA+^#MPb-{pBI)7(%WI@*n9~eBsq0d4aW|J9_k%5ju#lp1= zt6`kTK*MnH7;VHbQeJNq*?5X!??s^&A-qGJMM~ox3iDoY^721I81MW-h(!)h!94M} z@QFM<<^P$|Vzv!6a4Z1DyA8W?d$YS*G1?Mb4XUKu@64;foH*Q$Jr7dbu}`Ldk}gUM zK}%g($9%SsNV(=0naA&hBs#knE$EhG)D=pjujav25F^=(Es#D1>8~jCG z0K;)sa9i|&9}Hg~M#%V`zrJjvJAF0#2^GqM(0rS(o2?i0D^E>v;F$x^;QA|0ztTee zZaJ8s_!TXFIoJA&5s$8U1BSK;Qhj(HP;DJe9cgkYkZ;1m_~UCo5DG#&p~fPKMFCGt zArFtil7T^9hq@hvE4iHE8q8~4z?(7ZzKzrIIfS?%vd5x}M(eERh|UgWVMkU4}px<8Eg z(N*|f#zTj#$M)myb$TwegSzmw3pl=LdK0SXjYB9!vP4jq=%S9?x7I%-y(>U^o9@sQ@X#R!tce@8{eel6b&w;&hLwxz2r{>0N`f-rmn<~L< zvebVfc%$Xk?Pkz)B1>cOzsjfibNy_fFrMXiym?4@$_;c}>9l!M8KHR-54RLwy(!~4 zqG+z|F~8-qX*A1rTjjFBQ=2?)8=IxH)o$oRk2YBL=}m)JEuhcJJ0KC{=`v|wxqKWx z;E>y#V;ojm%gV@@^tfNJ(v`Kql`pUanh&`yW4h2O{lInqgfByG0lDNWFd@qy8T&sq zu;=4{Y1b_YO*$H~6YH$pyHt_+AhCWHvp~t*TSq!F?6Ipltxwcl zbe%??0W1H)3KEEf3Xzwcv>ExBcTn~ad7+!W1*6Z|+!XfwK!1w|_<&caBj3o`)i@5I zKl+|CWTJTYjbz6wdi%#@+~@m8W!aoB8Jo|hAHGK$QjNh9S;eBjDkjTJ=sp3h$NW#_ zav-Jo%TFxSQw#q8FvkAiAN`?jcxxWS1@$P+V-MP?GW&9Z5y3&g#K72vEws_pUh#D)@J=0 zZN@cEBhdzQiNmP7@4j=p=kB|WcMI>j`!2uI?2g+xu-4qmoLaJn7t2xgm><;%kmmUu zL|4_H*z>fn$Iqt8m*<;~N_wa{(!rYlweO&wytHj*tZFOTp?t3m2vBp(JtrFxr`N zZgsDP=}D7mDbi?nCfG`0fw@K^t8g{}|Y{O|tV_J*JOsqH;q{_^eqKk+9jcH4gB_y7Lw`+nPRv;X!F|M2a@ z|K{J^-bLZ3btC?e&-je(W#98X+fDb}vpsd?%Jvp+#v0W4->mc}A9+NLdUkt8gZ}A9 zA2p2^e$WSPFM8=q?Z5o%e%4baW3 z%S=C!@D9oB!jlF(yA2(t4%*dq<;G_}uyf1ba0t3S*NNLa1UUUc&aVq~$|#f7yge)9r8mZ-KgpjeYf*sVgMnfr1p`neXc+AAp+7;|zokqxwYs;0-5? z<2rz8*NZiDulfp#=2><3h8VhoK@9Lpz#V`(enQtsIA6o zr!tsVE^!M5X`z57mv`4jduFs_Oe8RTlpiuJ?qw=aL-@{=cJpa&*R7Z<5Q1znEhZa=Q#s?HhL z{?N5c9+$85;h*j(0L(x$zo&M4BjP-ytZCxM?XpcfaU)-~O@V<47ml`(f6{20EAP&4 z8f!Q>Xc$vR@+iM(rQD34>2h6fW7XaLy&`kxKrtET{J3j<$i})p^Zd@ns&$Uk6G(uZJf2RadvA z`S+n(4%YfW$_H!8H|Zy@|F5Lm2|`=Ex*rO}ZTST4U80z#M-Y!?fGzh+ zloutY{DE!v2j0bQ`*EhWg|NWQd*xq_Skmi3a{&X)ITDmpW&Ce z5;^zl`Uc*4r`LRJrh5*S{CmURt*m;yHgLU+J;q)1v+q&pFzsNjVE)KvJOAPI#P+24 zc={>+L#kdEo|S;>@Z9NnHgg+2QN0O(1HFkjV& zuGcyT8C$2)ouFaq6Mc&ldM2{4{SC@L^l6l(z~mWh0xnzF{kKQvwJ^fRL_;oP2Sh$i zSBW^fjyg`m%$uyqpRxrzzjGu5jcups(il%9|d&`#?!{h&Dr^uhSk zIS9(StMm*RyOf}-U+S|x0oFW;4tiZub4s3$;y?=hi2Y#OqIk+ZJ+>gS&fLrxi>@=S z@&m>AStq%V1k_J`p1AMD){Ap%O5(RsTYYP5>x*%U* zm4W{k>J*H4=pQlAS?unU4A&%mZjJ_8daXQ1SosW^!Z~DWMhHd;aSNRYG_f`A`r*gE z>6^;8hi|(7{#0~>@NfU8|7m;Tp@+8L^<`hS{mvizF+X|qymx>1O5L{i`P|R-B=f)h z!9TeD+mHR&?fbs?i?`=1{qpbpo!d{m;uYINKmYSC|D&JuN!y#g@+-F=Q$7yJUjL&% zy8Se-20Q@_@E`w&|8V<1UjFj!`#$tTw}*b=7q)l)?Z4e=&;RsK-|lP=VEeu=`l9X4nsD#yoC6#@#GM4?>AbiM_H$4dG6r@^6&`yqUJO-_j1z0B={lk4S75CR=z2U8dp4eN zt`{t5UX7}Rnp5Og;8-F@)7BKUp_~=XO`j+(OnEdz7VcD3gbc zl>EV{4C#Sv_Q&7-LtnKK-n4ObG0r*(o4D&a4dH=)!0;YM_rMs!Ua%>B<(WS5Qz$l! zvjEMLQqSf|DNyD4KQq5BQepnz%m-2>gAA$Sn>MxXs;#{N+3lomS8o>hOmJHl*rnT( zvO6d`2I{9((7|8i7<*&E=8<*Mb)SQkHVzx;Y$bE@45_(Ph97xF|O`Ng8zEIQt(Eq{+0 zFyZ7+TqbUt)y;jP*Lwk*;IO{e+Z(=>`!%Vx-^XpC}=F5hjyo$VRnZoje1Pdbr9T(`&m zwah*bWk=iKwbFvz3v)jlWz2uVM$0U7cKeU@+Rs?3xKUPU-l^OELgYX@IasR0B_BdD zW{z0%>8c|}jXB>3G~Rqzqw&Ya3I{m-DjILP5a;=XbjBqJE%DHDv3wUKPU+d?SnfCG zdA6Cnk?WLA$|-M`Gk;ZgZT#fm3_hxTv`6LmAx84fjiT`0n6f_H@?q3IKmS`Y@+ux4 z2EWx-#;6_P5i(CUC_h){#ZUS2$20tmkxSf`qr;>*y^;3z{j%~3P4@^RjylV=zi1b| z#-86uGd-6P%o{WS%tXAbHCQS&tcSKe)>HdxH*)njY8$~y#o4_ zbURx=;;JU6<-QFc#lPD&fAE1lu--HKeuN!CK6ph3xW`x*71y@854?$o zl;c<@ZU8=URHE8~KD9qOue1ZX;=otK(k6q0xrdW94n{!auJ({<+V0;P`!!xX_HmIx z@l%Y3UdgTP1fb<%I+Qd2DaB@LY@4V3>NGYzxzG2#e)edK>jlLZGsaEn~?R*cI&nBtXvjT~58{+e@JcPtC|p`PLgmq_=i=eN=|=3(hDK)!J( zC3}(|F6NCI^S_AQkAlo$jNhpviEH`5M)cZGrief54fnmi2a0dn#(fEB8=m@o*v=E6 zPvrl9hQ8NY)w<%I3_pPDg%@`UjMx)UHp(^XtGG~b$773E<06LyR|)3vYZ+sVgicG73L@6CC599EVdnO`AbJ(B#< zkH)OG)Gu{dXg#wWEG(A;M%#t1r|zxj1^w^!v({AkQA6uS{LCwJU0w64YsV92=S+Ojq60z~CDfYc|9KH(-k{zh~!`bQa$bK;Rd;gPTCkBeYdJN!67 zJ_~f2>6uVbdvet;1^CynGp}JoMUXc>!I}yG8h#sq$c6o;8@J_^)Y)caS)RBDxIPX$ z7go%*axZlk0_xH7oewkbSvVVaF(1x37b^UA6KWmbGH@DqRJkErmz^lN|o$IEx) z@W}u2zij`Xm%L>Am!I=F+to)N*N_ShTVU_bDt@BjYoX&Lt)@%R4T_Ur%7-`Spg^{cn<`H&CUu71aNRF1bL zT+mCMbX0zV#5un41US#)>Jn1E^qBA^H$|Z)NaH$#R9zm7jhgT@^93C@^UW}Yp@WOJ z=&yCH^#JYP(b0wa&d_cs8We51hjwIPQfU=8uo)cQ_Eh|Us;cm_jZ8Vk(}#z_pJEbRK1K{k>qj?D-TocNV2PiSDA z*&x#gV+SLWIE*J7IxIaYGo8zL9g9pVqP4#uXy3l@)xl3#;XHgAB%di(kYCG+w{>Tyf zrWTfrYMtd>3nu03jcwZ3Izz?KPv2)8u6@t175%iE#;-RNygB)lWljsySJ_Nro?veA z=CSx5ZCntrpS0q`eP4c~)6y=tjqncB9{e6F&@i2L(HRz|9hY$D>3X_c(@6YUZ?yqG zaaHHahut)52O~}By5L80LN?6H`E%vV+vPj&+%EI~4Z^#G_%Ca2ug%wZxX4|=E&0lHLHsURVDs%U}_XzJb z-g95jVbh9ykRA2=UdRtDMB}n(kY`{1zU}Vj2_6Rx`lR{@nUkhGcdM>@@4w%8-}9e8 z{LrE-{YwA(^)1Mn|Lag3eBbM`_ob|n^3S(Rc<`e7D09c9%WebrT4)gm@6d4h=ik5G zFNBX3?-iYUXrJmtZnxjTYuY%pVLWu7A9Pgl?&IpW?t|77`quqSIrleuQunG=?<`~W zx7u+JeTCc<$Hu4ox8@0t1<8n4MOiC8JGo0;)4Q93+;zMU;MllYr^^zQa@;4G02VTOD;|00@<>^*${L{CRJNFsJRzR}Qy=AlA zmRI%}*&x%r&;2$}Wau7In^u79s}09-s9-&neTg2ZkM8m~tFc~v$*;)a16^cpg&y}b z?m76;tsFjPp5X~xbd$CvZ|+r=4>FW2aa$hU_6s3Ru$>N#v8D0%aA|yWOKgjrj5kQttNq#Xo+RQ!=#a zLq7>+8w$HFc17D#vCCOz$aQ~r+fv?cw+W)IGxZWE$I$Wq$Q}0=@x`2h4tj1!Uzj_X zFTt80=zr!M%2H$MHzVQ0yvAO^K0;4zt4MEctM1x0`-%3of6xod=GCs5w^z`y0@^bI z2WTP&=$3l|7Se^fEOb}?j%%g#%WM<6!;gsa%1zkNHQ+A`DexW~T zSGAKyPd)E(7=4M|#r(+_%zVHr*e+kb!)s95arw^t@0I6#%@ve`S9IDotLCvM9?$=1 z;m>@QgQy;J@Y4*rUZHuZr|OZxgxN2~wIx-y?~NXJF7FObCV|VTA6tLN+Sl})!TCZ9 z85BRpg|_Q+M!WVG3@`Cr;G$C;CY)^E7j2#ii{vpVV5= z^N!b&Sx5FbUB_zrq~rjKPUKN_)!fYe$A?gb!>*7X+vY3RwhT{Kb2#L3wf=!GbQ`|J zhxw7Nnm_bdZm+q>2S>D-x_z+4y_9>Z=-d8g&MNxIhytl2_qT34bbGFHUFc)U!E3is zPuo|qzx#(h{FoS70MPfE4H>!(jnJh%6Fp8V&0|1*#$^uOC?90s+xPpr*RsYQ_c}iA zo6=W@?U@io2^NfCd0Kx5I&B8pn}ju-h)yfV5y`XAt-`H`6RI5*0@9*_3eYZl%e@ee z8M;clTsq=3rNquO*LgbYIZ&B#mU3?8flxR}#^&mJVNYqcY>ZKduR-;Y{54} zaI$##JwR6C7<_s{>2whCGpFmJ74gE zfv@}VAK%`r8`3Wa+^$966AwSU$qJ~LJv{ae->`k>r+muxnt%Imw|9Hl%eMd5H-4l4 zd+H`F8h?4<&|$+XPy5f)a-xmoJ)q04NP$F`ItFVd=o_ZPJE-zZ?sEHAA%u6>b=bwz z&~C?r2wdg^q%+Yr)8Zr||?_kaQYuw7{;RffG?_(+x2qI7R3h(0xEr-H@#_tv7^? zBnr%uF3nqq(Z@XQS170=M{w%AEZNZwJi3^NGC<0J$v8?*jF|iys0^loY&L-yP`tW@ zSM+5g*+veFUDb!d+lEjUjknt9zmv^T<7IC+utA2QgmEM(m>-^;dR%iYzt97(KF%-F z_!HZ_6FCMahIowey`OfZooP0=wfcy$;kJBtcmvRKY`{b|I=750IYT_Jyjjll{TctBsk@Rh|Obn`1t40_k%dV6tY-YF~W z!-HO21k1n|nx1&{;(_|bQ~%0x6>c}nfdNT#HhJ791MnwI9jpGQZVsfTj2luy*Cti& z%n=y}LwBGf`R2p&g?RV5`k954$Cmhj@aPXM7FcW`cdCP@!I!>(`*A@a^|N49+mL(l zP(^AC()VoeG5iiQnP4(r_LDv8uQ$Un;y5*5J zWCLx6b10Oyh0gM!yz>^S?9d;#FJq_?dOc3k{;G55uez&V!sbtvjkcKg_6xf~3y`PX z(MHwHxEN1(uvpfd%3M0IHr;b5lUINEz)4{aIC6-UjhGzZ_W{ztK{p3dcid42T<|TLK9rIiQ$A%l21Sl>1Ej4c`0k zB2R22-xwENPIDxEH2NBuNe+D=>OQ0YX^&-cd-W^2O26j75ng>@X!*iBNMBTcx=$;= z;>hcRE7wCE^j9`kZPOHg=(fjX&p`*+jbH`l7|%F1I;lQOp2%U5MY*StG|P~Aslsz^ zF=bb#uPej>oU-H@{(;V9m#}H3Xzl}=)7ij6CLRky#`{fh+q>{;O;( zYrbbA%Ntg@FZ6xfT+wdYg5EIRu}@?#S`VsEeHe8QV>Nm}UosZpfilb;Q0pIFK%X{* zrtXL3A~)Jcd$<>JzhXPZo0K(&fs`$Y(Xw;FnviB#aL@@BO{3^SyXZG<3aO=lLn@W9S~0*k8@;J`E^PC)JuP| zG3mNx&&iI0pRV7wxLl+9W3+Gd{RHz6^2Q3LPA4Q@%?K~>CRw^~J=aKIqzip8h3#V< z4cW#h-g9lL!MKn(^l!H3y^QtNyk2Nc(l_5Clv4gqzQq=6Yq$)A#iCEt~wHP z2ErRWW_{Fc9e%eXwi+AUK9tg$i~o+g;-{k?b(oEBHpP)Ub2|SccQ<2G^R4Giu;%9e z!9YLF8ri2_bR?VErN7vT#8H3A-UmU<8&}KigDLt$`i7p8AzmCEiiYCg&OB=%&U;w#=n7k!~_c!00g za#>cPpRw3-r5&?96I=GFm;RVzGvm7Wkh{(|(b4n`T5l)4bTOp!2GXumIQ0rlxQi#! z!9k8ooTScUvKxhz`K0umu|oB zfd{s`|L`BKm~DIFTfcRC-M{;H+hq=UbOU_tkN)WPlx{#j@Ub7eegB7l_`v_`b3Ug^ zylvriKlp>&y)Syv_Plp`r;6Fb+kf_FZ})uQ2X6oF6F+hLj`w=6?O%V{mu>I<+kgA^ z4zGM=<@l8Z^>25vM_u9d0NFb4Qua*gp-b|hkajB4W!e>LqBCmZYvV9!waI8q(cyuV zQ5W5uQbC1n^*T^)!Jq|-Hni}!i`P(PyF57Rz+d!7JXvXD#R2x)$Kaj3j{jfDL4~N#EJ`?$$Kq0LE~IFue}E_NnKj@*99kNpJL6~xCrSF z`o*eKI!m88x$+o_7j(nAa?}^$3SA5!sgmu?OQ7n?7zqzH;4nnWh)G&KDaC>gV`(*- zywHnbW`j+JJjT8MNLWS=#vWjUOZ>;UW5K9=GVCz^S=qQQ8Lk2q^Q`DOr1)OkbY=|vv1oT6$QY#`c@=@e-l&y34s4XS4eXZ7 zK=o-BM}%>Os)O*T5LaK#!08Wk3mgV6hWh*^q>WsRS-Io|!tcV9kh*9V`iYU7g@!Vl z5Ap1L7~Bi`U?B?((_4&h+7WtL0Ap0cBRr%Z;KY{!j;plUPo3t(1OzP?c!IZnN@}!6 zJVbDz6Stv>#<9mT2IitI3%2ePFeJg7YoWtNjsK`wE;515@khICvIvnW3DD@i2Ofo%V9?4xg&9G_8Twfpu}aD z4Vj+fISBFrjLL>?HtMs1qcV!`{C!B_c2~Q){g#pXmc_6S(-z#lie%)kG9`c7ntM&! znM1c6_QYLrX_NWmP&Q+LIR#$(@Pd4l$sq>FAyglJ@N`jVrLD*&w4ed)J6t{ob)!7x zTxj`#j;F1V1ARw-*M^SOxBHwiLcb}GbqT)d1J#?2mYNI4 zIGN+eb3pM2&(Sxv)v}_0=0P2OK;GQTG8W*E_GXhb2U9#%!-gqsZQZwS2=TiQOO}1` z1r6*IHi)9zHMXtW8JCRDo0wF`SPfbh@(nE6tYbZKGv0KIKK1ypzp;L0-{|>f%tMl+ zc`G!%qN}>)i(Bz_`8a$;a;Jnpu5aEGXdZJsb$CqnTp{^rfR8-7q520zPdxFs52l#w z*~ImxGWQ+L`P_TkcB4|)PkXF4Wv7U?^?!~It=il&A#D#dGS}$?N@gmQzMB2w`-U<7MZQTk_gdTDp!~+F zp{cKs{IT70V3JKrY%JTyy3dm*4Xd_LC-+5RZ_UmHSe=6lu7-*x2ryHg5sJ7 zHYA1Bx8bvpaJyoyae!Ii628J)m}+mwn>P^tJpQ9E=p=I->kL2Hz`Ub0`XAZ*;L`Q) zF}#Uqp{tWy}z5C)Eya@&S~MAtiQ}X%=7A(ap;3S3DKi*7}9;x z!T?5nRhMwg&&#?ob4uNZm|MWQ=V9CA$2H**nh4x$pPCnaaKk-sVE?CX_9@MZJ+5nPBa{BWy39AFqou=c+_*jmC{=10I!rt+pY}T(*Pe2Y zc}G5f63*oUsHV&ooq|bY=2>$`E)&)|uTlRf7r7D&_Gjj}ey9?o;B%vLFQz{wc4!ld%8&Cx5ct=K`7BNl-tO%jz7crM|N6gr()ggi^|!V=KknnU z4|vt9wik<^hPV5G57>U&fBj!?FZi6#@zYVas&4*M>dIU33d8GzMaDaED}emhIEM?b z1#8~I-Hl`M21>E4h(}Enkl(n_ayXlA!OFwM?gA#M3$(kAzS%SmXj-^Of8hYi;+$|z zzmZ5B1|E4DE3Z8Z9D)W{yYfMBkp(c)oS3*BG-0`G9q?7|fJE4P2F{5N91<_gdmdbG zp+g%_HF~L}Z@&jE}86hiT zFkwt!H~?&NdqF>pG+`NM({PHR3#1v8t<7#B1|J7jl=Y%f3xGbH>R+j4iq#B}@d)Nq{AIhiCewTpv(@812TiB?c@TDOo^309|6Ei489d zXEutQ0^K%famlOr$y@WDaJRz?Q@i1x@-e(IqVwyn`71VL+fros3K5M<09fq`JxNg^l!x1qU*qe*oygBYMK3 z%Nsej!1L4#=rlqLy$YYnadxK`}Jl@=gE+MI>ej2 zY$johw_Uc*EHl}K9ZO~06?lQq}5k9ZYkkKcMq z3V5^Tdd>5XKUSN!eudvt*|4f#QMc{l`%VduI-t`BQnod%JG{;{PviHipP9q%k#2+Z z?H&B^0-b_SeN0vtWZ3sW^aH*vBY%>o@88kEl4Z$Y-?3?bVD%TM46&9ee7FzgCyd{2 z@)+Fo)IYF33Umg(!*54vv8S@x=hyXNzgWI?Ahg&o)Yv?9+w=`oy|xF~FvN~_9Er*Y z|0NT*DMgZ>O-A3>t52)m^)Sk9F_as^1@Sj8k&OcR2JjFjllcx$@Z)>2^u>B29qBcN z52KiG6wV=d<|)c`Ww6mQw=6-mAH9s7Qk$%`@!A{7t+Od>Qf@}mT3ct9?ZFT4RlOr8 zB+ou_A6lt4SWM+PQT7C@3;gk>NBnR4$+>tT! zV~@qV>W2@^qkVtqC)4PwFp_$06PE5;f0-|+TY8Mld7?50VqJd@f3og-GKW!Ybc@E= z*yp1vdam*qx6Tq4qAS9lTYMiCpP9d=o`hHQd&;EOo;eVU|0&I9*89@;@x+H^4KEYB z5BLDBB|~SrV+76$Qq1KE$6Wy$u5Zw2B+(b=bGf3GabH++8DV4c8XJ}{>B?0RL4P_O zbf%fh*AYlefcTX4yktz@^?YN$^K`ki(S1|8!`m&qhU~dVm@!)G;*tZWAL;8L82Ut@ZpthKN+aFrHa zWIaW@X)}j#+*|vgXg@!ZHs)_ib~{B2ny!cOsJQ)HxXLsD~~@`z6rlq%6lX5i@)#-+mk=?BNe)Fc+2ZvxBcSJ{LJ>wzvFjozyA09 zp6z}9#Glxn{~LaTpN{(R&;R`G*#{ojuKv_dZBJ-%_u>!z(Cq_1>$A3Z|6RXpd+m?? z*!J^Z{^i^D=%d?{x&go2M}Fk?zJKyhZm)mnq3!V>_<`-__kaKGyGpn~_=(&5eD-H=_r1q^Y(Moa-?F{-ul`jD z`jn)9IiNcBTs=|3b?MNFe1|r8VmW4VF)kH9+&IVVfb&Cpm$nAv?;x&mN$*LiGdXQd zZ?`<78Rc2BI^b}-c$W-|f8LA*!@tADa;Sj;cZ(PL-ZWryr;IA{0=|lWnqYs?h!>g|oaGnY zUPRbfvzv4bEYsAt4H$w3h?l5H5C2^cSMuYeoEOW(IGRQhpOn(og^LEXgyAJFhO7;} zWrM2S7+0ywzgT0@UiAdRY;RUX2C7Nu8uj&}4Z{-NM5FUi2D)v4V>A*rO_(*qah)G8 zJSCQZU+d=jMkGMaCBk&z-=PFoDB2Uw^q;4)-6!dn{skF} zSK@mSPCoc{pXuwv43(!2FN$g`v1q=fbZfMy7ccT#x1fi9$?adHv7q;2q}qcHsT#Ki z-60NLgirJ;ZDsL|+-F^>Pj#_)*Fua91~1UGXwO2abVy$>+H2wB`l#Ovj?nKqJb#oQ z*PEEt*R;)xJ~oh4rw^B*qr5(VWy696k=p3Z6Q%DKL&%sm6#p5Esb}Q@e({+$va|f~ zgXx#vNVopnlFdxoY?rh`%d6bfx6}o~JkT9*go>kH@=P951Rh0gCo+ntM15?N8AJ=n7Hu=5b2dYed1s(Z# zp?*9=_ZiF8lJK$*|EW9`YP`YHrhBgC0hSOhp-O=$;W|*{h?zX=p&blZj;{| zrw-G9x5=Nw6aE;UHs!!1vZZ|gNC5d-#>mk+aC6DwP9LJ=HC2r@UXAn*GNw)Ta#_zwT@;|&?P(Vg(^{-S)z(Q^#rPxm0k=m61le^;MVE;5L|Mx`_6nuqjwl|#1b zcgu-36@RPVh97hy?a^p44{VkoFTY~ZVEFnpx#anaStTK!#Pz!@d6&O74- zRWEZ{Bm5ZPOV0Q|JYnll^U`tV)wIKt(fN|suQFmhVfU#|>6d;5^PQ3}`mCEq>b-h( zd%T{kc;t~swui4gx?RzF`O(Lf2fJtLdB$S`$6m+^t_-gOt7-5HbAahKe?s7qso z!zZ3%y{*PJ{lP{xPtkKv;$Fmkjys@rS32WEaL)%ROMR5N-1-9FRgSh~ z{J%MEtSsKdkAh8JBXAj)nJ>V|qjHE& z-)HF`T>1z7fL_bbd`dt2z|gK}bsqq(i7IgKm9`|7odOH81kDRT>YlX_YXwu{v!qjV5k z{3+M}%=W~SISk|ftFCC?@c&pn`nX+mo&I4iPM=sxrtkHX+STn0Mh?i-azJOK599tQ z9q|~iu}2wn0ov#`|AFWKG%lR459CKwX@lol>Q_Ch9(X=4bdS?H0_&d2xLm#km7!_5 zQGDYzto`P2xgo@b%&D{K^cpDq6;GhktH?FL{45y>u2JzP;A>|{pkw1x{KV#|=0$XM z$gsyOB}-pQ@9Co;;kq9?W647x(k{;!?RL8!>4?W&A^qPNJ1z4+I^ucQ^GIw^@*_tY zLVe+>=VrOmN#z@~+#z7QweT_}z zKwd=Pc6J+~%iPagAlKsq!8C1nWbT%}`@Ui~2U2|iMH@19m0#o4_e|GCc>4X3>n@m1 zkc)iDKf~=_KSYmC!kMJdJ#$T^JO2tZ%Gw(54ldwf2WM%Y1<@*UN2()yOqb`Fmbe6b zJ(EFz5*Bpc^I_$E@n2Z%*xqD5C54rf0eBB+mUY3oHq4stzW=`Mrk8*8$2|M$|L`9Y zqqI&w(dXMQz43MBoAA{dT>r7)ru*;TZhy&3wyWRyofUHyZhzU!wx|Bj|8sj*M)nR^ z$b7^{Y)^dqxAQI_N8QTPO`2$){(&EG_(tG)FMjd%mS6nE?N;&g^bh{v_EA6kv)ilX z{@5SKbXax474mVFl;c+ zve1d4e~T7`Z2V*qXv0S|yvb46h9|YzkdY4qXKYZ&2=W3ziw+h;JVo)O7e-I`6$!F} z@;^C}LF&umvlkcqX*-4^Mx*OX#DaRS?6F#^$Vf($d$*ohaP&wwEI8~OrJ)t z{bZioa*OFa|Gs;-`|iDayXUUEwimqM?Y9@a@CDo3zx~_!36JrLN4dz7P0fcM4l*CO zy}ZU#dO-Ve(PiYte8k*DojX_-QvHtf{5fwHb1e91Q#QKx*qa&FBhigMMPFNg5u>XS zGOu})2b#5U(Th1;j}7w5^F6-KMl<`(k355%k{>l%0H#@=(X4%Z6JTYCgpbV!Rxy&)&C>8yz5Y(J9i09 z;6FlFo4iqL+Trgh>C2tcGym6<=peK36WO?5`N;KVgJjoFBG4{2zCr#Q4jwq*Vs3zj z+oyPXxHnFq!3JUnFRnX!ggoezr;um2MPY#diQ;v7ywc45 zNm=)0Z8!>W=c{cZUuZpf^@;6CwE-FU|EA>9zq!w4&deAR>I?7Lq>UUNd-TfosK(`$ zhab}Y`ElzCZAITRCQ_buabSx6+$Q?gG4+FW#`L6Paw#9V$Id~Y$Nym6$w6y$o5M07 znh7O?PDhm(Dl>S4nC`bNhEa%{(S03FS zQ9B=g@Iko`=P3fwVczjPjE>Zpr3Ll1EPWWHF*|JpbUtlNUivz7h{qx_t9b_`&iC1# zHx?w`bB*doub9vLM7jFg_nOi-=AGCq{D>$z-n`&uOva8Q&i!E4;_rE!`>F4Nx-VXq zd*A)c;XIXi_jZ^1>lVdJ(C(jSC95~S{*Bvfe(AN_0}n}dk6hUvU=EktcE(k?<{d3n zTNumyubmIqbuYSK_lWyd$G!L5BfNXNm!}!kw>Ajq6lB zU|y{z{bVdY@@S3oAalXZo;OvO>diq{Chy z^w6Hdl+xHOWO#Kr?cc@N21dYYrX)@i}+xUXYJ$hDtSNyuso@v+U$O@44h)O*QvJe zB#W#Y!>iu{P<9C5O?jhuLceRS_m9jN8}x_!cp-aA#6!kz?*D`3kCiO9|Wp3*+(b)3}{+X}lktQmL zPKSr)$@gBl<{6%Zl$p9&_gv9KA94H8zxhiQx%bD84E>DxjQcU)sy^}nZoOH|{9{^j ziG(IT^agn~f6Z^`)%dJB{o}osE#>Hg>IZ*Rq4n?xRo4;7>GFkv&^ZOB(S(k9Mg@nO z#%q-WqaHc%T;KJwdIj;E$NzOfXFNsZ6r{tiT_c{zx{8wr+@M%E0+L%`vCA)1)R~Si(!bWG{|23Q-;G|v@+0E z!tyi7aO|(RBS44fmE0QF<#1g|`A^0H_(x#!*{}dw9t_ek}yn}}7{}wH%+8?-S`wj2$o@Kzrh_k`QBFcsv z3m&;L^4rKFFXc?N=)mf_m;~9hYNL}fUKEgV$wzS;Oo(|PN;Y)3yoj{ z2B62HzcGAb+{PHoq9{fq>8jPV^o0$sT*9wt#mxbgvFjvW*gUtv)$|(&P{-&?=!SOJ z=dzT6j@-^e9X3p5l;%%=pHe)sRo@hUq3ugt6AWeQ8@%!<*IAHFn6EZ~X@`7q^$khr zJ9+@gp%mz(DmKAIQ$9S9PJ4rF(h#TCShm(V7Sz(YHl$Cu=}Nb)_q9j?y@*gch(4j0K7`a_)tf?3v%r5c8#JQj_CX)& zI0E;Z8XKMBu#GkG zPncZdsg&EzH@ZpNN%x^{EymaYc6-^JsKpeEC)#6q(JyMRb$i9Z0M*y{qE9SGN7~I( z3M?95$=sZ|8{~XdNn3ztYXO%6gMS z*qa@>Z>H_&eGY}IZq?~FShghxRUAEzf}rPVgAh5G*Xk1?bbCY6KYnN$X1~Y&Gj6Y+ z7*jj&XAYulHeZ`(w_OB~Bck+yC3B89ET5^Kk8Jwh4pP5)D|xyPsvp1+7WO@nHlw@N zH_rQ|=Ew zq{_i&&!gz7=R1?@Hbltulx5)gBK{sv_B)fWBd+Dvjs`l=pd(}r~BwCG>*Sr^b*Teu>*j!IP za%Fq)!H2dty#9^;F#$g!;Ku}4xhFDz7n;v%0Xpr&gu*-T3T^*bLw&=OQfz+TroQFJ zD)8e&HnoMm=b)G$g7m#O^tW*!MIT!RmVY4ge|SJwGj4bi44Z>-lyN+4UAf5G_Z{7T z#(m!?KVjX6*|f$#`v93D&)i4mn8w!WC#8@t_41@t&ja;YGq-UN8U)Y2>T&!32l}_>hQS3vJ2l+vZaoMkT z^?{VobEj+&-?uDx#$|2dtK?_{byYjezvrwRN^wZdQ&P+o9I*0~)Grol4(LX+up2Vg_=HgZP;ghkGgSq@D zlK(X3fSE%&^n)KYA$Mh^gWSjAfx4)7V(Wu5kzEz+(6JCG8pihHIw=5ic4m19*cGTm z+)-Xw^|vgBzvG22=AtRT2NR2Lc)1idjgE092L(#wN>nZtn=qJh6W8mh(pUH+KVu5u z{_1gx?s(41yydxSnRftJZEoYftB=(vTxIn8&($8YZ$*pVe5JXzNPj_p2{>Ett{l<^e#U7P|lT)q*v zq)cx?ZC~&O+vT@=yX|}bhyP)F{Ga~Q%JHia78SvJR%5OK z(v7n^Rjxc3AWVckDH69LpgK`x-+;JrXd>QW-;h_{pxg9KtK)QY8*zn$Ay-shc~qAX ze`CWaUw&?yjnvU;TvnRxn(m%{8#}4D^CWEmUI&LqU?ygL4&r*Mj=$X{ErC-|bk}-; zL-2;dJ;r}@g$q)6*>WBa~`>7t8u_uu60ELbVY{2vlzmP%Q$dO zl^20V>qCd77iTLFOs1UD7rskY#u&&GAbxU526_LfD+Uq2rso&r7+x^r4@{YS`0F?u zlWGHojrk`r8O<(>^j?(n6vd_6`Og4;se0M}7r=jf@B{^mq1!Lje@%&D76q<>MH)IU?5KL~5C*Q0|AA*Q5d|d@22)%YYByhGRdWIQ)ALG zf>}6aQzB33upz^SnT=tk??82W`6d#OG$jQwlr47|#NLPjG01H!#}JRPoCRhK@|thp zU2fx8d>L`3D<{YWa1!||UFgNa%`uiSg!>nv^re4EFNApvYi~4aLCN9)z%FNWs`J<` z0xKR`M`)3?kL%io-Z-^R)28m9D!5Xlz4WCQ{|^63 zoQ-pD_GqC7dgGxMa~iXz$s*2+tkC`cv-jsg-(^`{Cwj7SelxR1#1Mfh1_1>bY~(6~ zjg%tQWe7^E421!=Tf`O=I>KnUEgC{!p(E6adueFHKX5>}h_shMgtqB0Zitj3iXm#) zh*z$~BvqO3oSAP9mG}Gke%IQ2pY!`Y&--RlbVEe1-?Pr1*IIk+wfEV>Ip;yP2ol=a z1PQv0HGbT=^Sn30`l(!={Psq-(kV`R_t>zrP4VbO;?MyIH_^9Fi#y15*{hUoHjjg;JqweFp zY`n7hNPm9tLHaifaOtsV8Of{5gU(Cek8Z8shpO)@e^_NhO2ynqX%o}M1|BAVm66+v z>ud}~*2p~6y!fk5@KN6AcP&%spYp59HIli$mZ#+sa=@GVE%mfNdNh60EB?@uoq(~I z#b4n-c&ca$8fF^$2@+TCf;*bVa$pgU<3-)=BzW85IU#++dJNB&^)(BEdd z5?m+Wr-U&PUrI*Rw`jUAsc&ao&e#kZq$dsLn+9=jB0v4K#`9<7GC$Egg||J{4e5;2 z(E;-@AKq#V;aGqfjeM}g3Hzr>!Ye{BTzhHCVI9EcGYz%iDLv<-A}_{AKB zw>LN_m2R{qm*1w^mX3Hno85jMS&g}nVnNSF1TNxWRv<#LY&;O`As$?Zd6uoqvF7m$XGTdo`E#dLnCz zyiHklpu7<(_5iqjZ48mm-bkD9@CWI8o_8j32P0VJ?6tFGOhm!-;|q_t^w0Hklb$Ql zjzTh}-aMR@xo)qY`V)|#-xO22ZW7k>V_J8;`qhtbukW`bd1LYM{12%d04|3+sv}Z0 zANQ;3_ucmi`yi_3e!Si2x1+A*?I;eHdD{@VEQ|8SzFb-L;kF0r8#j7w?bW)n#}wwK zwhhLW*a!KMv1LII{nucR3jN2R}l&2M^KmegXpPs z)?-V?ry7@)m#W1+IV`Ppn9%oUkIjrz8gsA*bYiZsDxT)p;U51oW{h>jY7aGLtm~Tb zzXusNk=cY>HsoTyu%N=8cPRth&Chdb(dJ&p12S4i&4;J5{<0maodAzZJuZ7p(zq_$ z59Uoh`YZPj@|usz`##5ky!t2);_x6(?w_+yqG!u)oo38yz4rZX=oMR4=GYkeZ4I{@ zh)Sa4*8TJc(ODzE{B@fAxbX_`jDTSJ;k@DsC1#K?Kjaq5jN(?GnV5KcD7E^}!IJF= zU22dRkd($mTD-}l|#@yVaO{ruN{Essi7 z>MuiR``cUF#c)yccf31}usaa02FhUk?&v#0V(tO%E_f6Y+Fm3VeTUchWE)@-x^N>g zy$;L#J4`0i5VqW=cL6qx;_@76H&)sh^cy?x3WwkTvxGNJWQgO05f^`)ooo&ACv*{` zR|qD@ zEZ2)U7M8XZO`LNm#Rgl{>gQP3YSA3gZ0!HPuC+0&n)of zX5&mB9Po%Q^<|OXi!v5PlqNiaym0=C%P*PzQ+(_rH!FSq z*Ot=V_T;U0&v>D7pk(g4rLHBnwOoGTsF$$EfG+FuyK<4kSaGele&cXp^#}a66YH}3 zhsre_>oBhU@M(Wz0kz04yhpmw_ZrU^39t9)#lIl;Rt+dd%6g;Ni(kfu<1j_`q0Jg zRXR_b^FvM{`CCSX`2i`r1d|omKQ`z7& z%@+$tow?sA5BcQx;g#YoL(v~H>u1|8d&@YSIO_nH^mh3i)`{TSIK>u4QaKj=$HA3zT|*TjX~UBe9)kKipLbux9rM`SSp;p<3K>-Sw{M} z$0leoPpFN?pvE`Tl2w{N7<+^k_U1VNPoJ-ITZk8r_$sP#2Z^x$?JL^!v!d_0iEVCf z0@~g!hYL+VWh-nN!pw6tKdQ}2A4vHj8qq9z$k8@H8_%~^Ld&vLKgk0)C6oIo9BJ>g zHSJD?XF2J}{fmB>ITJj49&srQeI4?3@LYMbj}3YpmrS&;ABd^Bhq2y>1FC2yYReaOJj(V=^`qp8T-;h$DMGx2mxtR;_P!bObzUDRg zeC88ckM$c;Y__xc-a3Z|YDyn(o1eDs`JdeU|ET9QMm(F(Kwr<>mDirSCjX6mW?J&} zr^oDM4${X7Q39exxoHaCXpt0i6OXxSK z^zaSu0xPX9wmaD z0m>II2leZ=frsb8YIlzt9w)drFmA-no3TBQ%^D*QuZAuFKihJR!Tj3+=0(^i@r*65 zBlk(s2>n_&3H>=?A#>*b|E;dOexaT5$m6^0z8^ebJ<3?m{D=Doc7#ZjM`j%@in!fc459yWuAHC{cKs~uxfp9|`6DDCNXBf@Sl&YIrwxGwA{yL}m` zOD7%y0SXbxxc)5blICzBeQ*u0`Al3eJog69&ZWwU>hhaOBd&PAe8{*rpaK_Y$*|{= z(A_yJhlz68Q>W}Ke}>Y?KFhVQsm+uX?k?31&So#lhqTN9E`-i4_Z2nD{|UeBz*ctN z%w9fV6J;CrmS2P8@ULOJ{u7i8EsqBIQ&jv;)X?()VR|4m3kUIb1wB|Ai{^5}nrQ?m zx5&DayJ+H8))eaUg-u(32`}}jJUHz)w7Y{f&WwvE?jCmo=J1!>WR5(JEYHgDG651M zGdvSs5&5;yQpdA^F3Niu43ZZK1d+zOLFL2o&69}%VIel7$%1Xv?WAzil&+Z!PrCpU zKhRKa6&tW*@r1!ZCQS^*i^Ulm#W!!}pUv_~2fvl$1CZyv(bRSx+mXvv7s6hcfU3ib zFUB2xPLPFWHb)=QB#oz8{V5Jjs(4GI|Cw(7i7x+pga7}Lw_{i^KFxxv7ZP<~#v;ZO zH*{EhhAzYlePt5d3yIbn^$+s1KYs&?TA*p~V`qUP5#Ak*rvr(J+kPNA+|v)DOD2s> zTG{CFlYLs$@nkZGQp9KR))3m>ihK!|c1j8xJ)W%B0yc~MYz+0poV1=eTW9FczrZ^2 z3=$c{Z>*?>&Nomry~!r}$j8LkPp*1W&PE{{hR<*Jsa|Y?^um4waN>wX-h+_1%V7jVoW z+A@~*;?B6P1O3PT(qV@wBah28VcIW?bhY!tbK@rc6R}!G4jvv*U*u4#4y5Gv6Ukn{ ziALKDi=S?jacG3CpkMu(E~G*DaiGkC=ou~geJI6~vix5dp*KX-e-YoCp0Ytco5VuW z{IE;aV=U-KyzH&qrOLFsMa3@6bUcJbM40Au%HFQkK=o{h47ddf|?iZ(y|cgA_->u}}IxD?1SowYE#D>>EHn8M+yh%88 zDYKSWe$TR0w94o5ql|$1NjE+$YZ-hmU;{pfOE0{z92!Aebg4UGA24YQp9fOG z7kE=j<=&%o4yO7|sokMeBXuTTd8|*@x8-e_!L`qq-7DPb!*$uZx_rXcmube|!Q}jB&c=nfwr}^LY#sW^4*F4)cHyfIgWU-o9~j zyM2?f{PuQJqHZW;3Zg>u&`-tx|j4M6XD;+s0^CI>{S))Zf zL#H;PIhc~`d!FfOY-&0sgAw@^rmbyjbWPY)(-FkG(?;Iunm%?}wh3+17pBjW zR&pQ+XeV+JN!px8zvvCc{BNmjSbDBha+8m8ZtrTVey9klKG+f+=1AV~rBA6Wa~;}; z=0|wHDJ6Y*j%+dDulA!H8<{HWgX*e7`nC##knr^sh(ArET>Hu+=)Bge zwza*)^&EtI5$&w}K=fbwV~nu<#ASY%IbR;|@_ZWfL2BmH%nMzwk}I;8WciDH_O(Br z?yKP1C(pqeUzFMt8g>)jfg=|_YB&0f=V8`Wxpgo}RcJ5XxMP0J8$o_k9poUH!yxGb zTY@t42p3!f=pTjsmXtpWTRe~t`6&wx`klvj#&yPbkMq2_#{&qA zxtT{%H_zXZz3KD+e;>)`qcaZf`{bZsdVTIr=~jNuK;ir$Szr@fa9-I#S(S>l|V zi^DTmaVLGX;s+?a=5Kxt4mY%4?(iP`cH`{A#twHx`}g1*fj#3g^NsAoQ*cZNX229j zaF=NhgZClN8984OTqMKY$X}XHVmI_X5V|Z2X-*%W3!u_2%Q9e}sWe<5zk@SwpAHP?7C-> zV<~VgUna3kBAa$-vWThj&~-|Ha=nQGoyq4_o-i_5_1j@WGCQvp&LX2cl1uVI6FS6Y zaT33U!_Uug9k~EBiL)R{#Y`VME-k>O#bW$gn+2G$i!3@p#(_Xd#7lsC#ooV}Q zEeM=PVd)+i3q9p+9a6m8Xw@4?n?cu$<2oQ^vAIsfT{&DqAAV3a3yk2pn7}n1m1nV# z1qFv`9ICManuWNZP?HO7WPlOXwGI^W8(>;Q^n%GYR`h#=!F;u_O4~q!y4NQMYmvbx z2>moO8!7NB`T#|&d(lQ)$l;9{<$Hw13=4Mg&Qr1ZhwG2=cH&sDJzzbLa&n_PEz)YE z(e*4{z_%9yK>BrLaC;Sc;l={0bxnP(YyOWI2e8@1;~*<;>e}s=jn4NJ{n_`MPwOCY z-6vxQv>4LgNnw`@?>;=jtUk=neFBS2Yn zN&ArhIrG9bvgxmKy#emJtol;iumSngjx37vFoB|zwkyXXTy}x2$UdlV*CBPn_GO1< z7yUL38wMO;L@o{+p0{m9ms+q>8`EhT*_`zu4Rtb&pmkaLLw@ZiNVfm%U z$e;do>M*H?D72 zuU+^5iSZatpA_=eJYz3%!lTD}K0lQIQN#a^;(yfSv$1(YcRmaOQh%#J7wn16Ykw|K z{5`Ix_O1tGR}SWN@3f8Q9+fde`Rr!QDXK2x4R=0?#D<^wLg#_-R391rj5<-b4wp^2 zuX1nH9DqFT<0@m#mr&%-nRix8xi|ISGUvDa-o3@L_ofe&M&h!V@zwiIpz&wy*p2O>O!C_Z?`fJ@_n_`-gbg1tr^@duPmYKSz${Zyh03Y?J!BZ^<@W-ym@r^PJVW zBQJd?ZjWhT%RwKFt;cS1j}*nAxh8P7S^F;avv(poI&$y#nV2>hc4-x{{OGl?i@s7+wK?Q@3DlovjmYL zJmD2N;v!t~xc>Z*yKpVVC%T;y~{dryF9 zk4uo`PsQeXW{_(_LZ8LOdS!W-ioa;v4Wd4M$7kcc$)}7vc$fUX2 z4UgP&Jzmfkumjnr`$Ip}W_u{V$BODJqYole^a#MI`P+^eCuP6ACh?rHhDp2Vw<{jw0U*~~Jlw~jtvWE~^}SIz#wVrQz6`M$>ymlhfe&%ktz}?7 zL;jQ-fPURV_6CVV=L4g&i%=K-Qm|;)wP&W`cG?lz@{e#D+jX$l>29MnBmQDE_`$!t zhDDYm@20;axz8e4m^3Uj?M4B^-)VP)neQG-C`)qHbywt2{t#?9soHxRPQ*~eg|b{A z%}e_NuYTQYxBEW*Gv4yjPyf50tf(`vwA#s{r$J6s^!3EY$7?peFXj3-*#!^qyA z3p4ECuEj2Zi?dxq;2vn-6Z^FzT?pq=FVVrg96)sPmk=Gd`OD1!-;zFJn&@V9qjzX zCnOl21(s@b9sOs&Bc~tO3uZ9Scuw3E_Mw*vyTS>KsFy+bBNP4lAyK-XN!vk@ znmqR07A@YDR_BenX#*z1HQ{PIFV_=00zFZm+SvE^B5ZkS)j0i?M1vl<)FzC6TF&;Sw>Lw=A;z)iSN2(5gD1;U}3>f zix^=RQ1uiei$imwO+LK{(xQwBttWUY5YWd+4-Vwkw(UED^=WhhamZwou1{&zndBux` z(+w@!u4@5z4!T`um&Ko)|xf1r_xGA&HP!76G*wKqp?53Hisvd74TK zHvEe7?jwc+*gfgQbUMaS|HHl!GQdw}XyxnSk@A9Bm3E zKdt5oc5g5V)pooc(i;|jyT@<&q#U$a6uVy3u@}KE)3?Lp{)+9Ao2>O_s-e5Ol@u=#4+M zUE5mfQ+2?$%2uJv6Rz;YE?l?b$6|^GPTNP<_R6IVk(H-IdHc~%8mo`+i6j1Bz#HE1 z+U<3(eU1Gs2QIb^eQ)HbZuG}|elJhj^2XjR-mFypeNa|*gW`KL)KB-BUi#4MUh~BE zdQBi-_nKF4uX**Wx7WS?wcBgN`!&)xc1hdte@Wour|(6ZCu@0{l?^&R{l_N9SQwQp z&3S;UUs}i;mOSCele)-Ae-FMWxp+e?Zwsd0ENWwe?*Baoo0n+f7iHSisCuF!FZ@Ik z`^-iXeJqQ=)yCUy$HKeYR_+S1lWY<*-uP1;*uz7zZN^^ghH>xVN26!P3ymvm3gv() zoA%Jj1_2w78XJV(s8Jl_P5-}=K9r*GV`qFil`*Bq6~=bTs-S6u{?v%a<$UJV2REY) zq$gx*tb0Sd9wVV4evEyiu>{U%nHd8f(ir!+>U_m@&gWG!KEh*wblV3!xX9;&1|Kfm zaecU7Wn+zhhM%`C!NFYgXd7>=)#jQv?o>yRwnzVrNo;nk^#OV_A>$-&Y(t*1f%llx z13Wb=or2hvfx0M*bGrSLgV2_B0u9|>W@vP53Df9>~7mWqVsHMvYs_AQ%177h!ke>Q`3&#(Ml7P`|WS`Kugd#uv4S(SEdoq^b7o>Y$QM31@m zWK%wM@O`i90C=3Bze%R}rB~ar+88xU$Bkn%O>U1pe14Pgn9CLY=s32-xY6SZpTh$0 z2rZlIm~@Y6-iSjV=}+dV_JO};E4Me!XghS#8|VDLPsSL}Gt_t9@VeJ%%+CM&e2?tu z=Jgxf^{4p!*46DP`QLr@>h>Ss{nU0_;{i5eME8;>pL2f6b48x+cE8p*_B#G&4fon7 zcsu?v|8G%vf$(iK$mY-k?f&GGPj0{P3-8)~{%3!Fdm7tR9|7Ux`2%*KKAy2Z_elI6 zccmxXxq*1U>h$>bhWB}c?nSTjy~zJp@!`xV=^4+;-aP-rF5DL~hrmXj(f#!4r*Ci1 zu(?iqh}W}1n$2U3b(@D!x~-`@8~#`74HWz!e%h9CHD3dG+W&Ls+SO~@Q_?f{Lu`}# zBlqL9UwEL~-h{>eZJG!n4?4h>J(e-%x=l3y6Vm>ShdiKxP4s3a{U!&s%$ejq&jD@O zpY(SGcLR9xGuy>}omQ@ZK5fO~7tih=vC-HiZJ0v_^!A{}|IE$&xhmNYV_D`)^!3of zg<+CSU?MWJL@7wR_T z0VQAOBWJN6un(>wsM;~aKhuX79_3xZvHsJq#x;6R_5cpT{`aZ`(8dH?HPyE(8DYq(3y> zAn~!imJgXtmwKtr?juXz3opi7t+5fJ=WDq=V%?`2>#j+eq+BW^<-kt)Xoi1$Sb?#IEO17!Tch(G*FsDVu?a@w=Fo%)M}9jSN@!Fz(c zXRREEj0x&`83EENOR}A2-o>9LpMhg0CN%N2J80(5?a->nEpLEA)jnN*!feYf`^FD= zql4ZU(T|B3lN}~%o>(bNnimD?YfQr6?+F?cGQvgm$gcTkQb(?Gg^u$?cGC3X1HWi1 z5P3Yo$VpkrM}yX1!+GN~NVP2ZYE-=*WwjxLx$-nql%l0%T^Tu*Ph)&k1Z%XK}aFE5lu zR~KT*QC9%DB31JzF4D@i2R-Aj*!J_ciw_#iplh4p;L)x;Ze!#u7a)hlC}^9ruB8Y5 z2N~sIWjeMo@%98i8&%LUef_$v*jzI`+N0#g-;0eQJM~ss%S2Fgnf$ZZ?}HR?Kwt-Q zq2on>QEkD-hLHtsCj8lys5ilj5%Q(q#HM;f1(&cFOYjhmg_PPdfU@(yw6a-|Lqz0o zom@wB0Xx-4VRT|zY*yHwtRTDI7!ktb(MKQl!CN*1*{n`o+m70v?Yf^upF@}Gcj<@3 zMK{zLb`938B9O@&30hSB?cgC7fST#M0UR}4awPWxod>xq&9-hnLdnqMsqLAC zu63x4@Ts<70h+^?eDcGeH1*T;{PS`SBeW1DoedZc73A-OBtLck?9hEuat+sd4eTlAF8mp_hy3PTDF1kZW%yyA=$#G+qE8)unt5A{z_{( zTaMJd>QV<F!Mdvxl zj833gV|=#_HllpCb=!S5A>9{DD>i{$^LDErE>K@!vzCJgzeRxVsITR3LD?vZtgfdn z^iA7q_1m^r`#WxlALum3m5enHy1z3nW}X6sSKPiDQf1547=6#gF^1Q82A|A9+F$&8 zjB&o?^*uvr+&6rW@>A!!Pxz3AP4$P|2DfhB^kK=9@4mX-xOK~KL2w}T9QG}}JzwKB zVp%lRflq+?N$*D<_95{Ty6-%Gg*T=0u+t;;kKw%$E2CG6#TZ4Y<>f@2ZuR4;KM(S=RTNv^MFF$lp=rbKe4->d-^_a zo6Lbh$2*_wvFXDMo%X#Y_kQYE^Nc#EfFCk@QxW@>E%J%6n>TN5*LbT$b@pe5i~czD z;{JtS=sA}7y57j*z94!Y-{{jC3uqC^WE%kQk6jYoeaQXIpQ}p0p>Fgs+nQ@;)$GAl z-N(B6?J>ZsyaM+J^5{Z;;a+DwNf$nlD)hn0@`g9}Y}>Nx+cIGS)1Q8kHsOFxdXQdy zpHx5Rp%?7fb<`iK*i`9D{94z@={dbC5V!3S)WwLOES_hWexVPy zs7Kz2Pj-!)jAb>R(T=?~z#V-Q8Ijkzh#grL{ir|Xu_M~v2b@)J{MeT5Q#7d;hfkjW z)d6Y$|0wz>2U50AjRQSzb7ce`cSXZvuaGzBjnX&kmGO4ejaz=O?%LJs{y$Q;>R^ib z3icVh(ww#Iz*ub??6`7;=)hx>>hHro#wNxV$?I`cc`T#!=lcQeFZ<7-a_$S*gzqKP zLl}U*ujaS*50CXBE8V578m;tPQkMKJZ@G#dKzk1w=lom>nx09NOoOhma*?5#EW8$P zL1LVHkt_a6n_kD;TaoL}f5d^vCSjp^fk}z>9-IS!piqi#tC88bD0nA=Ni0KiYBp&GaEB@GKA4_RqSna2jLSG27?qq!DXR(bi?3 z8>pDmgo1mi*%^4b#TNb-it;jmH>J*ezHe66dS8%3sf%FE`pXi|33d|qUX=a_OT|neCU$a~HsD;jio>0lo51ddUGZ^U!asvL5CaVT zs5vlDk3hi;%EzAxg`Vlw4YK)hLvcpq?4|fj-|>4uCg^n<{tm}aU`?z1i8A0F+>*(0 zOO_o_bLTjdVcZVvgwBp$`MH!Yf#Jl@wecLR=_~GFL?-1V{Tz%kfy>Iw7O7k(<#b)* z&I0Kp+xg5xPv?`D#tS&8OGn#u28f#Z;RDC%xQHp>68-^i`hYiOks*FeWLRLc z3C;xPJuhfdQ>aoA$ZM>A+5Ht%*poRU{i@>SM>^#4;sKY1 zh9-4vF!cs48<3Vqay&NENN!K?s+{u)anZG(yww7Vg_IZAswZ`4QfU?zqTVoIg3RL7gd<-I zGXAkY2!HFi^Kd1Fwhvk!qJ^m=p7M9X5ugii{=zoA9F;WdfHsvLlCLwTUA+* zwj=GJdcxlmZRJHS)37)%WrL>BF(vOPV;e@_P9Cl$P&#>7=wxBj3lA1Q*))QVH#MQ7 zN&kGRwKkkR3ml^-|(gvnumVo0d4Ay$AUCEbD-oP+O9m7zvwq|AlovK zP63LdXY}CB7B(C2+{uP|Z>lTY2Q+N3=iq{aD>mNyKq_y7urTJ3#+w78i;x6-t7*9WqMrqQ*yK9qV2g6liM?0zULxU&Op4vO-lu)oR&5E?n8n6 zU#N_Cl=ZL1qCP0_lkch@Z+9>@BfrNGXc@W3~@g z7gM*D+HISk{&JPk^{epg<9yO9V-83h*$nLpT(%%;%=ifA4GR=b8uI4gziIC0F2!4M zJlv6SZr*o{N(Z_B^g|PD{yxfqfbNOBg@ujq>8&T9e9He5#s^a95SwAlE*;Z;mRItl z0}g-k_S3_@-#qd7m3l)eZw&i?8Or5e;D;u}`&E)-fQZp9eXzhd=FK+Zr3Zd_hz0fK z-j&TbY>~cS{a@{iPPkSWd zTKATrSJe-fI(h>VztW(OemzdJsqeS2Z*5nvUe7@aGN?YNHEo$TrtPpTc%rAsmcy8g zQQ3^AA7ERI)!gG_b7(8}<8qYo_(0!DeKLl?m5f3POvi4s-vg1Ijy@6#*zdT+uk)a6 zGV`^D%nka`0c0)~xg$Gb&3kWTOkdC(qI8hAWAb*p-x#k0kr$rN*a~mvL_KHnyswZx zVqN=woC6W-TiAUYq>|0IWzc1W)Be@ZK=;!eCLugE;MaDGT$!sD4Po0*>?k(c2#}BR z^gkcEtFC^#oc_WXhYd+5h)bQQD{t*Tc18C882yp2FZ5S2azG1h))v&=1tm)NIsWIH z`L*q9oW@@Kb`)<$$#$=CAa(7!$BTFW!jt~&CI?d7W2mDE3OxTQU7n!z+-<=8Ieo|n z53&gkA>5}FMxW6`?70u5km*_8`ok7#oX_}yjuC*r@bfq=o1o6ni(E+uXeSDBpR?Do zK*~EWG=_atvLIzCjt?69FntkV`ginIdCo7kw^vDK}vsOlaiCv3K>`YZ&oz zo#w#A!M|j)jOa_4HcL6G)VA4c7SE*^TYMm;HUQBDq)k8NT~E`+6>X0b8Be>f_Ti*$ zqHO^isd3ykT>TE32#MZ)y8^Zc? zvsRs5Px=P*8INUuxas4}XLul0JVeX)T*f}uMt(E29?}5e?eVYlq2JJJ(E@${<$gz4 zX{&yHue5MhwP4XX6cJyS*ct7J{de1vp0>3ul215n(dj{5nyBfXIF;7dN}8~XI7yxI zk9r7J{{lk{fxQ*pQ(gB%>Z;2K4L{JnmBLH~T%-KylKNRMh9zwW&b8nkRt|e^OYY*u z<&?tNNW-2Up$jtZKzWXWTmU<27m)7gdnBoRDRIG}6p#!@ewQ6e0XH@AawmOnp%)`T zI`c0IE>K>qyzvMZ6%t&MG~i60ebgZX!F!;4ah4*`bPmA;GSJLUC0OFlp)Sehtoyiz zPMh(LF?N*G70!To4wI)RsVUQORbJ?U({yLv9z;p!$6sCcFnY=CF3Fm_N_9{`SE#NArqIWc#e}SV8 zcoAHd2Dto6=!j=vL{o5)?VRsO!@)y+&VsTe7uUwXIeqWoOnll?YVM-ACC5;`g2b%N zLkwk;78=2Q9DZW~@o#$L2aUYRmkgN@X5j=d@p49mncVb58aIN8-;+(`5R$LuH2G#xG#o{qa@WcJIF!r&QopVb)|RL|6$|}7gh59;a zSooT47F+PfHGNR+NWX$-p8m_{0^L?}`LVf?&s0gby~y(7jyI*$#x))c8(95?(SIRy zC6wt)%&{R{`bHjP<8W%ykIfzF+JN*&9W5<r*{m#bVxi;^}vgdJ|6l@3R{@EK}E2p4Ls;BKm08<(xUVv%^tuT+y(tL4*3FeCib$ zqO+yENlF)%zw6Nc2gDmJ8pvVZOL=PmB8)xBbN~I}z3O3<$E|!0cU`KjfGOLgj~^Wk zTa}zGN2j;H3I*hcpErAGXO9JW8kkKO+pN-}vj^NK3~F0sbv*gmumRb?5#BFsP@#HN*Vtz1fH=vXaZqvi z<$C-n^t@_-sYO+kFR*+{>oKiA`;{?8_XwrsQ$~|!`gqHO9FYsXXWXm2xY8H?vEeKI z=S>Oek+R~>8%lgq_1g98+l}kDwr4dy`xAP)4|Effi!BC$F6RR+(drF&AK0kgKA2M7;X%FmEFLls+mp+olkfNF7PoL5 zf*9{NEe=fBxaad`w{G6{!4wC7+z)C0muyaIPw6#t2Ij+6@2WF-sb>yem?zZ6uYT#< zX%_%p<=(A&xh@>!a3GxxW8z77US9=el5s51aMw7Zhy5>M|Kzr9QV(>1F8Cy~4?sC= zXsk_UkLAj1`eq|{2o*?OXbaUtb0r_n`0%;klF0#l&t<6pgQCkDf!I6k0?!A;Gxr_o zs$|{G(a@v9>p188n$Qk{*{`umseK^jzDav~tW15#OTM&4+QT}htO}>UN=KG2dV)sy zM;FxBo8F!y@`=ehyj5SpUYj2LeK4&4#zE{8k3TWqkfIOzVAwiWAM|}GZAEo550q>g zY-}&p7ud}2d8rSic*D;&e9N}XCoUN`I24P{V^1EBM7!^iJ!j~+Y2)-8wa>$`i$1_+ z{_62wb;>x-n~m6S^pFSo)b_Nw{61i1jKZc%25~Y!<{q9`l&-tp;DBz|26>0PATld^ z!lbWvPTIH14|nlX&a%igozTk|jy$-DpL#0NDa_vq8pv4rQzj|)C!+lJ$_f33=}#uY znI8u&qjWdt3#s?C>DaT^VCtJ4<{nG!O_g%0pJ^;K$J$73%-W^LPCq!vJ!*9;jlPk2 zYvfCftm^RQeqh^;4cjJ!t6f3^nm!C)4~Jxjp+DzGfa^)W(3qL^O8TQ8kdZC=FpBY* zz9^nZgM5qw^y_{zD&t+|Gae_ZpSKNIx7I1T4L{dMbX}LK8*9SUHR;y3+ao#vMysNG zx2M`wJbG-yP5&!D@twvzQSOSNvqzZ7mvH?mGQ53BVECOEC;kMc*DyRspp{q1+N9ya z@>Ogt?S2_g`PS5n!P4hXm((TR;hX7~zzH`R569O%gw{g0-G-g=&cRCc(u@ZvTd(^eIH6awH!(f8nXc!?sh0ehyA4jI`=OMF3^qPva~aai;7-0 z-3zH*MyG%*7iT-AGxO_8hr-!~3mtbckluMIx5jn2q2nE0*l|G9F`gOxbh$8$KR5BU z+@$G1n8dZ!ul{rcuEg5KLwPw3rllC16%8tcHMcS z2sRqX(&%7*J2(Ufq`(CGmx%K2R9{!~@E?~YR{miVi7uxC0Nx6IkPoFgf9FXU^X$zT zk_h)C$5Qz(F7(pZQhvaoxd3^cCw$UR$tsr#>e6SufDt-_09F8Uuz&{E2{0K_E{m1W zY(9fiC*c+;|VbnTTjNAcrg*1G$TbOh|SkJQf$-DFr)3VU+NWYfTchjK#GOYEqFC-joLLD>}_f0hgZ;w(#n$d!k`n*8!3Y<`u7KRFF1 z(L9Q#>G}_U$2lrMz|M#fq#S(EP1g?{GC?L>*-}n_#IYb4i%RKlY%1cCE*HU@7i7>m zd63T_2c7Y`owm1L7_iuZmM8qRS>q?`yb+KMxq)gY%f!Z|{7`385BXVGd6UXCvvA79 zKelFm_?2%}Fm%QfzdXt3MGu%K?piH?`wp%yam8D-;cY%uKb}HnbHe{WgvC7zB!EN5s#o;c9;w7#R0bu+j7c=z4#i49t#qHR|Yx+S{+hq{~q@ud^; zwp{4VGVv=K8T-Q5o0}}QAE-_G+O!9;5g%^g_6Abvq|5CNq;M6_cxL(@U&&9|EV4&_ zxuK7}urN-24anAvrqt&gC}njgH$pTj7-3cKAz!)O}tOW8#%^YMXH(eMT@ya0qid%uf@?{9=n2bl=( z?{mPU+wwWmd83!um>MVPNzMfqXjdNiO}iHMhRj0`JDj#dUdiQxl7lfo@^u;hmn8Yw zpzQydMf94Yo1bi#%DEARt~XFq2HBU-sE$5jzgx|Z+#Z8=4;uA!)3fH z-rg8<+YwH`%f=WReTpZ)xbyi(HkWw=f`h5+H*bvtDgLKYHpQ|rhF;Kn#@77*&;DPE zypi$56OZ|D%AX6X1F0NJsjp){l0}810M*O27wvxQ137#TR9ybARyNu^mO5+QkE-3q z{Zg`WIP7`~x%cIhW%YkjE8aFNaDBYtm%|lRIIh^gn?I>HW_Cz(*JF-eX&kIU6 z#d%Xw>9#@l^OQ%cwk33B-Q|JKN;m zJ9RZSgF*S%jbQqg2o$Q`AsD}PkhW$`;6n-ClF@uAhf@8U@O>O|8L;_pJskbs_fqfJ*xjh8bdv16pxHE zv2QT*G1;2zjXk_)q+1SDIgslA8O3M&@DoqJ1kr8 zCZn&EZ&x0N+AW|Q<>B4B7;+W%eTB>`KeIdnk))TmB9foW@H=tFR~T^V5uRnc|9T9f zUm_RrRX#SI@uTf#9FE|(9#nVJgEVES<62+t-E}XM4e;iZ$Ir5bwh7;>s=uahy^?iliP!#*5XBr>eFsh?~FI-P=2|z z0e<4rKhoZfeLreCAcT<@uYDt}m=?tA%<&Jm;zl`QD~|MpMnJV%{CH;_iBD2x5KLe4 z2jUxHc}#bO4z2_Wh>Dn!w<5~7_XEy&7dWuK8<>9n_T=qfBaV>!;3+7!^}fRe9Homf z;Y+J_VcIdFh!vbwh1+L$D0MXnN03A-`Coa5QcLw+k)yI+IqWE1M)pkFKB_{O(GXmo zd`cr;uEPA4Uu}94zBjBA!w($8D>$uu&H^2Nz6plriph6C{p1;86?XYGJ@NRUu)73o z@MZN4OR~#BqoHZ7ac96cghJmSDn5)%!J)`%U-=u)#RUgIpE&%TALj^@dF3lu;@9Eo zNF$`7-Mp3@tN&%f*^)#;v$5c8-UgS67ue^$sHEZ#Nh5f|muuHQe#7EDfs5h`+6n?| z=4zCz!UW=IFYbzp(WNI$M0APE;Dg`zLFr#iE)<5FvFz0UNLve4CPJQkJ)cdSRbca0 z`MF>sd2)9RuuXww)vx>X^P*4`I^P6GA?DX$o5}lhVIZU#6TC) zABz-$?Z*>()vshA7j2Zjljv+xuxNPDWO{K0KNcA|l**e@(CUpV+XDF%#}uLs>X?PdJWvFn9ec<>?Cy;xc=v(|u#NFUc*%<{%v*e^gKb%H#1qaF>Uk1y z$6M*vt<{&dwky5iM-TX2Poevs7edLGc(on8(5d9X-@Iu*f2s)`6hAQS_bSnfZoRnk z!ZDkI=waeU)R}GY{&o{Hb5$cOSwY6xX_GTCRh`=zwsrNzA$?K4ased&!wYht$~zI9?2A z9CmwD-fj!1#)&TB<2qD*=7COhmcty;aGxl-El22CKI9ZnZxEDDqjSQ!x9DQlaF3mY zR_b3H7?dd*J{VHjv>WowGH~<{vivC9Wm*qakNZ^C|7Wl3(sh80O9K%s<&ZzL$nQAo zGv%Wf(I~xw)XDmEqotiACN4Y@ww-v>l=fny&=oM)ao$rvm)=mS|0UG+ohOAQuVic+ zcSYQ9G^Q%-d=Hbqo?>Q0wGX9~J`UGt1L+1Cys<<+4j@#9dbuBJjKSrekuko;GW%(n{pn>f=EC$`uZOu54HMTo{K^k8vpVm^a4rrf%>)wTTL17#LM5PUWbhWau}g z_D-4H<_TqaWHWZI`!)a=&zaRkyK0FM0fy$fFN?$pSrXy zr47rr6^DATQRK5+by!BbAU`&u!m$CyI+s`9wtlL9t_S*wj>e`Yc{Yb7 z98f%`xwP9%=~kLc^L9LMXE7J4c^N$Af@3~V{WkYf+Kury@`Ossql+xi7h zqDQ|o4YfbCJp;gIg&uFZoc_#HdD5rLUgUbr4m-CKZAKkAtmP21-;x5M?Ky$U@W0{u z6Mc5C$bGczpK-DEAReh(Wb?fqds%FTu@GJJ;9L%*7#nW-4Ji&*`M+~FZ`~RPQq1Wh zaY&=TjE#NYnuwiRSCW}S!`-JGHD6_%&tbuXril)GK&HOpH@2S6XBz#9W65Wmu5p|C zM26x;+uk2L^tw~11pWflKOVdWl6LaqQkVMe!1^m1y4bh4K=X2G=m?P`_mT1=+u)h} z7lP7FtAZ4u3}q{EM`OenBt)i!fv#&1UA4}**Ct*USIP5TP~%d*@FAD62voe=KnRbf zNj`heH&)LTH^#(3v`mvN`-l6r(+belc` z*p?V0+$W_^-kmZL%her)d4Zb_yqBdusvL2+rSBJ{K6q~`3?4$ zXTXTOCs3=Edl9UBg}xN#asez9yWqfzE_cF7gdue&jj}5y%6JwoOb~blR{ryFcvEWS zR+zs5zw!>HW<3IHZ4Pir(nYW zySOvxJY-qIOZu927D%(BYv)`1T`vB13;8*c3@+k0hw)WDhn(mVPIec?(}DI2GB70H z9%fwbISN;Vix)rp9V;lleUn;J)=a~h9Ez5MBcox~rBae9`74}X!N?fs>W(-?EXhtc zG(EiI60i&n_?O_z0gA170t=r@^8riZ$m6&rVZbqCU<8ZrOg{Q%*xn^*IEGhH+DY=L z=e!}VapS7EHFmT)fPWzAzSwmhT|W4YA3V#DW+WsOV{D(vq@xsOlH>fly%dF0(4n^N zgc=eWysB0%Moj*aZaPH3hY6;08oN9^n24DGa%pntNf(nRCQYUhNQvf{g)vcyEt-vr z>4}&Zg_>~1%>)b{(6I|&UC>WtF*!pPJUwZvMXY(KtS9G8X62$!=OL}%l*&_nJc*o% zZzi7BtNDgE6T_}lPY^vJ7n9Cop2;#5K%Kc(J~AdLWmKb*ktinh_??hk@>6zBOv{Cr z{JnsRONk+Y;}goYNE)E)1x>Xp6J#$MMce!#7J?32Mj`o{Zqu+!9#B80@vAiWA}fG@ z${fG{DLD%wp(p$-WY_>wTl6N^e48h21mDmL4{u0eM?(B#Aytcjn})Zr4852~Zc>0-LCM2I{HVe(IDr6X0K)9?+5K#XWU#$bYGqbz8dc#Zbp5 z4V#WW;T###M!4v~i!T;9JQy!vyp{XEv~9>n?}=xaEerT#mvGOD34-Xd4S&$-I z>Xc2lw2jN7cXZ~lKqy^~MQrhL!z|GK4UE2%k4=Ec{HQStcI3#0VHWPaai}sXOS_@B z=sfL^1CHpeKiM`8J)nVZp^5H<-S^T4LS$6Db;%|eG?w;gox9%1P;}Of$Iz_}6_tO; z?GnE2MnBP+?N9Y+os7@T@pccgns>@M?0N`A1HR}l<3E~+F03n;gLmx3sJP(=y(u4C zf*vwp%(OKMC0K+_yW0-={Umeoa6adVO;4zBGMJBb0%|Ps=WA@=RaeUteOeCN+Y7E^ z+9XJsQBL_u-}Ph5`6`up(0M>)rO&x9)`pSCGmqhY?-RfZII@}SztpCx;mw}gCP+YuAk&M)}ZByih4^>2bnsNZ*QzYzvk%8<9O1VUrXX2k%ujddhk#SY+X5L+|$V*>IAIqVX z^-#*Ot_rL>bf~r=)%B#Dv1|f65RJ?q)Q(*aS3l&ljwFNSuq>3xyd!hav{~Me%0Ucu zb~|d0$ef9{%{Yw0zBvRoy6@$1ig@M)zNaFW>zlcD#+uoe;0gNBPddd$`9ED57kESJ zmf!Gel6*d-6ncC|MwLT8>S}rFK+1;zLc-nV zAUakN!?am!6*p}R4!d%Lraf{|d?izUMP0~Xa0&kGK<6B-B2`5PtTI7s`0 z*dYC+#^Js%rEkXXdyC!V@fd;asDFr{dB7?Mys_Q3-}Vy^kel}R{SJCX+w_SOw!Nml z&`#fQzwo%Lekfb>8?Yo-N(wngZPYFzV@zKf^l56DpM(7HtjJszgN$VE2$P0fTYYSA~0 zxXy#0ziA5tDtUNt5^)NL`08CdSj(Ozqaw3Zd^;)*bWG}DIxF3Z=(-V3vSJqh9x8l{ zz6)HGeelR%WJ;u9c+Nm)jQpK*#s*gZ5iU!+H#lE<*BVQstW83jzh?iY)ERLq*s;H2 zKi@a3`B1-mhKtLQa*}#Bau27;in*Z7#iS}_$uPWC=X9lT0?-GC|J-kICh8&(%Nb74}4$31>7nA@s62VNZnt z>Q2~;n~F=O{dHAPa;8mPPuvPS-y~#NT|di>tJ1{b9+CXMxTq+~QzYYYbli%koHxQ< zs58v=PR68k9ivy;MY<);VNJ@*?|KN(wcAAHoJQNp+p-d10mNbgTkw-`wIBe!0m6b< zNZa&c!Q`6`Jp4p_cxAyo7d-2vpY)Dnb6^GJABR%Hxe&)+pL9e^792o;@-AR|&>#7b zq5qEun}XRq&!&lW0=N#PkA)!i;EgNlD{MQaFQ9-vaKR2*7PqDKojzgRs-DDKApYPh zKMTApuF`+q$HWiXZu_QNV4vcm^Wd-pWRpMRK;rz?8hWm_@n#HdAlLRhFt*%iO~`GC zIz!Vxg`)!{lGVA8B{EnR<;{k2EiSFHghw5UuMZL)WYL-h?8APd|55)B6xz)jN^GVo zpBK-rM-G>=;AQb!Zz4Qed9l64{yd%{rCbgMO#jg<;+1mV#DDk^_XBTiQvdr(e%rmq zF3a3Ht8vq^z(WWxAw>r#b71gH~@O5xuoyc{)RR;RCVFweFm(E?r zby9ugS|2{VP`u0IvLT0_vmwy-k4z9kPkm6=G#>H+AG*xjOI_dI%%qH6Zcb%seXn#y9b|{iAAS1J#pNCyWrjXR9qItbx)v?uL4K4>9h6QT9(q_bL@Nh6 zvutDQ!5bt(lfbTZf6~ReZE$}mME2Gta*=K3lni##M*_(JuN)>}%hWGA<{;8_DmkD} zN6AJ^9ZFdj%Bwng3{yF1_)yJu7#&!T<)Z84LkE14RsD}~T{P!d7Mo6gj?DB|KaAk~ zsfBG!E_@g}dJGBQxWkT+!F^hB$f7ilJ#KgU6!){%O#$T^iWku12l9Aa6J~tGRvFWB z|G1y~CbGb{T=#XM`&sFB97stYu>-95;6)TGRM7^CYB!A9K=*yMg%y#4t;wGo?dFQ`4J zd)JZvV;!+!$L74SHwkfV=eGU%DYO2cQu9zM_g}`huA>ioh9A1n>_aJRGwqVLP2cD3 zLF*cwTko-pw28(k>VRC*u|M5+>CF5xWD+QTj6Y-Xh$AoIvj476>@@#J zYT94yp~o}o5IT(#hOlV#eG{4IK}im1axlRm1^-Vd2Q=iROv{8EyF;lO|KLsA`F>aR z;D2HEc!WNo;dU6hkv!$2ZQKTooASG!)ds^g?K=D+Ke`qIkTg#ry6wIQkY&`j>e`>| z1KCsa0g&=jK5s?kbA3J(=0HmKL*gFco=`e(dBn(kJU8(?jy@J-UWUE&8~gM{4y62l zq;B5wn|?VM%iC$#nCCUpBmAg4{oi z+)Jy^MH2#wyKd2?>up=e_=p`~@9V*&?rzLyZsVqv%eyE4o7-%Ti-@*{<*F zmM3dH=3B8jjrrJ@$AC$@$7`zVy6Q)LQr8+Y-6zs7@MD*$JNKj-=R^}be)LMuubH>B zU1#p*IV^pkbkSoSa<;;$o5%a&lY3Chkv>YDjpA=vRoAw^ZiCRRHW0759zq|fd(Qzp zk4I1Vk;_;%I%E{`LrnY4rT!8c@x%3BTZN$d8h@Rk(7_|P;4DqPH3Xb3cLLJNvEU9` zj)PZj%W0P|K|{qEW^7;yACoSCHQTu&iPJTIL!Q9!+!LJSzEn4NqrNYEp41r%&Uist zHP-ZR$3v+G5gjcQ27UeC<5227KcoTXo5OJh2C`QKJ5r}~-&+9X#(RZ*UWG0wd@;>^ znc=M-N2>AFB2}nn;~AfF!3+wjHZ(bOlXuVa2A#AIyfbN;i<_^7O)9tl16-`DeOe}7Y$_S zr$qs81WPu0?V!uA=rPGMN7Jly=LvX%Sd&s?@ilK`S03vEWb$Y1$_t>Wx@VHv6X)U~ zbXiYcqrZ_Zm&v6Uo&FR+zC{f(`CLaKe8`(gcs~KEOJ%{|8xyl;6Nn+cDLTIpmBObt z^1Ggd195F5t{?g-zg;1&8OX)q>Ii}97nLR{AdhW=Ke=Rg5%VTP?2@*D{z_oz)C-|0 zH|f(KaJ$g9Y<3E;6&6z_1=-5V0wt62`Si8hTls01J{rpJ=(0r$F)ZPrUOfzvq*eKU{k#6~DcJWlJ9V5DPy4(`YS@yfCW8 znHPGJ$7P*26yn6+`b5v5`(D|yu@;1bx$>spqLaAIhb++c6WxRrpdUj=#fncYY)z+$ z0`d2?CeV%HltH# z+CrR4&R5lfIR`;a+lMKlvu@rXzc;H`h-+~^+f}H1)@dzdu_=e;nwNFd8%5D25;%|Z zai}JF`K%Bd*N;7x`eUc13l{O&M95~^T*yP8HcnslK@vI>QWxu`>ePo?xYi$XTGpn6 zi0|>{q|224&_!gCE|u0gM}O`=jY^9@o1B)V=&T1y$F^&oz@PEMZ;Sbm@JbG1s6)Bd zLDj*BTGkeovyeZ zZ}O=TDO_+>dn$K#3R3Wjg`rrXZ;`Ox+V_JG|Y z8)e0ijeZWKI4EHQnvG~4jNl%$Zpa~zn;L%N(Y8Qe$)OYnQMEzt&2hQ(OWB^Z$z=V??b6u!kc~* z%@4rPuGGzCyG^+_mui%5n_aNyAe2ix(}!YffcvxZjRPt8t55ZYCh56{MfHH1G%ko; zU{BJw?W$zz&EY9$@(lYcTAad+F=<2sEOn$DXF?Du(2%$|& zxy`I|5#g_N);hv9ePy8@)W-)y&)jjJ^nsKQgPyfbN;{UF`gDKu|8wJhs<7|z!f_Aa zJ|;etLkjHJ;~itJY{HxR*sARRxfh-npZXtC^+uErq-?u4+&}Lq9o=V5B4Vyn`o=_Y zqYadRF3|_JfGqu~J|9XwlD^9Qi8g^8_2=*jIeZ{hV?zJ`C?7(r?|N)>jiS?xzvxD; zZ8V5%0otLxRTA0xQJ#z?ul*$3iJ!7P$E57BA2pba2}3iqKw-E)aivR;YXp zE?<7qmg2iK$1iCU))*=!$N8W|8f&f8FZZAxhZfhiqduTCX9l1Ff1&k`{_$5h2SFLL zJQmwNGbW^OGA_tfdiq7sG^IPh^J{DuyY;v~$7_!P7`Fg^J6_PPaXa%s^-oZ=(~fFW z>goPhheFWj9#CVc4`E&RrQb8YsSciZaG$Ar4PzelgZJQ9^M|(6=(+(tQ@84q@TFeV z&+;&DiOv}lEvwsR+M|27{#sw~Xg#F=Wz5Zd27hGocn)H#v>7tm9~q6djnJl@2}Js; z!|IeDVTQM$OYPt!D)l)JWS#}iP&0Jj44k8OksPV3>t{@Kl4e|BcnVx{AA8Kwa8bNq zr7^O}6YA}cTbE0K^m8aUtK=D3xh9+;ad$Xo+_!C-DKY!_gkJOd*Y%;)H3f_FY=MG= zQP3~-q11c+o;u&Bs1kqrrR`N;lgU~GH`|6!d-JDlAM>#vv;CWY^KU9<+uro1H*IhI z-EZA~{9pWdBC0x{|M{Qq&Pj)T+uOci`_Ui$(WJiu_@qz%5m!9K4-X9}OEx6eDg)qcLxp(>Qe7>>os25gZ68#OF%<0* zpUhR9Nx^;Fhknq9RJ_79QSqdR*ij();q6H$3t}c{HAxEX$YuV}U{R$>jhUI2ux=*Z zpByJJH_8b!S#nIt!lJCxm=KW?szWBp!UPmq2z$Y*u*U*y`C6%y#iJ56?KUYXvOH1IzhDci94a6PG)jiuFAUIFtHYYXnO)JgkP6`i3%v-&8X7nkkuLZOJ+o|+@5&fA70Au z1ym7?j$g{gNY%}g`=X1V1%dzXk!_B)Ok41q^-kI7M2i&pz0j&fmg}Op)F<*}vr@XW z@G7S(ww*VnctVv8!RNi{!eX3_H!=-98ysQGLL}IEkcov7Ps)y`^}HBiQPvwlkeM`b z%4GL=!$0R1nH2Ab}c>Ncl#l?rKfxlTA`Wo!*we#HE5UX zc5)v}dy&7xEzen0-aBMD_>-gofcY7xulUD z9of!lqdtUC`>>eh4W=B{d4szomRKJ+Rc z{@fqDYXd*^c5da@xFx%1{{Eu+fOUo5gtlp87TQKHaQXiarK74hvSw_SyOg2Mk3RO8 z52YTxl0zx#6IXo)Ta}x6dh?Eej1|};-`eLu%7+|wslRAgr_qmTu-S!tB`fr(WslFu z)Q2U5#zU>k(mOxe%yq3gQy1j4tPd!^%BUQ)9(<4krEEydI+QL_3l!x#J%sGWE9U8P zr3cI38!7NCJs^_|Ew-k9NPc9e4!kjy1E|=?gAdU6iQHM!N|a zH(Wns4y9&03riN_0K|hBGPGVfP_lk4C!g=@I;BnWWb$*{3(wuLt&qW_;Z^jMPdD{{d*d9o3Qp4kNE;RGMd zU;{2I-r~WAxBp+v^&9?dos$513qs`F@B;4>K@xy-GtnKEEgNp{4XIk5Xmn@ri_UX z`Ay+`z7LzwxFCIaY!M>G?sI*$2|17rL2nH5dF70uz9*qu^jdV>4zz*jrA<;ps)T%Z zsE_o^#=bZB<-e^s)ynNnGh}R}O|cb4sIbSmvPWa-VgT)cO_nV9>5u%sQtW#iNTDOm zAv|V+2zCHuXgBld)utp@+x}|*!sL%F+3pKzBlnp?A4oB-%C${ecd2vw5VnEsHVx62 zOMAxtcx&i6AIkXij<@T8!t>{PL&Eh+-Auc+thg2N^_y3Ucft2i?{^`09uVE*>M@0Y`>uLVe5DJM5F3VJM&hS4pTj)hVmF_RXQQot~2$h8cOv^8nL zlpG6?W>Ibc@?|1PaOXQ2~Q0j@6F(YUG{`trR|ZPEup>pH+~P)cxdWT-Zv zEi+!^{!aVyhN|rVRJ-S(dd`!2T=krw`jlx3%|(~CS!^H=BFr(H49LOQ!FZMa4C?mM1)R8N82lkVC)YwOsji{mO9=e4hY z-R2RTg#MXIWeRj)7x`X?m%dZnKW*}O@j9Cz6Ra36!neKcZQD0}(>Lu5{mcINmu=tp zjo-L^(f{>}h892X^FGgUpZJNNxc%Wj{Kea!`*VM8d-I#$tdv()mp}f;|M>RFpZv+& z-~ao6fBT^y`k{(F4^y(231>vRo3xYH%60L#_e|8;xC?>KwBo$!dZO9`+(|xSRC<@z z;mOw)sLE#G4HCg`r-cwJx5}?H3w!X*4T8y1ZkX{`Jb5V(6i;tBF%exb3&rG#$1J(Z zEt);x7hTg{=>sb3g{OJQZ@S^<`WB>2l{c->^+cyI^wS2qkh@%pCS0Kf#4k>K{2>Rw z_Altj2FRD!KmgwHEO>;=()O^+;Xq=#bmLj5gtO_F6m|_=R|ne>;F}>J`dkPdXzp`Q zSh5UQdM2v|c6Qr7)G#7QZX=_YOY!}(YWWvYd7rOzv8OhrdI8WHfqcH6V zZze+=pzxI9(@*F7;(nr5$O6$%&S|m00=^fAkttH8?`RPudv+aM5A~s%-~rT|{>MV@ zA<4iKy==1bwfu0l>>AUIw0n=6q*lFbA7$UYD8NrfbOVnb>s&9&hz@!B326@P z(l?Ukaa3sgrmif0WH;CW?6GSDpx<#9q3UkSC zfn8ui+G-gKO(XrZ7v_!KAFUsdd~UPK<2J!99C;&6@>dy+0iu+#rKpSL zN!boNu4rY9D4Etg?RUQECSS==a!eHeQ4jnxZ7iZqzx32}x~$xcO_VQx#jj`?(DB{m zu?*ZRLZdfXx_oS?%6EG$G7sLCeJvl|q-^3Q|Dw}a@vgi584Ilkw`+x=L0RQ-yOtit zczw`A^Bwbce%7l^(P6u!4NPA1+{IHcc~jDXT{^hd_XG#+({hgQwla+~a*SU?#=#z+ z8ps=A{kL8O${Qz8c~0n?MnQRrO1{;v-|8xU)^GJ!y`8V<4Iw6sG(7xKrph^9Mewz* zxYze)us@rnu^ju!23T!+X@1A&UFu-x96_k*j~DMbJN6e(}ZQyO`As9>3G%!f9ixj`kv&mP=2?+ zkpdRVwjaBUzw;zMngRE)@VB=E=K;CE36#)JV|U)(W-gKkjq=a$GdI5d44*{I+b+-Q zUdX)TB@0CR2;IL+PQ@dI=StFPqxCPO81DbJ(VZpSG5go=Q=NO`gtxW9YvS*Nn_be=4QM> z&YLoMgO|BR9}G3L9)t07u@kS|^%@^(C@tM+XqqVx9Y1oM7`y;-ZNVFvA*CzkUoeSW3iA=iRS-Nwmy>X zv4U~Ohfq1d_GgAQ4p~O!M?UP7H6;JP3QT)vuIPhD>1XtV>ZkB9FXR|@)p{mJY<^uQ zyLgq)c$V^2U;z1oMWbA!%ZG-G8AZnqI-z~RgU$HYg$fd3EU5&YE_cc6(iP?k&DEb! zvMj}R(2}PX`Y8Zj2B9B+gX0?E4N`Ax!S;|bpxfRL2k_99>^2WDl)dtx0CRrjV~l8< z1<>`nj_?6L_F$XfK$?Tm=Y3yC&m4I58X^xSKsyJ}p2NvTn)Ystq2Gy?554Jcvb!AQ zv!{_aqX6bqsZVU!bx>XW{|b4ih=+lwhvr!B-_iq$3&-f0DTv2*`l9Qm`z-fO>Z5x3 zD;ZTkkKY>OuzOmp_1t}~b#2#UbneYP<|2!+*K77qU1!Wr`7stp)B#wf+ZbH=y5< ze#zmbzp9S=S<7VODGBz~5$zoO zf$eYp&2Qhn_G`Xo`+_g{g6)6(!5^$j>SEVfr+Wds?sc!*KKi3SdizKJ=pSug_N8CC z{jKl#j=LJ!U6KuZws#TYZcq&E=n}WOSm|UY0K>ER4nVoWT?a1IWeKkXmvBddxOPd4 z>&OOG-=>PeGWnOQ>4mit%^dYZS!lEmPtmWzcOAqVds5Sjn3loR&0DCONuxJCOC~uJ z%!h%yF?1Wdt`-hhXnW#Qlc(JzNEk_~9tX?~qR@P1Sbxrg2bOrPk|98Y_{52iKX~gm zWJ&VC0Y!TUClQxH33W-KU5W!2=RJYS`Z7uwQb%2_|6{$U4}f-X`c{Z0PhriyFA;#X zCs48rFFNJ+rY{p4?9iF?7aIDhJ&tzAxn=ngiJJ5k?jT1VbQ{5(u+7DpL}49pNBNOfs;co>X!*y7WA&K;=MTG*S1VYiN}ty zqNYp!cGo^v@%}A9T^F}$n78Q^EvFWs9{sF8)q_duJV@#K=CdA5WSPKw(P-ijsKpMn zaf%;f9_{T(d`(ij{Ts7c0!nYj#E(tvn*4ggFKkcC{ZpJtD|u}f%E60>*y%Ihbd=^sLG5ZM(UA!U%6r`fO@=s=?nrC0>>q&N$5>y*tS z{!f7xD}FmoGQ>`IVykx1C2el)`VYUerd;_cI6^BCz~x=sSL?4}=ti=70fS7c8zi{T zWI^5e3gKlQ@CqH|f^JiuJ^{4!V;o2ytTuK#2i+%yy{KWK(}&z{la^E1n`1tZdOn*- z{yz|)b<0>0y_%ot*P&P0BJGVDA|;+kY}E|ET$1?NpD>riFaC08{|p|(!5R;2&(X`e z0Mms(V?bf9m7ZUvJ7WhAIjNH+Mt6BHMUsh8i(2r5*J$wkD{-i+r~6n zSFzh{iq=64^o-C8O_#B(lSa=2&9n0l8G_ME@iz^jb&&hlz76)}?024&>$zy7^Vyws zNtyOnyyY%Atjp+k=wdA^`EwXme)Ljt)|1d>sy$3A7HFDy({_$3K1mBw78xo$%2xii zW#>88dDbH`;7RAYXoaeXW=>16`+zpOoTv-Wk`Pl@4*8j!PUm!)`pjYHVdQ7(X(h7w)mXuQW1d zbEu)Y9P>HPGj&fv8>S~(*w2FxRzH*8Ih4AOv7fSRDC=H&i;eAT*S2dYDu5 zx0}~*Y|q}jwY^7q#MJz&frQAsO5gIwCWhV92ZV0-vI7joday0DO|-wLHtET(9eZ5E zC2fUSo(sGxD@An4eK&ItbT;N^o>K&QJL(qmH0Gt6tC*VTAmiwR)CCYtuXT&2@2T7) zNOL_iAM%?%a^3cnsr&8=&%IEas(GWt2REikE%y~PN93pwu5~GlOv2LnaZb=-k1Zf$ zX8Aq7c)STZjk>B0(>CFeF`@Vg<*}UD9rIbw9W-CyP^9N(enU#O7W>@aJ4a25W=NLn z1ScD2IFVdIM(sTw21aRCZg{!9kfnL!TEgz1^d)R0oBM`pk1ii_ zT*(o8YV>^CLN)Z&NIe@w!_-hCRiZb2T z;7*&_UVUH6xXM`I{x8=Lo+=)DgfBE$mv|i^`!a3${Q!l=ALbg!OFf~@dJ(#Qh(NTU z%{n>{de*@ZcBj4q_}+&t+m1P4%OM-}=&uy;W;k<9&udhV(wx+}*rSj>*y}m?^nnx) zm*(IN-gz4l`=v0l+s3=Ep)X;^)#%)McYTGDmGy3~*ZOeEZ_Bs7L3D0gwWxjBfs=IU zKGuD`<(cDoWAYpM?IS8Kv?s*oY=`z%I8LDa)p6g(4W!`bw(b5e*MRc>oRw|pn7{Vd z{+b694xJi4?8826`*;8D-)$fHksnzx+xFof{^8p@-}%n%-S2w$_6PsqAKbq9i@$h# z|M!3Y?bWY-_4YTv<2$x5{jx9JzV7S4Zu{16{Z<D zH*R146<@Lao$vd;?YqAFyS6|4U;beaGz}mB@gHwKKkz^Q&)fg)zx{80AlERogn{EM z{+YU_&Di#gFyM|dk+jB!BqeQ~cpXufxXS6ccB?!DNS%PqVX!qhSde(bzCeKqPRBfi z9!MEPdlHpF*_#Dm=WCkHkBN<3qx=aoL2{;@C@NjNng^Hxya#oD0L6^DbiHOhO}TO` zu+(SX7>e!zbup**Q<$r1B!y5xaa&~R$ympgC)szf(kW>2XOb?4wa^~hXK-MI4ExXl zs&F|M!=Y*;c(%?32cpw^%kK?kO>>8$)oG4x{t*%Kf94hUOat~6W4im*l}^4KUo7-wy<3x z+0qngC(vb*(l2mw1Qz>-g`uz}zO)Sc*s(&j6jBwzZNb*ubi ztaf6YhHe{Qodzx0BIAJDJvI=mz9RI3uk5Ac$PYdEq4V(sAo8kRd|*5le0k!zH(4k{ zn|7aPjPgPfd!nsNpF<^hsI&did0gU_Fpk5Ci#x;P9(=?q(_`kqGx_0AT%ch6S%<>tq;XFx zZ7R!ymeE#NziE?=UH7$JL0$Mu{wkBWV9J&J0~a}pu8TF#c1i0p_Al{v!Ib0L;Qm$U zK9)`vaJI^Cf2G-pzfyGx4|_*`0Dr~X8#%a)Fh7S+b#^Z992AH8nf#2ST!m7{&|Le& z9A6)_4!KY8Z~z;7d=iU|e%_YiKGVp3)%Q}-gg*3otQc=f!IOKj#(E!0@rK2<8{3U1 zuWeV~_2hQ#-A`@Tp1SITDdvof|03=OMs`4b23DO$9dP4M+b>7~xfHZb2&X+29f%)c zZRzk*AbYyol#ZC9CZ_`Zi8*mCV$Q&uLmbxhLoLiJm_v|{x}{E2e*cDF zZYP$Xcsz-j;QWg8m@LmaM%yjFUDaXjw@~BPbnM!nOXo8dz0@^xSZ})5yuLs6*zSL%(CFJal&X2T>X84X7W?&^Qq(JKttZdzG+*>Vfn@8?oZ-hVXmjMFEooQg z;Ql{W>MQdQQfo`vT71cymJW~bDVu6kBnWseO#fN?kU%+-&+PymE>1F}uN2mNI+(N} zL_zah?cs>J@Mk_|z)Ki`z&04HwSLF_eq6B!#qAAK{X(e{3uh;;=0zifiF8)F1*OBrZ* z9Z_~>dxN%QV9qmWtAecQGG4Pr$s1VQmpJ^k%`wK9J`dV*2+LO;u;th>ea)Z3cE1b4 zlk^_vS}&!k*ba4)J$da)-=kje*LclmUb$~@FvUH@Z)!$Y7YRxKt8`ax;UhZgZ8A|5_T|eO1XiFB8HTL8AoIa zzFc_en=dZ)(w|YM{jz1U`coPV-ll2qw^jHI&{Pa4|KUIUhwURi;v*dQC13I-9x(p; zU;pdd8{hcG?L$7~LmbclnfkZ?_TS2P-}WIN`k~vWe9EV6uX*ijbz{HcPx^5%#HRqM zKSuc%-~Jc3FaPo{cjx_ezwXy<-}}AayM6gre7R}yYBTv5RQTpD1IWjI?8k1u@Av(_ z?JxX=zpy>_*kjK37vKK&?JK|XD?JD>;XpRhfhV4L!k_;m{JEd|xu*A)x4gxCzVu7K z)HFH3`uhL=>$lJR%+GY0-|!oL!}gc|@?Z962a(|q{h>d!{qQ^9vAzBcuXp<8ZPKYr zE_p|&JWHwqWGE!)>xA^n>M+%r+@Z@|@p#kLC9cXluH7X*D1fnKV3N=it40sK48eN@s0V3#P=0K|7`yXE^ymZRi3`xmv^Tcli6N7z znyfKdWWi`#u}u^+yh7WPFgBAU4wTruU^3{1;%&a|u0@0w-&#Ou5y3dzlSMBWk!#2n z`Mf}?J{Ad6b$u7iU`>V{Fz3t;4yCo)yIAf<~2jqZlWAEb!; zOV!Is)`j{?={y_w(GB%X;prHLzxv;c*~Fr*-DhyIBlnx74UieKZ8u|KBcA9I;%-05 zNICjkJ_n^fDZKMs^op)~d~n-{D0KWJzsDyQQC%0>sk94!TwPo}dHlma!sRcdtX+c8 zF|Be}JQr8xDN6vqBNL3buwoM=?6`Q=up9$nfF9s6Tl(z-J=%oBbZi(~;WwZOIIgjs zeZI9GC$P)v%yyl>`c}q6$C?IY5_b0Z2d2zA4(hMUrZD~n`znazER<=#-6d>n3b{^L zDsvZTdiJkmT7ONdT<5okdf*XM|8+l($LL@coTNKXLTABQ$W!S2&^!1Y0TQ^WKVZ`WA)M||=ne!=Qf`~K1Qq4F>BL1-3= zrm2RwBlmeLt#iup97^M=H|lhMZX}*u-ten2-iK2fbL=0;Sd{x|6K!5CYlH9EjPW%$ z^#7>Rn)uqa>)W;KH{`#y9L(_f);xWlHoO>S-QdZ6t?q9i_Xp-As*^Y2G!LAgYRY}7 z=Lv?DRr$M*d7NGRXQA?BtmfXC@!EQ;`&>2|=acuiJT&D)6!K$#6323NJwWPMY~W+) zx_92H&uTN|#iq?)Jlvl0`_p!|ccm}&p$MG#O@=EtC8*5iRXqDp&>PO)!1H(Cu_#>u#jwod06*VzI?$!7)W3312J!qOFsY4 zL6vuHs;;~7${cImwj6+aJ}2Y=!fQ@!K>3lW=ZWqwpxU<&9Ww{5`6q4dxwn~w7w+I+ z;UpVW>+%&ioxDq1I-C;ZZ(Vt=@B2vR+8qx``JCVMjNL~jPQ4`Ec`3prTssh0ybg?f z1&$(=w8Ssh3uzS`RB;kknaFF8g7_=VHJ$}f198cky3(iJ&(*%n;k(^)VDqpKqkI6R zxcRmYV?vJu#jJEc##Y~tGR9^sW8ITC^jLd5T!&rxA5P&3ebe@XE~yXVd(7q@RrWs) zgBYKM*qX;qjhpZXkPTXvd9md_@Jc;$$m223JXNQ~#!#*`sJf-!*V<@c>dC#hZ4JPV zdK~ThTf!M{z1|ZqBXMK=&3NmzeBBTI5TW~CW9htgWBD>Zw9RMz9GM#DUK4dmow{Eb z4Z}m#%_7itH6jL#n5F0nyAsfhb4XBdgeM`#RYLnChrEm1>G2(5#^MW|_Ow6V9h{51 zsFD{!(Oy}cCA$N7EPUj3Gc{K9>T5XCKFyK1bJ%bU69{kYrK0R&d3NI43%3IwN+o9* z9>4NC+NtG9&i39F+yVcPI^WlwmU!vElCghv1_a=~pHRpA(EGRT|5FinfuHz^pYQ<` z@Y{adZ`*$Ar+#YthHv-=-RPe2|C0KIPxyrGectDNwtxCh|LKJ-OPSC8oX_3f_W5ty zKIxM`Y5V=Z|Mzcy>u>$7?Jb}2mhDsj(@!;z-~QWwyYv2;KmBJsk@%PY@?UOW^o3uz z{rJE57mnwK`e*;_pWXgb$pG+#|95=HcWiI{&;GORKmY8{-u~X-`+GhBd&BErKg4*) zJKnK<(kFe=_B(&)@7#XmM}B1czz_Vu?d@-W`}SF%`I+0BKkd_e==FPk&+l>GKlvyB zq~EamqA&U))BCb7`!d(-vp(yyoIZf|YFPazkhF2FyjpkYe--p)tP77F8v6V;jZ}>7`3bRoVAtjSoMW)yzB4Y zVUp3+bqxF9v<)T;u{|acu@T#c1nc(Z6AGRvvWW5Itv_YeyWPatR|i>)c*)Vi0GyI26c1&bnBy%XTY&s zgq@{zdloCRyaBr%m_YhkZuZlp?Lc8^$Ze3fXQ(Sr80O6>KSicC@qvmLwVGf{ztC;| zYy*Pkr8+Y~?Tv&nnZ@-)H|^fo6Jp2P4=TSW!i}M2o%YxU?_FJ}g_|K4#B;t{tf~$3 zIX))rY_7}$MRXH6d+{9-q}i^pC9v(P7PLN80gHy~A$j51Zv$rYD2G!l*vF!nzGa%> z5$p;Crf;bF0!U7406Go@Qbc+ve{>QZ0_5fsj$Mdo`UrLDr&051hSI-myUUI;pf1#b zdU*q=HUskMl;zW$eb6agu#oE~RiVMS(Cy&Gm~4TB(5+26FS_Dp^F{r#7J`I(F@|i| z6pK7R>FcpcizjG{E`8KY$yof+67@*+QcwGFmtUoJ#`4E?nf|$aW2O5?(ZsntX}JVD zm;9Y}4#`)<>%Yq5Wet~(1SnzxWbNWnaT-)1VSTaT%YQr!Kt&1 z1L4`2GSG?Nau{W%86!jc`Hy%ZQ?iuP zWmp(bv%2T7co`ICd|cB2XmbzXp6pF2#%k`@ic@y$fN|P7@_jCI zwO&ib-OtIIxJcXI%|sh`-X;A$D?LB0`zN2OyRLi4Rm};n39s|^2M;kZA7cYhv_%+l z_XN}ozO6&gJyaLZTR_#3&BNX-%)R5xeQ?0S8#!8*b0Vp1vc%ThC$S}L%(fLB_JI`l z%id`BXAse)^exLXEo53B+=<;$xzPh)YPr4n=y_J^1NIskp|E#9`~g4doj0kmea~5t zx9epn9pD_);our4zu21(u4Qa_E!Wh4+$FrjHIDKNRv74h8@=U_hR?})Udx#2JA ze@J=0)o>MUHG_Dg zTjqM#uHEn_`)=IG*Dt(olFk_$&^2;U2ahRg#~yE7=dc?mOrU7Eo~nC~zkMJzKGi2Y_eK389iI^=-uOkKFo(Eirn z)LYjLWuZ3gDGys7>y`s{Yzw~V#`kRQWvUOzCjseGxsMCg$9yj@^#6yZoXcjvWX0&b z^sf9pc3JP(y6^WHf6;;UWjEtRW5D zzx~@Ce-3{9$A5hLhyU;&+RwojgXgdLHNR&2w*S|+ZQt@YzQr9J+LytTPd>SQ(>H!& zg|_WW{>UHk!PcMu^M8K(-oNvAwvYO#kMhLg*S`PzSB`D_$)Eg5A3Op7{@?$5PgH*J zfB9b=CjQ;;ez)ZSKJ-IBRFjz7+aLR5e{B2KZ~M0G3%~FSeIuvdFALnB1+*!SuYziC z=W)J6YZIt4N65fTr=M=)8Xo>Cv!X6D?4P29mc=)(IbeD~9fRbYgdyaKgb*pmbj>3- z6x#YM4mnaXRfk9T`DR+^L0a-07X&IE8orV5`3|hSDmWL~rkR`exB(~(=|R6~3$tk0 z%ijLvnR#6<7rSyMbYMiz)LAPF5%(7Zklryr184H=qDG)SsnJM32bI_*?gpclrT4qm zx(OeMzg%q{Nzm@G^Rm0{1NhrUY#=Fz4U9u6Pjnvkq{oL+n#5oW*ef);oXkiz6MOPv zlL&#>=1@ut#M@6l>!<4abAC6j-`Z|!Ve|B@JYm;|QrQrb&9>~|q2GXQ^OjpO^&;1s z)tZ>GIAbx!M3e;@NSG(37?&|_>cGO*Z*pmI@&jyK7A;+qYHOu!wkwsBe6P|7Bpvg8*-+f5kfI6_1eK~+)V}y+f`Y;E~K9u7Bvw5Q+8(pqHd_}+8J8hl*BQ-gnpGIU6 zSeQ+-)-7+L%>zHTiD>v>jqyo!@dC_kobkzZ7**&>ahZL@{x}02!=Eu8)4KpYCh%MG zheX8{eO#j>I!uCd=7*oC_WPI2&aVsl#I9a^PZp>?kdkh)F(XtPwGEGr5`MTtN#%sDd-MRf?P#fLw`#h$PIRjCB;CGK zze@=g`Pz*?{O`l>)&Lun$8rgg+hwLZ`hnfOpSpWvOZ|sU>|pv7oA=g@@&}N$s}(=} zKl6e{a_!Mj35iQyM;DSl$`5~`#_Yjk^^dqJQ)B-`2%n)R+|BPRk2kC|Ml<*G<`nll z<%1tL6UN`U2`rsSZ;Z!TSK-!t5`96t=uV)2rH`@U_MFOlvs>f)P0hVHjCxY@!l!QB z+OFNaygm)%QK>C(N7LY})C*@i6n+p2sMkaLOz@ zDc=A^5Qhl1qslS!EoDhZAGR^+2%Rxj3(+IUz0%0$Cvn&w_c@PGB12t}pSD20Y7g5$ z*n~G>rFpJZdE0gSkyQj)Yx<#&JG!sxUhRjk%rEWMwoZ{?bO?pUTo!P8CAQm@o46=`{= z+^3aW@)xvsM#)#@S6aAkqeGu`qRQ~Y4K`HRY_Y7|Up()UU1UC#|CPmj%Lh}>%(W9` z9+bXV_xLTTE7^o+GoA;Xv*DjjYx;n7Ykg`)C7#S{ZLf@rioho5cl-~jXQa#9S}(AE zd-ADk{$$@%a<5;{r~3Q}bmj5-P32ux+N;odNbLsT$1WIa9+FJRmN#VbCJTL%afQWDTae}~$x7vgRqCXS<*&E~7qh-5(pJ`?WZ2o8 zpv#ZG%R17u6R$tJccB&G5#*kohe^mo ze&5d?deF4}*-HM$^p$)PF#pr39?FyatdiR9Jw|Gr_SnoC1bZv=L!3%uoe3Y>6B*ef z!G4XR|H#9~Ln)!_Qfm}rbW^IUCg}R9|9OndT*Rnwud67>xY+kH`1lRT+N&YnYc5dw z+us|6mDYKjf7}bnSF)~R1{xc`$u5H%3^yl{*#?#^zUb>7VmcA7A!p7@iY2 z)~Vs*67MB2gFP2QBj!W^*Y09i3ZKn)sOSimCyuU%d?&CF$4mCcmjkmLeE?)af!);v zq3jR48K6dH^5n%rO>{ij@kB=x6{TSdql5&sy)p;#gnHeDYP7u2iiZGc> z!}3c`0xWx(Xpe;p8%o07P{MYR!}eJ=dc^bK3Jqt}nEWp1fTIjl#d&ER{Hb5}Jz(S} zuYTPQ)Flg$__Meq$1K-9oEUL;KcH_ir}5Le>bL2~x5nF2`IJmHF4KpxAas~b!A9%? z8KLWqO|=*3jVAOeI~@m8;)ia>xFAJf_bi_Lw5710@XOQv=mE(WT?vRwd_4B&GSPqa zt30?P-f=4?-8iVscs%VI$uV62B`mbl@r91>2mm|4rWnK7Oy^MQ`FdiwHfHFpx)jC2q8lTrErL9MyZ6xeMyIg#zkyN#k=bpU%#D+HqQ@1tWx}p2W zQ#W~2O81qgpWbdVN6}o2dja`FU{E&{2Sc+?<&KRquyBDpuX zm)r7YZV;FG;7A8WtLyAOE}e4k^V`&#BjkUka!6ATm8otV0Cpd?-Bjhe22F3Ef0cL< zjI?-6r#vDtXtlhn+;bzJjL9Duo`Px%wF&LzIg$EE=3N|0@dv%XJ#_tTA8z;A2d8ZHfRWQr)qa>qwhiUeyw$%#t+*UG)3EQWc4ADuph`v zJ(_mtF-Cf9mhIE;SRZ?i&6py4^O&M~`3n$Q&vrffkjL*#cj)H( z#O<~a@&IzuuYnQ=g}D7K>Q>Qd42#y|4m0HnnUmM8u$ z6T0}hhQ=wJk3Ws9s-A_Ut*P=~R=}`NgyYK9a+PqMwrD_bDRDsya6!H%)x?hw!@;wo zG@?g55&kn8DiEhUi5mVu#+D<QSo*LJ`!K(e zbg%F|-}5~lJihPyzHfW;r@eXLXMDzIY(M?ZpBB?&4bKCIWIyynKeYX>Pyb!pANk@x z;sGO=4!fEV zSJ;Ei7<4M<43~e2Thj)VC}Uu-o5*&ZX_v22H_RGX#|B&t)V?V=PZxzZ1ASx3<9H_r z4X1n!1fjW1G?>h+1H93B<{}}yb2IXS`+!3h3leBPV`6u}v2d;YRHA8&i2{qI@>>RH zM((1&$ZkFQW<{M!{;sEgD~R3@b;=CPd}cDSXn?p_(_hy79dc3j20*n7A;5VqW=wY*)|2M^CFi#}0XhtsN zjI3Tf33~zu5|7M0O$~1*U`!a1eSFUEu`Ao-SA_K`Kb|6dNcFj2d0vvN^K*VIdg`Bu zviM<Vd#V(uv92Vv>x6JuRa#bnw%j=(i`cWAk7^k5e_4B)#n+Hesx? z{_Nu_v;76-8R3G|r(B_`<;kxX@hnu;R!oG`URf0NLd6RN#8BH_ArGWW4&92=>@}If$XMkd;1)$Hf}Tk6hjE z*0t-n-1Ddx^^=aRSLaLqgi}7A&^!V2o_&u)=6uqH{+s_1mnT?jqr@9Rs=w=updf8p z{ILu8Ipxr=r7v{XpJa?)voND=$p?&20%~#Th37ap&th}dIh<3ugagBK3N9m5X8&-F z6<2;PGrVB`n$8h*Assl0@A9|dx>cFXF^EgN(Rr3I z`HZu@$-@{NUTmV~3DW2Ktr9PqMML?3$Zt>NjyG=qoOT{pyEMpjuz$v0c=bW$2}-7C z`9EZJNSDp39Mtug9o8-2W^O zbZ8jC+O8w4OSfkQMws9Xx1W-T7F$Iobdm#V*_)6zX0W@sY~o8M>O~z?7s_PpU&^ql z)cGa9)2R#fb)B=>$-S>PpZdX=)V1_gb)|9^IiVeNF6wZEk^fvl6Aexsok4PzUw5p1 z8gaw#j3I049ha~{G7`@??1x1(X0jRV|GS~N688}5f!(^#^G1sGwDv7@6uavCB=%`& zMs_Q<_z!u_zfj|}=W8Cbl_I&wgWxYUEyFG#pcYGPc^s#*1Nq4{Ptf(Q0~7NjM}g~s zirm}qRD9~;5dM+Wu_Ho0($(KQeg?arO?^4n_)*p_fIKA|NPBc$8eE@B%Uq`JdFYM| zq6Z(w1jcE%oj=!CZHH{a5_teDg5hU>r44XgrHy=3-kwX1F!K#i`MSO1(hh~nPhHww zX~2#9*c=nM2gn{Z*O+}b_aonnT(8+41E@&Jn>y^lxerBv6!X=;QLx-KICNHgy3Pr= ztvP4PIWHc8k-wHdoi}f+Ke}&_&oWmpLl$G&8wf-Ui+F? zZ%@e219+@?{MM1?MSPOWx-4ecn%92ZPZ)FMN4GvC7QdUU%Xn+?+V$<~Q&+d^a(OG> zpIg^`)|>OX7eLANq)pov<)3w<-sm86Z#L`m|E}u5On!g8LNX#B?Upv~^$GtwOZRu$ znnNg!M~w0AOR@f@{!V&j=YzqobM-Frt_A_H?BSTI@tmAF#?6nPWHh7d@9y{m}_y zTgEueH(tQseUSTSXwavbuXCUAL6p#kI5lSn>F?;NX@_3=LZc7xg*~_PJz7B9V?+HA zZS!DF&~FBMA3{IN&Df~&^fm5hzONN+A3!ONblMx>vv<(NUh^ry$G69ycw&3(iTGyEPeZS&nm zf#FuW;rjlt*rP7a6MbZD*K;D?fXrTq`kq{mvr6Os!C2h-&$>ai5_bLU2kAQ=W8zQ# zO7rg&kjRMKi;NvUqAn+K!7AgrhiJkkHn4x~kLygi#<>tkUb7st@9WqaY6On%y~0YP zK+GfWL5d2_*hBg~Kyx$dvf_jp!S&QvKjqou98(*0>0$!rtO72Lqr9sf;ldeionx4dQhqyN<(_5Y~?c*AK2N5L)-^n1Ved$-rV_O;tLebYB>pZH0i zw0+j^{k_{i_y_;Mhf*VZHRhS(1{+9&+fQxkUuw4E3duSR?8NUpi%VKT4=ODc{x-Vd zl7`p87^NoxOi&o4HCS_V=%@3?z?}k!VV(@gjgcdXQ<@{dH&lGJUq92o7`sm9Uc5cY zVbGS_3k6RWTNd&fijOe#+ASo$(#=GWY7`whebF>;enmV*Qr7gr<}v)?G4oZy1wu1K z+Z{mOefESa{+RS$h30eMGD0WJJMYBx=eSNPr@UtYO`sclD(q_#E{BEiWjuCpNygYh zHu`MyUL;TS1VWPy8H?MSCvKQH<`w&TJ)I|xG@-m-lP~PTbjKy^WjZeJh5!B18W9SM zgBKkB3a~fXa?_%3L{1I^yos%a5sRZSF*5J)<#QIh zx2F6AVLoTU0o9|tIiLP9VLjb6~R$Y3$k?Zqc5pznJD zlX&4+q#z^yM%_E0*t+WWj?R%uuF4P~$vnlUH#r-oZE2#5W3>FUx0CpAU zz#bAuUec$V;^h*Tr1G!vGfZ(%p+VD7PQ!?{OYSORItL`Bbk$=ti&ifnpJS}7g)}-+ zTH9QYmwQ~&Yy8J9dw$R^@wB6L&tld5*c^fD^2qIU^e@~k`r%*gzvH2D9k+`IEqmnO*$}jc4DhF| z;m_es#ZxQU!?T6row(<^TQa`u56e3>h75S2s6c4d1nWCJ=6 z4@~{>zlw4Ym3+1d>{fIbw;G|H%}?^O>8X3YH$CTul=UW=EH|ztY#=30tD>V$VAONM z$aiTb65FODZ-OFAa`T#1^3r_&G)XpFMtK1++p0tfV1Nk5@A2?4ijgd+&|zcfAmN} zc!`_)2hQR;vC=9h*y%eUj}ZH@EtfqTv8n1mU3SnnSjdtWIU_6jahk75OZ%a-)C1S$ zoGY~9`8;o9F}LGBA-~5Lw;O!Lx%2{tM^gZt$lpkDsxNW7tPciC*33I<%}u=1I$zo# z*m;vb^OvfV;+dnpBp%q9`?&1R{ady;4lACmd!XVOr`&IcUQ4sc8E5$IEl+ZB6;_b} zm8r`BqkLU)$rhxJ?N)sY@so&?!jl=YW-RM_E%&<2&v`>Co7H(MiT;V6_{2K@LyCi` zC!XMN>WO@|j{_gAc^-PGK2xT=V{;r{r5~>caK+d2I_dM4_^~d#cKwDw)Az1lcv9}w z{hLv>9&$Bw={`j}j=7!nqF==`?|!(}l6@f6Ysv>7%IEr!hjwNRX6*Jl%m;F5=XHE` zKQb5LVo#0cYh9<#14a{ZAGP3Mqf>b`Y*C{{lg=9!q-Jgir|0| zvBb)|%GUm56TJ@Q?;Ze|OMW?{DuR%P5OPj5R9^2NNGIz&rrp*PfKIW$youXyNHJeu z-;T?Ak~RQroAd?l)9wrD4}Lpt*g)oX2!wu_`-e7fDaG}{jmHzwhqkH^*(0~(xNs#68tx;DorB$$v>p0DCiVj8 zcgWR;Q@;O?Ie+GYnGdEPxGxm8-|@DE+;4o(!ET&}n7TAXP-DwRT+3g+{A?#P-^Ap# z>vY2H=W1f{R~nIB-hSsQzcZYNV^T`ZNNTp!POuLJI@cjB#3r7BU8!BZLxB$ZTb+lj z_Xr2+E`a3P$t7?*8{@G}@N55y*vhUnFvBgo(|-Ne zfBp7TKlM}FpZZgOYWv|I{^9Mn{FdM1Pw~AR_?B<^77uRU`fcC3eaV-6i9aXE!1Eh_ z<8N%8-!qW?2fqLNeIWILANYaWcYW7)ZGY)6{iTgh_qlFv%ritCb!eDf`w853x*Tw} z`$S#%n{gQ4h@W^0J$P8q#I(5OVG!`I^392HZbayAAn{6{^td^!GV}nO$qwmu&xr;= z$2@VS4B@{=$CGZpr3(gXum*SE%s~&}kwq8hSJ6UvOuAW+Oddv;5V$UdRnNXL?CV5{ zl=aOGdiWPU#O>-Jw@aP?^jBE&ovu;`)6CjMej3#2aj zbIj@hO=CyvK$Pr_C(wDz?ciD_HX(Z(6K(n17G)<)PLT6{)y;=eS}^cbQ$7RH-$~Ad z?>_Z0b2$hTTaxYgZ4{-Kp0GVOl=6-8o7>fE*S9CN;Ng=HPrmypf6~wYXX+U(e3%?E z`H@`6I~E4Gp8Pf{UGdhf02`%GAp3)wAaW?x3!(m`1qV_riXP-}P4z&&{Qs-xGeOiM z(33Gu%vgl+IS_au2bFETx>5@e+Vu}Rgz+n7xWP_43vR4VbipcLSiej#iNhtP(wI}jp!64V;9bv^Qqv_2zrD$uRMN{2td_)}j zGd!{(olVZ(tn-GQg;x{h( zJu)Gb`$onIHcZ$Mx^erqKgW6VW_`lXPx(FTMWg%e(l4QITzCV6w(@zAXji(k4%3bx zy5;FeXi|#0(w1|ft1(Ibu~4hUSL#;0g@z@LOUIA6b{&DNIFb3T$gf-o$lE?BPN*R1 z?K&T2@#_*j5szCLu?25(N*`(mbnS!lTB!FMEw;VV|CnbCBn{haoN3N)JEKi%d=*VT z59rSbfhqY2|Qt8|FNt;x^ z_hGc$@m5yRNx78w=CIw^_&5Nn_5;&~H3rZoLk6XH`xs|A4%~c5)ks{mtAFHR^4tIb z|MW>jK~&IzX2}5}H<*5gUv;uP4UtH=r`U044Bsb9tdZ%m3` z4vjcqng>!olwzEfe6ibQ>?gmF{Ljc1o?(Nf?3DD>kFv_Ixb8pRbm9J|@Yr+_Z3z|O zN=A#k0Qzij*1c+Wa40I6w4)EOb+d-|Is8EvQd&8DQb5z^R<4xpLNXyf*8!g9i9aEX_0?7eU7Z_ziW8ra* zC-;2r5%p_uU1-dM?2fl5eaLw#c50w}N<%MR&mH>v=!=b&-!4@^7RwlY7!}rEfaHp& z4n3F2J&#O65pzGHPt$i~+u77l+c92;CtN4EH1y3cJWbL}D}yfflFkpe__tg+V2a-d zf24Vi7#Zh$G4;&6WpRBE2eZ%PKCZg3cAyS57^qcyndYOL!Ewk(3iggNY zP}zaKS|%v_pjDT46A0bL)U|jGe;1G+uOYU8?5?NkLYpz4;?souAE_M9@IR!Uv#`7= z#oFcyZysq~!zQ`E0>N8SY?P0+S`MYqrR9K|>Sy~DKl7Bn&@K1TTbj#VyK!TC^6Iti z7k=U0+t2>Y&uzc(^S`h?_2g6AEv?z^Jo~h4>W=LC1y!N$`PGi$k-5C%b?N@rpE~oW z)E=1!Qas?uMn1NK4roKhZ0gU$PW>i+Z;WT2lXiAR29LI8{PHFl zdl3E1p{A~kWnYa3m7%OX*?Bh z9dQ!ZRiW(Ai~aONGjl2#(Fx-S_Ubu}T*e!Zag2LPCtvHKA0Xops=sc~|NqFJ>nlCM zg8u6SL4KaFo zxQvbE@1NTr%(yzkL1iho@B)bJ%daYu8N;q-@(lmopcr1vH}L<}pzyx;Z`=P*48Kbq z@OLw5uC!kSaEQcPL|^xHU$-Oo`JeZB+xxuX4ZAm}{_p?Yf49B` z6UAimfFx*Hvf!7$-9@IzbAe3I#0Ndo7c%%S6B8p7bz|Qs>*f$KT<_*L2J;H*Z-RN0 zYaU&%Q*3}XL?-PyD6I8zkpm6WE?M)X2TN5K zL~Q#7$0DgVE_)1`{Dq>IO^`X+kNtwZur6U#9B-FFZbv8$%8+*-D*o_GJaLIfrYQ@! zz#-Uh5b)Mv2erMpoc+Eo+y3QezbWGi-H}@0jUAQA+Rf(6} zAeV|4L{HUbJFJal*<)`;z>5hj^r4f3a~8R5z|fA_Xm!2Kvv>)Hx55??e4Gb zK~h3@+QEWm>I}L>)xCY${?g!*Kgf@8a)*E1iG+dB61HbjH7)1E-C-P)0C5~QBmGm@ zaXYNBsj>4_DgRZz=(OB!&u(j6%C+k>7j8=9+Aw_$v##WIeiuo}TuvcukVE+_7i~Cg zF0!?JLE4yj-n=m?U*3k9IO+rw`cn5J#u)VVL*7b*}_$gaV{v_$PH8GUGs^adg54KJbM_yn>y%rp;R z-ors~Z`yf-knu)3%2<K7ENB6aslZ5E%+Mhl z{}N;Da>6lY%5UsnldF6n=>rxo;3}8j&=DTfWPhyrxHo~^r+Q-%cli7MReCfXNclo2 z^2cj=Y{#?-_ebnmh}#F?b{E}ONncw;(<_=264~u(q%YE!Y(uf5Mh+f){}m<=#6uLP z$=k0>=V^MN$C;{o7w~)o`(e&O{Zv=x1=$dn9V$L+n6|ByTN`Nb20PH{T|mw(F<~tm z*vClPQK#0?@Q?V)*Lkbt3T{W{RE$BYUk=i9NRDLmPu`AV{qp!@SNy3y{%?4HLM(4b z<$*!k16?*`ew?{KI)t{@U+UkialA3Ewb#vCxBSHlKG(+~)Ghhhoc6vDRC%i=LN_O?=jj!agiRK6H&duBT{c%*nw(a6jhE`E%(Mo{I}wxr<&CRq=)X zk;TYGAi9uFf65rt<9hRL-or>t4q7D#_m{Gp#ttmWIjSnp;m;hsY&_Bx=Sbwv)G zu-hD>@7~L}l(hk)gs<+!?m~1>;@y|A7{LH;0_l@Y=I$bBu3)!l5U>`;F-D(hJQuZexcnlk(u|Qf`{SVH`^jBy8w}iWn)o3}@h&?E)y3 z-vgvICWiK<^p`}}^AZTfN&6gJoN_MhVp=hiq}%M4Y@O73W*U}QfhBIhh4oWHwe`Tt zTz5R0v@#g*SJ-hQ;S3;vaIL`waQ}yBAp6fRqxFk}yGw8ZbUh1y_Gf?AZ#YE<=a51d zK@xxLpZ!?87v$}vv&kbdo@$-#NB_w`*{)u@>a>nCB%3e0_yUtL98&}?tBcSB#0jpY zs+~*GV<5%#V5&jVafES`XB053{!8E_+agm9Vj0vbS+Mf=1irE2^jBD(BNK>qeN7w8 z&CvrggE&u+8QY(M+j&i-c=(2~$Du3unt!ouUSlB7jbOw(r@+OkF0NbZ%Z(@WL1!zv z3vHoDmtXRdXZPDfbhLNvC(YMAgdP_&IidGng~^PIVfbDF<4I0eP@xfb28I^$w}f{Q zr*IL$BV0h^SUk;(?E2@l<}1Hk;<^sD zU9NS>7}Isj%X%}}(VE1A83W`V=R-`I z0?Wl%i!NnTeQ#TVU`IJWZh{ERB{JLBmo-naMZQM#Kik0N7t|$mz!e73K$6$86m5Ue zFJ2?vWeO@h7lVkb4wA;0$9U(>nq}Mr zT_AN%zw5S%PTW?F(pATGeEKkpU?DoSezQPk4DAho=$pQxaw_xEQYZ6wU8!!%5*hY3 zInxP5M!7AE+R+;uyWZpod)<&q6Lg`ulDE02gTC|eQVxbc%0f9oxJ zj6d{3-@0l)IxD~H5iM8`)DMsA>-6MrS^Le+k%qg|%`CUa+;{!v@{UH@)@2Es4Dr&x z@(p=G={K9>%LbC&;MC=mCv_T*z=mqybPHytc7|&}MxkXAFSj{Q*NgWewMX7NYc-h=Jp$pxb@L&ea~kZU+uvnd1V~-}9IBN_GmR^TW z+dTtSx8$HD5}r_ckvz_70C+%$zq%PKd(31E_1MXLXYef<{0qrh-l>j&dg_`w!v}O6 z`jDYDKyq{+w7P0&)_l3lhKplG?BKG&E3i~Gdm zkNbZ~_5YB1g#T%?ycy+j-z96DvfgTQTyx`oAdo)Jn(O9G)?K&i|NGw9Zr{G;wGVG> zjEC?kzwM#y%QKw zE`P|0Joxv4CAo523=qGKu()PLq{p`l*(ov1orlxYX4bkxU zP`2Q4Pk!Pfxa`uluJKFb6?1!RfkUD^6q~V(&&a})65Q{h)dx2IdW&rSA=#M^W83zN zzSp?2A3tC!n%v9sC6ux;A5O)-sfTKw@eKMLL_JdfwxISnd?3YGtuc-yojgCROb z7ueBQV{_k>46c{f+BpbQpHe)0x$jaR9*#9W`snsB2e>|DX70m%LHhF?EbSF`WvjGF zwM(x9(l*m}^*8+Jz4Hc_@rL__^4KQC%l%or(MikEn6^zDJ0ERcIPVc)r8{rpVjnqR zq)vR92R)2_jXhSsO8>6@1(s~FHQd-8TEUOajwS(|p}fl#IOHri_7lOmY``vF`mN98 z1!51Ua6W#@hE(~TW8C#UO*`e?G0kzHF1Q!LY1)i&`9@u^mxK~-=yWcTq!Z;<#?&Ia zCz+ik?1X}@soU7-=mlnk?|Q8pE0DlQFAvU$`{fO&Bx*Si@11Ih%0+})SC=q4n>5PB zli!ke2_(fHW1JB?;|l}BGayZU;Bx|GnyJyRei&(?2hR?-zw$V@8|1Q90h0Hke3n3R zv{7fki}XXhlA#B5ZYW-OwM>Q1y1y)t`Zn#p!DpcD!FysRb;(cBRH1osQ-QZ+FmeM@ zhLvUWGBzIucKFUTU7S7SLno67aMq1q=yyGECLQCTn{38hcCwjT(ZqFJyS^?^#mkXL z^vk=4laj9-R=EX}cj3!~QOur%E+R9ufTx_m5-}quoFUj*&t;D+h0_^8bI>NXE!+Bw z+tYk5p^!-%lc$%oka?Bzve@iTMle}pA;iSZZ%b*?k`1vl-1I-##pPh?Xn^xE2jzdgTE)UlCA&86q8y0 z&lC$LA5PVihSY-z812Vl8K11-^BKIg#W$ay;hX9#)Few!D0M-TrRHjLT^#NUI&j?n z=XmmyZMoWU;ec~1IP%P-J&;tqoX6qB8Kf(8p2dI`r)t;!r||xE{q7TN*_`V>+hdEz z8F=7#yY|GA_-b2^Mda;vPrkMNvv5fsqj+$&TQU!%tm7<->);ErxlWPMKI}mg?cS)t zbvp@=+j8IvJ%N^8U>oa1dyLI6goR5}J`?J$uLN_Ro=(BrdAoqXHfsBvHhU4wr1HRj zIsx>6yW}N|99_3D(RVm9cAc$HW6{|GM1wOe`6)2V3Pn4Z_Ut~*Zx${(TtdQl^tX=q zgv1NaXXC`}=LMw}bhTLZ6O{Dv8Uv7nw(^EGi$$>85FIGpvLZuR71L9QD7xB{a&{dg z>|p%-MqK-eRd~4q9U8w*kbeeZ!|Ik9i^sh`_MBhzouD& z=27FxkV(H(EKZ>8?+ncRxU-N$ii;i@XXhbPyWWh#2IWVt))hJ@%!4P%`{4G7@Zs{e zy$lmJ(g4JBX(+uQAmBX8jqHP8A@Rn} zYGVG2TM34aj~D7`m$+tU9i-pP11Z@dD7||O1*;t#PafKWvXMM){uYR=$nA|+-y^e; z+iw~SJEl$)fd)j7ufzV?J0N}%mDZ>>P5aJ1H^z^ump2^?oo_XOR#piDj~fHwf?$(Q)_~Js*g|EGQY5ZRK@KjRQs;&)_#x0vS+WgB$vNN;Z63e6|sG2Ak&>Y zJ=WBx3R!1q-0?$^vghMEglaKPh5zCTyUyo;wBTZ6on5qJV?7oqY#>hb7rZZ(} zeN@_D5LOPyV`st=xg>16~AUo?%(s^G&vLF^RdJds^uHTN5&Ds?YANndSb(wa5R#&Ab zuVv_O9JUGjt1IKdR@oZ;h+N1V9+9l5S(AY0O zv6^itBp-;3?YHe$Te^QsAFgj9^^Xn`pE*_T2Q8luqvU#?rF7zK2gKt#T

3qx|rb zF5Q<$eC)F0)6b%F{SE!&=i;CJLHr$Th(CD;loz`ROy8ModmysWK-s1I4larZPN^qm zCBFL4f#|D^hX@s1!^TVCqU4jJgK)sX0*{%MKB5(N{D7nzs#>m@-QL&eG^Y$ms1qkJ z;}BFr{_J-rb%0A^eH)ZK#KN&Da_3+e(bhgGR0kCjW|gZB+mS}6z>dP^G0IyufRMDk!L4UXxb&5>|D8= zg6mv77+mFg;9byzvIkslG$o_}>gK86oQNkbH=5!a0g6x8FW3Wo>UaRtIYs@MXHgeS zy73>de)U(Ehy_9>`YP9HaVLbf2(BTA6SKl;%r6SU$X;Ma{S~qr#qd-e&XFo@_-g&k zkYL82z*)J@Wjw%Au6fzQ{5)j&Skus*E@32c^43^s>&>u5~YYO^wvr(Tq* z{`68Cul=K~v_utF!A(E5ANb!fu` z(EW6^Y1^@|-2Qb^rTUWC4p&41Tmr*OI#~V_*vC!2Gu~3t-jbKL_IIRJkNKox4kGg; zSJhAX0QYydUEE9*(~ceP^fB&$T~^rfZ5KTj@A7wA;hdO?lG81Y>oTzF)M#6l?4E4f zc9YYz`Pq$T;Eg?cq6}iYWxv$N3@Pq%m4C74)H(GC#p$?!V3iHj#qoy4OCY}qMIvqo z3QVhD=1curCi~0XAwtU4B?Zp`=}p(clucYE2s&+m<=R!Oa#KEeS0rb*ncK~6myI<) zVW_czP0?PYrcJX+Lfv|yxPFGphbl|E*v%gygsJNuoL&j>>*_G2pi^<3w!~Lr`76Er z!FVRkfHhT@*a%hgB?*ha)ADpA_8py#Lr;%;%`UubqwZ^EujR%cTZyi4DU+}lnbIYF z!Uq)Aa}NII6OuWo(1i*V#4k$2wH)o5KW-Is8crhS*pPg^m<`_pwp?|9g4-LWu2a=3 zw5py;_Zwm-SoujmmMzS96xyph#tqxu%G))I7F@s7Gjt+PWB4z;LG&>&aXbFVO_}1` zo8W^^*NO5a!;UZIL?id@Y#gm^>&|R&HZ^?B7AvujCtTF8;j{ zvTlf2rmDlVk8CIzb)Wa4OP4A8$eg?QOTK>ii}9q#A8!UL-twOcO7~Jy&6ypZza5mG z`1!ABMTd?`>I@ihfW#;7(RCh2Tn0y(QWl+AR}V!u?k~_)4s?-2seVJ8|0kt6f#TS_ zd!DhGxt7N3uD?O$$5;Zb3cC*0bB$T(+HUUCJy#nC>+)yql`?LFGmx5*?m+Mou%;?s z=DQ^eXuhHbU-K*8_{i_Di#oot1lko)RkY|j*BEAfV4v8mkg=*a_{@wM zhO$#=0ODqVG?7E{+FqqQA4Gv?=Ep~X3>T9>2c(%@>XI=>?dg7&zLvVAK5>OLR&?LA z9txtfsjQ|w#=+vlc&~)Yl*?Ll)O|;*F7Y_-1f1QJ>e{|@kkoKKh32y3CJ*d$|7=Vg zc|f4U2K)IH;1SEN8+oCJZv7^O4|uK=^7hgC|D$qWM)&9uVy;Emfxjfde2O(7ovDS)wIWY6RNw&?{!9#IbUvXfNH79E<4iEgM6mvbT4Ur$6OuaVG8P;FqLvbgN zME}U^@C54#oAMX{R^KVUX%uG_OgbL#%p1Fny|378>_h}g`ZprL_Lk!ulplV3LpR>a zm^7!4gbTpwGgE@$6RiF^h$S!8M+e>`TJ&PwlzyV4Dy8zPJ@k0cc4;00ia-O~>bX7- zz-EqL2a&!Hub=G8ffVCq-XO~TJqIH-=IBSiHf_d5>|JXI*0tYw_<27?b?m(Zio4&4Zw}eg zSGs=Pr*lteUf~m7#pj5MC(Z3fUhLX)oi0Oq^3HJPJfjVJT(5hS?^BIEpAyDZTT3T# zyZs}#-C+w#Tk<%cU3dmFwsha>xCJXd{@9#OI*dCz)9^ctS$+(_$lO2N+~0Yjb(;oFqP?- z1)NvLIUwgT_q~^X84zXlRph~2O zeFJRJWxz4;;T!(?OI%|1V45pk7)^)t9^iUlG}a($f8V49yGaVJ1n~fGcj6gI3YszA zMF-*$-jajC`h<7!FMch9ZzR3IE16(#5zGtPIK`*y+2KM%$!;2^m#3;i6YRPLES1Yb zBM}ZSG%M&dKoqQ+rem6pcZi4}&@}9ASqG_KBAC>7d@Mw~F9-HC5?V>E{%C>v@0@X6 zBF@63eg?YifkyL}W*wfOg@SC$ue)m^_sp};)k1+cg|bP(=j)zRJpV*pX5~X3J}bf7 zM}B*%UUg?;#{@2ux7ai2iM(t*i`YW>i9p1hU0%=X|cqDi6^=; z!Q}8p{iGCZy;4_x3QTlL1^A1<^|Ja2oSZ~id!eI9XB@7#bYP}9w%t_ zBn~%FX}aVPbJ|BQhP+TfPLa1<*yP|D8Vy7tKMNubX0uT0MFgLasD+Eo5{Z}0q`3EcsBiWvcE1B%) zhyR#v*N%WZozEA?I(|tT3wU+gujV^B$Cbx}s>jYPo+|cY7M1vz;qfeZ6fP5iExN*gxeD

** zt>}}#Xza^|FRt^W8{{-C(ukXMLkl{s2W+9}v>ZhzbVDoHajEx|MNa8;4;WwiyWRXv z*aa2{S<@T(EtxEz`*(22V>u4>asH+LlqrA9Pyg`P#C>nv>$uN>2y>B;fcTX!F!LTY z0f=4P{#o9l5>KU11ZKI_O<9LJy)zsDsYX3hsT7kPWAAwqJWrT2E_lq9`kEmV3HDG=-zk;E4}p&0zJla&*(AUJme={5=U0&21fXIVK^mY&@`G5MC}&pvgcXydc19%X@ig)hblhpmVW8Ci?kkLEy5@IUi0c# zZ?AdciR}rH{~^U;)FWD-NdK8{-Y-H~KecW5hBF60{a{blRP{f6wVvXEKMtSx|51D@ z-RnE$nVaT$8-4ca)_3*ok{P?O+!{~(xjqif9>3y;{8&Gs1LWcXIAnmo-@<%4pHpK# z*S6gANbF?j2VQD>_(3b}TuLmU3A^!Jj$i3E_wGs%*amTFdw4NdUFNGnXDOJpdOV8^ zk_VZF-BtF2!Ls6a$(^{QiGnU*VxSp6(d7Xa?|_}W1>`dgd+b_H%N$*T9S@}L*w2ED z`PNgRl@o>(i@moE7uJ23c7s>T3?1y!E`2R!?b%@5HjH+JLu0sPf}*8^MXU$M$m-{eTXc^0W=)oM!`P13fOWODxjx@RbKjwO# z9e1^nJ?6f(VA*N<7?>uAj==Lk+pOUH7iTx@DWd=K3hq+$uIVoqU_ID7u$Sj@*p)88 zmS)ODXsEy#3!Omu`Z?YioIb;j8kkFg6cqiIr$T!K*dytAy5fwLbQfSnDo}w~h)FiX zHX6rq`4tMSi+j|`y+ik`Xvk6b0vE}#qc8%|{{RK5Me=t+c*J$6(~RXIy&Z*LF%C|^ z)b=oN`yDm?hNK&ZgE02Qaj<#hDyIB7+>;6qd<>Ff5L6ly4QPm=JDhF;zCjPH_(CMG zvkJ8XGYw7L<{?}M;ErqEgf0`!Og6oU_Dv}_CU7jMrWR6O=m}*F=%a2$8+0k?o5!r% zKxGhh9U+C9unKsLu+s z{`g8E!{J)6c#lK~c2pJL;ea*c8K6m+wA|5x*H5rLleeaLnywFp{8@!(pY`YKWL5R% zQ5GbA&0Px-CNNA~vN@POW)yF1kbbtCxM;yk|3g*|M)*vO-`o*D7Eb>EQsVDV=qb*x zzcb0yQZ+m%Snm4BUQ+`Z@Ib3^C^|@bp=Kq+Q zhf;iYBHyFVlY9toyOmCGMqk)rH$gldPr{lniT(?e$4LXkkOsq7?rDQlbRR+1)>Whb ziL~mFv1RHGOx=2c7-USzX9i6D)8n0>YK0r=w#bcD;=qqw*PlT3MV=BgZlBn*&7RqyE6 zdiwvf_h&KNZP|Sn`s~`x*Y2*$afqubO;tHUCrXZEkbQ!bSRu%b4t5|yAYlj>C@~kv zLIMd9q{spZ5hS=m%0;d$q|ih<4#W+ua$*`ONEXR))_Tyjbx%8_jiA@M&e$6q_y6io8&pYL zT`F9E4tH7H_&dCl)*;HpPacy@Tm_oG!?@1um>p0L#+x21a^L6!sc!GyJn-Tktb0o8 z9)20GYT>&>Y$s3p_vXM%vkqOpbP7FOXcfI9n!~n*y6_*j#|5MEHm{27wAORVQZ8wY z=HYV9pZE&5OPuJy6J16g>XW>K-k{a~ieK}L`XEx|N(=44CfH-qiUo_e!8A&?)^p3! z`pjlV$++kif9jC@M%zL1k2u%c>?~WxsAc>@=2bYVXlMO<4y3>)A3V*sTvM^XXja+= z4f5FEZb1cIZV!&*o)!J~+{dx-lb+mvJa5aP6!F}ny@|$C<;?vw#u}<1^P_Bh8z5{&XVujLLQekj2=UML$X@e-tr6*~r@M zo_aKvEmWRCTo>!G3lDb%QsLd$eJl3cc02D)8E1M7WB%+Zr^3OOi5nTT!;_AfzuLf5~f^HTPlO!~DnN?xqex z5w-pb&7}BPrsl<;-HNLmkQlNQTA!Asuy{@^S|$Pk6I<#zeC40%TcBXcGRcf^obVZP z4#WoH*TrGn_$kN*NVxTk4&5)c?(ll#W*to3zAYEzbA7kGP8plo%1?X4i~5ZDldw0T zIq=P)6l*#Tq_UxTWweL*;%S#|58JWi;Xtj|Qf&HD zClzqtdiLxC@qco9_<;4~d#8tw^11N*ZXIj7+AtSi+lp1Tj4LnqxYGKK7M(rbK&3ek zWgQke%{kKcLhFmURBiev-Z`8P2(O}JSxTOjb`g(TY;+-vzr!;yL8Wv7D>Ud4o$!E- zw<^A=e8G|rf75Jw)}eJ-^g$@1j~)-1Q_>zr#>vu8F-$R)hfZPIDgCnTrFpczGk(Q3 znZvUVW37k;=##N44^7CHZstoS*c69GUYBdkVJspqV0#x$?5FQvkDth{WK--F5GI~H zG%oFiU2^~PgS5~W-Fz*jKAY!zBkMVM)i^^xsJd8Z)|dE8AFfyRIp0SIrj9-U5_mjy zKhL3l?!g@N^!OP)rv0L8>%7~)iHJ&h0we6QEu_26>EU zU0X6X_IU3+%#)G1bkpq}oMF$|Do@`BJKeuyIm*OV&av$Zoo~lg>&SWvxGFouc`g)< zI9<3wW1xZyn~x7KAXKuHETdutyWk2*;%{bKarK=;)l$jHHS^g99Z6z1B(AZOoN2*= zR^X!R#s<%$M$Ghw5`C@ljJhBpulT#DOhp$E#~}x~;5-ldhWp}JH)?L!&%vWAd=&V` zOa9%2MelBhCC2{g>Hf{t{srq|HvgnR)fBGF7{({wS zXqrVBr=o)4&p_;(%+X*-JhBPkHy1AR=HNkH@sZUz%LUGa5=4kufA}ytSov05@irOg z!>?{exw+)#6gf(Uln_)cn}-MWIiNQhini%5H$K;mG%8etQ$l6AZfGU!R0r(cg2{vy zZpe1{9Lxi39_82Nko=3lpy|*8BVwkUy6tuVuw>6VDY{P?-bZj z2z1${d~%*nPi0&EmK0C4-PZ!*o)#<*?mzTz4n5Yw<*61DK9o}VK9tHNl7lHGFxjAI z;^EDpr9WYdp8WLW#S<3!KgB2W^28H|KaaFfF>e-6S~&4U-$N~?@-66o^*#qeV^N_$ z!aX_66Id*Aawr0rzxZZDAWsA_(Ylwz|y3#mD| zbMs2E!)`(Uk*2FL{*DXo{7!A?kP}8qIOZCs{KE~5{GC;9#tZ37?wC-6j4vKzQm@u4 zfNW{Uj457-44FZQ7lR&4(2d+?P9Da$-s~85P?J*P*-UY`knwTwGL?X9Ks^RJvZT!w zAYa1NHDg=GQ6}qZL)j7w2rrDpY%>;mfe|FFylbsnAQzo=rw!-5C^$exHq~%MbP2hZX#>Q!rvP8m#$Lwa#+bh5 zMGx&RBp>&bu1NFp{b+Yz0uVX(p&)&j4HXubY-X@g%AvRq$K|K};9f3{Uns%P)_3HVBbXY?}(YcfZ`rb z?D+tC%%%gnm7YbzdaiKNB1?EpbR2&UR=82+GdEc=Y0BNL?Eg6BH^iT~icHjuFaP2= zD}0c@7q#BlRe$RKM&BE6OR?cw2TL5NP&RasKYH*OgNvQeFP%SSg6YFK+(};7TP}1G zm-MqTlCAsyX530e${%{8ylEwDx-ZGCI=7BT*@ZXhcFn)y?XPgr4jsxz59xP1U6k9h zL(Bf6J8XygxOM6Z(E+glT>kv;PF>N4HL&@lW*zMBg;*K(< z2lxOPo9IKi7ch4DKD8A#`BUAA4Z~k$c4@_`KyvbDdL=ip1f3cL;wIg`O+v+w$co;B zL~K^_ibJ~qY~bmlpM37TH?kG^r$4z+ol z_8`5`GS^0)?hA}pjPcB&%)|4wgErU$$(6-bE5A5I9{f=fmj$AW%y;S@R`V3%8#|x9 z0Dhc=944^F#V!7$?wQ{(W^w2=T=cK;0^RrAAsc-vWBCtT)PP**4V{fQr4%;*;*ITl z4ucG(4=0z4v_tl;fIK6eGNz^S(2~rc`V(4C+Fbr&@5q!f zr)9$5kYMu0R?;v1<%#ROEp^=w$=$kjD|l1>dP|BgQ+T78O>>ol4$@Pf%*~h^^d`0X zwC1<{mePH#!vhYd_&oc=d^S95x|+kdHsOs7oACshgro<%9Rt>|^Waq8 zMnmN{&c5GvnslyRUM|hpGSJR$+bK)u3sAuS0X!qH1(h0U_?t%4wBO;v4wK*tOV;u> zPDp|92)%*Sg*sTDvC%F+?u%tdeShq6(#k1LDrS1KeY&6aeK$OsK6Dx9v-T2g#xG&U z{T$k4oaqNfv8A@DtjYZb6E?5BBJ43H_YuDElKU6ygvWjhj|bTDJIT}o*~5b|O!ms5 zl+Z4=EA)MlzV&Lq!8F%&J z=X#D^_bRm!dApt6*26#BAHVY{e~;g}C$^mktKRrsf6pIU7hr6*`#!plh6?13HBNW< z?95naUWx()cTt8JuktI2D`2!7a#raLG@XUsKG*Eg6FMeo1!viI5goJ*t>Fa+ND760 z=uUH7FqLJq20bYeq+j?eNfz3%t!`rj(gHZ|Ib*E zO@~YvlO|0H0c7w5RWe83d^)3F>F>|^`H)I+eMm(;_^l2JU;QU_;m=g%k2hiemp@6# zSpoYireO6KD6dIWnySV%uFw`zar`^raMDJ`inI)-8C-R5Mq#!VDOk#sYwSPyTgE`- z3i~3YWO%Wl?a6CTcG0c<#5H`8cku8As^*H`FnG7 zER-2jnAb2~OJKRXi=b(m?aA&1dCrI=#>j z-xWg(Vojs@&N|n)W!mLd-pB!p1obeKMrYF)+{RT;^j_MLGDZWQ%Su<=W9@>B{falQ#a@v)_Pxid zk42CCsfQU_nC|mEK7ihU9?ur%+e{%V{f~m|)Wa1Ckh`)Gm;*(az&VXprk>3x> zUDFz*9+JyJRNfRJ-W&4LJNFsNL}ck1l%CzMgx-`^-^~MtIgDcMi2$=MDqvc4UtDk|1NztTJPfTwZDeZgbX&?XPTx(A(bo(iiO? zJQ}MQSEXKYRhDXa(@qh}pF6Q`9^GW=k zTb6#vEH@=-qqI%;L5~mE1YeBckeYF##tQdE{h3bbJ;H}xY%1k_NG}`8gMKx>aBpGH zz;X+bTXeV|&2>T620o-$xbG_f^!tFzYkT#jo44{wK-To*Q+*sjDId1&`5bavZ&ho_ zi%zmH0^jtnyqP7t#-^Xv7hJ?2-MyqT%(JtGojM6JUUk0;pJF9YCDFRdn=kD_eHrsL zUl|8Xjl^?bAfIvCIBnc?8p5l1OE#~$8Ph@OA6?LwdEht?d-XW4wz9kv_u(V-nBDv; z+_>h;{A9{D`dOuQd-zYf>b3zK#!XOAK0EtZU8}%C;tF`7OoHTl(cBGbZMV4pHn0mG zQg>lpLS&+0cnmW+oLtN}V)E}WP*=kC&U8b9J4pKskiS~Z0Vf?)V?-v~76-shsJOEv zL1s%ID%yy@ak3wU67dNG%*rY6gu}@Hzk!Y3b?I=oO#M&41LYYGk&OjQRN6PW_Tcim zc;G0P0fs?m3_Ol!09wOAGZ@#|lHEDt2Dw>`!FX&wb3+9w%ik5yOaQYkIch*0gXc!R z0%R!OmS^=ForR3f9x#*QgKyG}j^I}}f&Gm@dTRdA9XBAm#e2xm^$(brE*borW)>j$ zUDp;ZMO41)vC1!SzLL3v#2e&uV8*N7jl2eW8%9w9lXt7R5%Cd1cX%jf+1 zQ+}_%e(QAm_N`20Z{-QVS5>c9R3{Fo)hTa}^ak)Wnzxd&NhlWfG5E!Y!Ecpv{<<_RV_ZOG zL3H$iTw@c!^kCcihz^GR3Ufb8z0H$9dBQ(soL+fchoTqS{%;oc@1f%?4}Tu-+fDsD zj@+HUIZn?G7G}dsUg2zO1@sSTbWGo3k>t!! zkI~i8SfH~NMwY^k(-kt7q0-uQt|czHSE%>{>_6v@M{DoFg)5B0Bl8>2hicpNB{e+5MHr7#f- zQ~}eRh4Ce{TxYUio4SSqlL=yNk_4`U>SdcyJI78nPAZ=Ivq|hZoyJwak;+D!H~Sb< zN>Av|bqc-MP&Tv1mHB02lp>mY?nxGQLBTu{ySxxj@r1P`LFev=fTqQLHxl zthhks9d5~peHYFGMZeKe;g&ekTZTqm>l;5!2FY>T?qZY7D<}vmj&c=VX;QM~!xjPG zXLX+g#~hXd0`k%pxWxFr*=@$0*E+}UgXvdWo|ctA^JE;rR?;46lb(kyNSfBEJ!bF+ zP4g&TS)a;no~|=e1C$7FE6#qo(85W|xbg=dk9#Fo+Y5*eyrGP%Jl0_doq=^Xu)Wbo z$JHeOor39|a{90Mg=b^(ok5eX{M3s+$wqK~n<>9_+h1?!H>1!y^`l(tx;LYHt(MLD znlIIX)Pn~PPWQcb%5O;VRum8S<&E>2Gl(AM=Jphi=sCJhqX_49wRwraY>*R(QaWIon*XmCmiM(nawbV=+RlIwZf_sOI!SvUGW)BIFos zb?CZX%np+OPG|pQG<_wh!ypJ~8$IDm>yVkNeP=OD6v&uhZi1ylX#m82=Kde+)78 zDc9hL1usA+EhQJ2g`>7B2yI}SkLkcMtobIdLpY#uIrmXY2ym4(9Fd!lOtG5|;A=Y? zVPEB+(6umj`xK;o_Oex67b?#P79>1*2NkD;HQx#^eg6;q(1(BPSAL~}4XeP@|7&0U zYWW&IJpCEnG`|HO2M$_iwFqDzZgBKke&=skyn(Zbz$ktp9dJ^o%QhBM83X8m_E-K* z*$&RIBQ$`;Vhc5p#h=BEe08}1D|>;1-68E%hE>k?w=6jJ3tNtkdLHwIEHG~>U~aU! z>m-~EaUpxaGk$w!+zL&50a6|nMtBXYRRrGmv{uv&N?>F12hJ|czwMNP=|@IAlxdq$J`)=_rDll`?o2jt?*#4qKtsFkaU7LyWBmNc0I z^GS;vJ~+y^rh`27)QeS5pj0k5N!Mg7vT#Q{)zifj&O#0;{B&QT7g2Vfxr{%ru`CiB zsgF14tA2Cx1#c#R^~90l$lD78Hj}*(RGUFP(L*jKo&A)dH{><>Wik0k3ruut4eHYN zPZ-+9#UGuGr+FOz><(rC-xjI+Hd4y65&FPnMq!)Rq`o8-2{!L5J9 zmkoy-JgF!7%8u}-O=@va3j^wd-tXMIcY62E-P2d!xno=4vzB}g1zo}ecDVc~_U+p& z#%@{%`7H~Ojf2@nEJAoA(CuD*%Ad^;q9Z?9Tfal_wRi8FzWmj%o&NV<|FS>B;=YU> zs@^;~+l$NAN!mLbpGM{HF~RLA*9&2%vr*UbXHf$^qjY_nLt++{H~Fmv4vb&-0uw*& zhV^DaJ_ciI`+G}}B@|tv=-^P<)`tP>&xKZ_J#4=(V zSuA_Zs5$02Mr|UOw zonA9d#`$a31D``-A^zm?d#8u@@1O49yK}mK=iSqzhYw0;jI%x73FY#{CBM=4N_`gO znHTks&G&urk6zR6W8wPJ%j%12>o?yz-Fp2E)$jG%tjN=wFKJw%CAxq4DOvD+wKW?B z5AR#PNB8fZZocv6>85PuhUx@O%TZ6qilb@3+vAG#Z+lcbJ%031bRL}EdvM>jeC@`~ z(`%q;L?@{a#=yM|+k*dN)$6hPCgU|S<$gxHzI=M)t+!9FyUpHo``CyC*wOpXG!DvM z(a+N-{C*;D13z*aZF2pV@aApXg5N%_jTD2(K*^gF9I#8T{K)s?N0I-L@VYl<-Vm+Z zu0wzJ5Lzn@_?(MRMe zuKWB48iQZEeogv#!!{SYOZn@P5u0j2hY#-FJ>9?i>GeU$d~bPa*9(cwfKi{LzE> zsqa5f{U694?%lhaO;U98IEUQ62g&6PZ00zO1&lfPT{q?;swd+J56C=X6HNUUKwrQO zjT@O8Xbe&LJY>AL;UUJ*#)5n=Re5gcf$WZo~&%Ay5 zwr_j;^wwK%o<8&Gw@-iHw}1QT+rRBIlJ{+P7^4KihpP9x@4S2Z+Sk5z`pQ?n;xROz z9+uwp=YtaS|CH|oenaGe`l8y`HoYsj_jC+dfQdsrEo@l!{{$@X z(Ec_^dAz|Yp+Ncuc94T0&4Ktzg8Yp6)&cn>7yT-8r5qZe10_gq^& zjODTo#-W$R8#*9%m$x-C|I3(4UyzF)vFEBw`W5;^c-cPZ`fA9W@H8M@8-@WK7x^E2H?l`nP3I?nxo`olx@ zAMK~{CSzIZ37D@g!|l5IABh%#ha6&$%7AWZUC62I;e}i!-#W&I5)fL{bEQSx@GtoS zB$l_p!Opu&14h)6!MQ6uY40I*IfpJ&u6aXm#y6yV=%oJG76hjIBnOq?mxLE?r=cjg|~2(ZQQHSkNg%# z`9nf~(64?I9%~=MC2#wM`pf@xoz6e$DSPv^+9(M)Hj2K?6(+ zS`K&)zJ-;ntNq8Y&^F>oC)|`9iMM~r2S9e^S3IY`aRG{&9(0Pf;|Z7F&@>trKiQUY z@lk$p+aJ#Xlx!`l$;Uk#ERSHwDt9CI$AAqP0+;Asi0>k6@`zg_7q}1|Ko8MLVA0Ew zikjDeJ@JD=T?Y9|M)d!I`VI>oFLJcV@NZ4kW-p5!-d18!_fVg5Q?%H_0FPjj4RbkEieQlu{p@ zP&N~rEXKVcXM(pZ_5k`BlVB!fEDrgNtlq%M#zdZ0%w|2C7}xx!R8Kf5+v8WOa0OyL zj@Tc5cXOB1#)M)w zh*(0$FG#%o&JZ-fTv7>pf{EUVCqKNhkjaENlVNOx34P=l3l)c3wv|!nH)e4KPJUn} zx~W9hojS6(C*PRhv$4!1-?q%~Sn>&5Cj*o|B%wZ*PbmM)R*DlOUvOqXM{x!?EP4HB z?*;~G(p~QglAA!G|43^rs21zMWpKF^bo-X>vtbdNhd=!(hcMYZ0=^9z0WkfdENq&aMf zZToGJ!t{^oE27u_k;mnN*#ILx`P|_=`{FNxfDqdGIUWig`jcX^yjz!cO|#`L{cq$=dDF*V zbkc_?FWyBbm~!p!_#vCihHlf`>ve-G4g4*e+~Ic``9nh&{^O3ej+?wy*R!%2|H+*E zwv>dc+}G8n{AML+`+fh}K*y8rdwBHIvgQCv`PrZ%Kl1_kWAnlEiR5<~*ALn1%~