jessikitty/unity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use multiple threads for node split in vector quantization

Browse Source 
This change improves the compression speed for both DXT and ETC encodings.

Explanation:

During the node split iteration, identical computations are performed for all the vectors of the split node. The overall performance can be improved by performing independent computations in separate threads. In order to avoid possible performance overhead, on each iteration the number of threads is selected in such a way so that each thread processes at least 512 vectors.

DXT Testing:

The modified algorithm has been tested on the Kodak test set using 64-bit build with default settings (running on Windows 10, i7-4790, 3.6GHz). All the decompressed test images are identical to the images being compressed and decompressed using original version of Crunch (revision ea9b8d8).

[Compressing Kodak set without mipmaps using DXT1 encoding]
Original: 1582222 bytes / 28.892 sec
Modified: 1468204 bytes / 7.578 sec
Improvement: 7.21% (compression ratio) / 73.77% (compression time)

[Compressing Kodak set with mipmaps using DXT1 encoding]
Original: 2065243 bytes / 36.943 sec
Modified: 1914805 bytes / 9.993 sec
Improvement: 7.28% (compression ratio) / 72.95% (compression time)

ETC Testing:

The modified algorithm has been tested on the Kodak test set using 64-bit build with default settings (running on Windows 10, i7-4790, 3.6GHz). The ETC1 quantization parameters have been selected in such a way, so that ETC1 compression gives approximately the same average Luma PSNR as the corresponding DXT1 compression (which is equal to 34.044 dB for the Kodak test set compressed without mipmaps using DXT1 encoding and default quality settings).

[Compressing Kodak set without mipmaps using ETC1 encoding]
Total size: 1607858 bytes
Total time: 14.753 sec
Average bitrate: 1.363 bpp
Average Luma PSNR: 34.050 dB
This commit is contained in:
Alexander Suvorov
2017-10-20 14:23:47 +02:00
parent fbe3f6ca10
commit 1028520280
3 changed files with 77 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/crunch_x64.exe
BIN
View File
Binary file not shown.
crnlib/crn_dxt_hc.cpp
+4 -4
View File
@@ -669,7 +669,7 @@ void dxt_hc::determine_color_endpoints() {
vq.add_training_vec(m_tiles[t].color_endpoint, (uint)(m_tiles[t].pixels.size() * m_tiles[t].weight));
}
vq.generate_codebook(math::minimum<uint>(m_num_tiles, m_params.m_color_endpoint_codebook_size), true);
vq.generate_codebook(math::minimum<uint>(m_num_tiles, m_params.m_color_endpoint_codebook_size), true, m_pTask_pool);
m_color_clusters.resize(vq.get_codebook_size());
for (uint i = 0; i <= m_pTask_pool->get_num_threads(); i++)
@@ -831,7 +831,7 @@ void dxt_hc::determine_alpha_endpoints() {
}
}
vq.generate_codebook(math::minimum<uint>(m_num_tiles, m_params.m_alpha_endpoint_codebook_size));
vq.generate_codebook(math::minimum<uint>(m_num_tiles, m_params.m_alpha_endpoint_codebook_size), false, m_pTask_pool);
m_alpha_clusters.resize(vq.get_codebook_size());
for (uint i = 0; i <= m_pTask_pool->get_num_threads(); i++)
@@ -920,7 +920,7 @@ void dxt_hc::create_color_selector_codebook() {
v[p] = ((selector & 3) + 0.5f) * 0.25f;
selector_vq.add_training_vec(v, m_has_etc_color_blocks ? (selector & 0xFFFF) + (selector >> 16) : selector);
}
selector_vq.generate_codebook(m_params.m_color_selector_codebook_size);
selector_vq.generate_codebook(m_params.m_color_selector_codebook_size, false, m_pTask_pool);
m_color_selectors.resize(selector_vq.get_codebook_size());
m_color_selectors_used.resize(selector_vq.get_codebook_size());
for (uint i = 0; i < selector_vq.get_codebook_size(); i++) {
@@ -1034,7 +1034,7 @@ void dxt_hc::create_alpha_selector_codebook() {
selector_vq.add_training_vec(v, selector);
}
}
selector_vq.generate_codebook(m_params.m_alpha_selector_codebook_size);
selector_vq.generate_codebook(m_params.m_alpha_selector_codebook_size, false, m_pTask_pool);
m_alpha_selectors.resize(selector_vq.get_codebook_size());
m_alpha_selectors_used.resize(selector_vq.get_codebook_size());
for (uint i = 0; i < selector_vq.get_codebook_size(); i++) {
crnlib/crn_tree_clusterizer.h
+73 -17
View File
@@ -2,6 +2,7 @@
// See Copyright Notice and license at the end of inc/crnlib.h
#pragma once
#include "crn_matrix.h"
#include "crn_threading.h"
#include <queue>
namespace crnlib {
@@ -37,7 +38,7 @@ class tree_clusterizer {
m_hist.push_back(std::make_pair(v, weight));
}
bool generate_codebook(uint max_size, bool generate_node_index_map = false) {
bool generate_codebook(uint max_size, bool generate_node_index_map = false, task_pool* pTask_pool = 0) {
if (m_hist.empty())
return false;
@@ -65,6 +66,7 @@ class tree_clusterizer {
m_weightedDotProducts.resize(m_vectors.size());
m_vectorsInfoLeft.resize(m_vectors.size());
m_vectorsInfoRight.resize(m_vectors.size());
m_vectorComparison.resize(m_vectors.size());
vq_node root;
root.m_begin = 0;
@@ -94,7 +96,7 @@ class tree_clusterizer {
end_node++;
splits++;
while (splits < max_size && split_node(node_queue, end_node))
while (splits < max_size && split_node(node_queue, end_node, pTask_pool))
splits++;
m_codebook.clear();
@@ -144,6 +146,7 @@ class tree_clusterizer {
crnlib::vector<VectorType> m_weightedVectors;
crnlib::vector<double> m_weightedDotProducts;
crnlib::vector<VectorInfo> m_vectorsInfo, m_vectorsInfoLeft, m_vectorsInfoRight;
crnlib::vector<bool> m_vectorComparison;
crnlib::hash_map<VectorType, uint> m_node_index_map;
struct vq_node {
@@ -172,7 +175,29 @@ class tree_clusterizer {
vector_vec_type m_codebook;
bool split_node(std::priority_queue<NodeInfo>& node_queue, uint& end_node) {
struct distance_comparison_task_params {
VectorType* left_child;
VectorType* right_child;
uint begin;
uint end;
uint num_tasks;
};
void distance_comparison_task(uint64 data, void* pData_ptr) {
distance_comparison_task_params* pParams = (distance_comparison_task_params*)pData_ptr;
const VectorType& left_child = *pParams->left_child;
const VectorType& right_child = *pParams->right_child;
uint begin = pParams->begin + (pParams->end - pParams->begin) * data / pParams->num_tasks;
uint end = pParams->begin + (pParams->end - pParams->begin) * (data + 1) / pParams->num_tasks;
for (uint i = begin; i < end; i++) {
const VectorType& v = m_vectors[m_vectorsInfo[i].index];
double left_dist2 = left_child.squared_distance(v);
double right_dist2 = right_child.squared_distance(v);
m_vectorComparison[i] = left_dist2 < right_dist2;
}
}
bool split_node(std::priority_queue<NodeInfo>& node_queue, uint& end_node, task_pool* pTask_pool = 0) {
if (node_queue.empty() || node_queue.top().m_variance <= 0.0f)
return false;
@@ -183,6 +208,9 @@ class tree_clusterizer {
node_queue.pop();
uint num_blocks = (parent_node.m_end - parent_node.m_begin) >> 9;
uint num_tasks = num_blocks > 1 && pTask_pool ? math::minimum(num_blocks, pTask_pool->get_num_threads() + 1) : 1;
VectorType furthest(0);
double furthest_dist = -1.0f;
@@ -312,20 +340,48 @@ class tree_clusterizer {
left_weight = 0;
right_weight = 0;
for (uint i = parent_node.m_begin; i < parent_node.m_end; i++) {
const VectorInfo& vectorInfo = m_vectorsInfo[i];
double left_dist2 = left_child.squared_distance(m_vectors[vectorInfo.index]);
double right_dist2 = right_child.squared_distance(m_vectors[vectorInfo.index]);
if (left_dist2 < right_dist2) {
new_left_child += m_weightedVectors[vectorInfo.index];
left_ttsum += m_weightedDotProducts[vectorInfo.index];
left_weight += vectorInfo.weight;
m_vectorsInfoLeft[left_info_index++] = vectorInfo;
} else {
new_right_child += m_weightedVectors[vectorInfo.index];
right_ttsum += m_weightedDotProducts[vectorInfo.index];
right_weight += vectorInfo.weight;
m_vectorsInfoRight[right_info_index++] = vectorInfo;
if (num_tasks > 1) {
distance_comparison_task_params params;
params.left_child = &left_child;
params.right_child = &right_child;
params.begin = parent_node.m_begin;
params.end = parent_node.m_end;
params.num_tasks = num_tasks;
for (uint task = 0; task < params.num_tasks; task++)
pTask_pool->queue_object_task(this, &tree_clusterizer::distance_comparison_task, task, &params);
pTask_pool->join();
for (uint i = parent_node.m_begin; i < parent_node.m_end; i++) {
const VectorInfo& vectorInfo = m_vectorsInfo[i];
if (m_vectorComparison[i]) {
new_left_child += m_weightedVectors[vectorInfo.index];
left_ttsum += m_weightedDotProducts[vectorInfo.index];
left_weight += vectorInfo.weight;
m_vectorsInfoLeft[left_info_index++] = vectorInfo;
} else {
new_right_child += m_weightedVectors[vectorInfo.index];
right_ttsum += m_weightedDotProducts[vectorInfo.index];
right_weight += vectorInfo.weight;
m_vectorsInfoRight[right_info_index++] = vectorInfo;
}
}
} else {
for (uint i = parent_node.m_begin; i < parent_node.m_end; i++) {
const VectorInfo& vectorInfo = m_vectorsInfo[i];
double left_dist2 = left_child.squared_distance(m_vectors[vectorInfo.index]);
double right_dist2 = right_child.squared_distance(m_vectors[vectorInfo.index]);
if (left_dist2 < right_dist2) {
new_left_child += m_weightedVectors[vectorInfo.index];
left_ttsum += m_weightedDotProducts[vectorInfo.index];
left_weight += vectorInfo.weight;
m_vectorsInfoLeft[left_info_index++] = vectorInfo;
} else {
new_right_child += m_weightedVectors[vectorInfo.index];
right_ttsum += m_weightedDotProducts[vectorInfo.index];
right_weight += vectorInfo.weight;
m_vectorsInfoRight[right_info_index++] = vectorInfo;
}
}
}