jessikitty/unity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prepare for encoding of endpoint and selector indexes in non-linear order

Browse Source 
This change makes the compression scheme more flexible.

Explanation:
In the original scheme, indexes are encoded in linear order, which means that each index uses the previously encoded index for prediction. However, more sophisticated schemes might require arbitrary references into the stream of already encoded indexes. For this reason, Zeng function has been modified to accept the ordering histogram as an input, instead of the linear array of indexes. Note that Zeng function itself does not rely on the indexes being encoded in linear order.

Testing:
The modified algorithm has been tested on the Kodak test set using 64-bit build with default settings (running on Windows 10, i7-4790, 3.6GHz). All the decompressed test images are identical to the images being compressed and decompressed using original version of Crunch.

[Compressing Kodak set without mipmaps]
Original: 1582222 bytes / 28.867 sec
Modified: 1570534 bytes / 28.524 sec
Improvement: 0.74% (compression ratio) / 1.19% (compression time)

[Compressing Kodak set with mipmaps]
Original: 2065243 bytes / 37.001 sec
Modified: 2051509 bytes / 36.388 sec
Improvement: 0.67% (compression ratio) / 1.66% (compression time)
This commit is contained in:
Alexander Suvorov
2017-04-28 11:32:14 +02:00
parent 8cc5f19ae5
commit 19f05aadbc
4 changed files with 72 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/crunch_x64.exe
BIN
View File
Binary file not shown.
crnlib/crn_comp.cpp
+52 -12
View File
@@ -1315,6 +1315,7 @@ struct optimize_color_endpoint_codebook_params {
crnlib::vector<uint>* m_pTrial_color_endpoint_remap;
uint m_iter_index;
uint m_max_iter_index;
hist_type* xhist;
};
void crn_comp::optimize_color_endpoint_codebook_task(uint64 data, void* pData_ptr) {
@@ -1328,8 +1329,7 @@ void crn_comp::optimize_color_endpoint_codebook_task(uint64 data, void* pData_pt
create_zeng_reorder_table(
m_hvq.get_color_endpoint_codebook_size(),
m_endpoint_indices[cColor].size(),
&m_endpoint_indices[cColor][0],
*pParams->xhist,
*pParams->m_pTrial_color_endpoint_remap,
pParams->m_iter_index ? color_endpoint_similarity_func : NULL,
&m_hvq,
@@ -1362,11 +1362,22 @@ bool crn_comp::optimize_color_endpoint_codebook(crnlib::vector<uint>& remapping)
crnlib::vector<uint> trial_color_endpoint_remaps[cMaxEndpointRemapIters + 1];
uint n = m_hvq.get_color_endpoint_codebook_size();
hist_type xhist(n * n);
for (uint i = 0; i < m_endpoint_indices[cColor].size(); i++) {
const int prev_val = (i > 0) ? m_endpoint_indices[cColor][i - 1] : -1;
const int cur_val = m_endpoint_indices[cColor][i];
const int next_val = (i < (m_endpoint_indices[cColor].size() - 1)) ? m_endpoint_indices[cColor][i + 1] : -1;
update_hist(xhist, cur_val, prev_val, n);
update_hist(xhist, cur_val, next_val, n);
}
for (uint i = 0; i <= cMaxEndpointRemapIters; i++) {
optimize_color_endpoint_codebook_params* pParams = crnlib_new<optimize_color_endpoint_codebook_params>();
pParams->m_iter_index = i;
pParams->m_max_iter_index = cMaxEndpointRemapIters;
pParams->m_pTrial_color_endpoint_remap = &trial_color_endpoint_remaps[i];
pParams->xhist = &xhist;
m_task_pool.queue_object_task(this, &crn_comp::optimize_color_endpoint_codebook_task, 0, pParams);
}
@@ -1418,6 +1429,7 @@ struct optimize_color_selector_codebook_params {
crnlib::vector<uint>* m_pTrial_color_selector_remap;
uint m_iter_index;
uint m_max_iter_index;
hist_type* xhist;
};
void crn_comp::optimize_color_selector_codebook_task(uint64 data, void* pData_ptr) {
@@ -1430,8 +1442,7 @@ void crn_comp::optimize_color_selector_codebook_task(uint64 data, void* pData_pt
float f = pParams->m_iter_index / static_cast<float>(pParams->m_max_iter_index - 1);
create_zeng_reorder_table(
m_hvq.get_color_selector_codebook_size(),
m_selector_indices[cColor].size(),
&m_selector_indices[cColor][0],
*pParams->xhist,
*pParams->m_pTrial_color_selector_remap,
pParams->m_iter_index ? color_selector_similarity_func : NULL,
(void*)&m_hvq.get_color_selectors_vec(),
@@ -1471,11 +1482,22 @@ bool crn_comp::optimize_color_selector_codebook(crnlib::vector<uint>& remapping)
crnlib::vector<uint> trial_color_selector_remaps[cMaxSelectorRemapIters + 1];
uint n = m_hvq.get_color_selector_codebook_size();
hist_type xhist(n * n);
for (uint i = 0; i < m_selector_indices[cColor].size(); i++) {
const int prev_val = (i > 0) ? m_selector_indices[cColor][i - 1] : -1;
const int cur_val = m_selector_indices[cColor][i];
const int next_val = (i < (m_selector_indices[cColor].size() - 1)) ? m_selector_indices[cColor][i + 1] : -1;
update_hist(xhist, cur_val, prev_val, n);
update_hist(xhist, cur_val, next_val, n);
}
for (uint i = 0; i <= cMaxSelectorRemapIters; i++) {
optimize_color_selector_codebook_params* pParams = crnlib_new<optimize_color_selector_codebook_params>();
pParams->m_iter_index = i;
pParams->m_max_iter_index = cMaxSelectorRemapIters;
pParams->m_pTrial_color_selector_remap = &trial_color_selector_remaps[i];
pParams->xhist = &xhist;
m_task_pool.queue_object_task(this, &crn_comp::optimize_color_selector_codebook_task, 0, pParams);
}
@@ -1530,10 +1552,10 @@ bool crn_comp::optimize_color_selector_codebook(crnlib::vector<uint>& remapping)
}
struct optimize_alpha_endpoint_codebook_params {
crnlib::vector<uint>* m_pAlpha_indices;
crnlib::vector<uint>* m_pTrial_alpha_endpoint_remap;
uint m_iter_index;
uint m_max_iter_index;
hist_type* xhist;
};
void crn_comp::optimize_alpha_endpoint_codebook_task(uint64 data, void* pData_ptr) {
@@ -1547,8 +1569,7 @@ void crn_comp::optimize_alpha_endpoint_codebook_task(uint64 data, void* pData_pt
create_zeng_reorder_table(
m_hvq.get_alpha_endpoint_codebook_size(),
pParams->m_pAlpha_indices->size(),
&(*pParams->m_pAlpha_indices)[0],
*pParams->xhist,
*pParams->m_pTrial_alpha_endpoint_remap,
pParams->m_iter_index ? alpha_endpoint_similarity_func : NULL,
&m_hvq,
@@ -1587,12 +1608,22 @@ bool crn_comp::optimize_alpha_endpoint_codebook(crnlib::vector<uint>& remapping)
crnlib::vector<uint> trial_alpha_endpoint_remaps[cMaxEndpointRemapIters + 1];
uint n = m_hvq.get_alpha_endpoint_codebook_size();
hist_type xhist(n * n);
for (uint i = 0; i < alpha_indices.size(); i++) {
const int prev_val = (i > 0) ? alpha_indices[i - 1] : -1;
const int cur_val = alpha_indices[i];
const int next_val = (i < (alpha_indices.size() - 1)) ? alpha_indices[i + 1] : -1;
update_hist(xhist, cur_val, prev_val, n);
update_hist(xhist, cur_val, next_val, n);
}
for (uint i = 0; i <= cMaxEndpointRemapIters; i++) {
optimize_alpha_endpoint_codebook_params* pParams = crnlib_new<optimize_alpha_endpoint_codebook_params>();
pParams->m_pAlpha_indices = &alpha_indices;
pParams->m_iter_index = i;
pParams->m_max_iter_index = cMaxEndpointRemapIters;
pParams->m_pTrial_alpha_endpoint_remap = &trial_alpha_endpoint_remaps[i];
pParams->xhist = &xhist;
m_task_pool.queue_object_task(this, &crn_comp::optimize_alpha_endpoint_codebook_task, 0, pParams);
}
@@ -1641,10 +1672,10 @@ bool crn_comp::optimize_alpha_endpoint_codebook(crnlib::vector<uint>& remapping)
}
struct optimize_alpha_selector_codebook_params {
crnlib::vector<uint>* m_pAlpha_indices;
crnlib::vector<uint>* m_pTrial_alpha_selector_remap;
uint m_iter_index;
uint m_max_iter_index;
hist_type* xhist;
};
void crn_comp::optimize_alpha_selector_codebook_task(uint64 data, void* pData_ptr) {
@@ -1657,8 +1688,7 @@ void crn_comp::optimize_alpha_selector_codebook_task(uint64 data, void* pData_pt
float f = pParams->m_iter_index / static_cast<float>(pParams->m_max_iter_index - 1);
create_zeng_reorder_table(
m_hvq.get_alpha_selector_codebook_size(),
pParams->m_pAlpha_indices->size(),
&(*pParams->m_pAlpha_indices)[0],
*pParams->xhist,
*pParams->m_pTrial_alpha_selector_remap,
pParams->m_iter_index ? alpha_selector_similarity_func : NULL,
(void*)&m_hvq.get_alpha_selectors_vec(),
@@ -1703,12 +1733,22 @@ bool crn_comp::optimize_alpha_selector_codebook(crnlib::vector<uint>& remapping)
crnlib::vector<uint> trial_alpha_selector_remaps[cMaxSelectorRemapIters + 1];
uint n = m_hvq.get_alpha_selector_codebook_size();
hist_type xhist(n * n);
for (uint i = 0; i < alpha_indices.size(); i++) {
const int prev_val = (i > 0) ? alpha_indices[i - 1] : -1;
const int cur_val = alpha_indices[i];
const int next_val = (i < (alpha_indices.size() - 1)) ? alpha_indices[i + 1] : -1;
update_hist(xhist, cur_val, prev_val, n);
update_hist(xhist, cur_val, next_val, n);
}
for (uint i = 0; i <= cMaxSelectorRemapIters; i++) {
optimize_alpha_selector_codebook_params* pParams = crnlib_new<optimize_alpha_selector_codebook_params>();
pParams->m_pAlpha_indices = &alpha_indices;
pParams->m_iter_index = i;
pParams->m_max_iter_index = cMaxSelectorRemapIters;
pParams->m_pTrial_alpha_selector_remap = &trial_alpha_selector_remaps[i];
pParams->xhist = &xhist;
m_task_pool.queue_object_task(this, &crn_comp::optimize_alpha_selector_codebook_task, 0, pParams);
}
crnlib/crn_zeng.cpp
+4 -110
View File
@@ -8,16 +8,8 @@
#include "crn_sparse_array.h"
#include <deque>
#define USE_SPARSE_ARRAY 1
namespace crnlib {
#if USE_SPARSE_ARRAY
typedef sparse_array<uint, 4> hist_type;
#else
typedef crnlib::vector<uint> hist_type;
#endif
static inline void update_hist(hist_type& hist, int i, int j, int n) {
void update_hist(hist_type& hist, int i, int j, int n) {
if (i == j)
return;
@@ -27,7 +19,7 @@ static inline void update_hist(hist_type& hist, int i, int j, int n) {
uint index = i * n + j;
#if USE_SPARSE_ARRAY
#if CRN_ZENG_USE_SPARSE_ARRAY
uint freq = hist[index];
freq++;
hist.set(index, freq);
@@ -44,49 +36,14 @@ static inline uint read_hist(hist_type& hist, int i, int j, int n) {
return hist[i * n + j];
}
void create_zeng_reorder_table(uint n, uint num_indices, const uint* pIndices, crnlib::vector<uint>& remap_table, zeng_similarity_func pFunc, void* pContext, float similarity_func_weight) {
CRNLIB_ASSERT((n > 0) && (num_indices > 0));
void create_zeng_reorder_table(uint n, hist_type& xhist, crnlib::vector<uint>& remap_table, zeng_similarity_func pFunc, void* pContext, float similarity_func_weight) {
CRNLIB_ASSERT(n > 0);
CRNLIB_ASSERT_CLOSED_RANGE(similarity_func_weight, 0.0f, 1.0f);
// printf("create_zeng_reorder_table start:\n");
remap_table.clear();
remap_table.resize(n);
if (num_indices <= 1)
return;
const uint t = n * n;
hist_type xhist(t);
for (uint i = 0; i < num_indices; i++) {
const int prev_val = (i > 0) ? pIndices[i - 1] : -1;
const int cur_val = pIndices[i];
const int next_val = (i < (num_indices - 1)) ? pIndices[i + 1] : -1;
update_hist(xhist, cur_val, prev_val, n);
update_hist(xhist, cur_val, next_val, n);
}
#if 0
uint total1 = 0, total2 = 0;
for (uint i = 0; i < n; i++)
{
for (uint j = 0; j < n; j++)
{
if (i == j)
continue;
//uint a = hist[i * n + j];
//total1 += a;
uint c = read_hist(xhist, i, j, n);
total2 += c;
}
}
printf("%u %u\n", total1, total2);
#endif
uint max_freq = 0;
uint max_index = 0;
@@ -134,21 +91,7 @@ void create_zeng_reorder_table(uint n, uint num_indices, const uint* pIndices, c
for (uint i = 0; i < values_remaining.size(); i++) {
uint u = values_remaining[i];
#if 0
double total_freq = 0;
for (uint j = 0; j < values_chosen.size(); j++)
{
uint l = values_chosen[j];
total_freq += read_hist(xhist, u, l, n); //[u * n + l];
}
CRNLIB_ASSERT(total_freq_to_chosen_values[u] == total_freq);
#else
double total_freq = total_freq_to_chosen_values[u];
#endif
if (pFunc) {
float weight = math::maximum<float>(
@@ -216,55 +159,6 @@ void create_zeng_reorder_table(uint n, uint num_indices, const uint* pIndices, c
uint v = values_chosen[i];
remap_table[v] = i;
}
#if 0
uint before_sum = 0;
uint after_sum = 0;
{
printf("\nBEFORE:\n");
crnlib::vector<uint> delta_hist(n*2);
int sum = 0;
for (uint i = 1; i < num_indices; i++)
{
int prev = pIndices[i-1];
int cur = pIndices[i];
delta_hist[prev-cur+n]++;
sum += labs(prev-cur);
}
printf("\n");
for (uint i = 0; i < n*2; i++)
printf("%04u ", delta_hist[i]);
printf("\nSum: %i\n", sum);
before_sum = sum;
}
{
printf("AFTER:\n");
crnlib::vector<uint> delta_hist(n*2);
int sum = 0;
for (uint i = 1; i < num_indices; i++)
{
int prev = remap_table[pIndices[i-1]];
int cur = remap_table[pIndices[i]];
delta_hist[prev-cur+n]++;
sum += labs(prev-cur);
}
printf("\n");
for (uint i = 0; i < n*2; i++)
printf("%04u ", delta_hist[i]);
printf("\nSum: %i\n", sum);
after_sum = sum;
}
printf("Before sum: %u, After sum: %u\n", before_sum, after_sum);
#endif
// printf("create_zeng_reorder_table end:\n");
}
} // namespace crnlib
crnlib/crn_zeng.h
+16 -3
View File
@@ -1,9 +1,22 @@
// File: crn_zeng.h
// See Copyright Notice and license at the end of inc/crnlib.h
namespace crnlib {
typedef float (*zeng_similarity_func)(uint index_a, uint index_b, void* pContext);
#define CRN_ZENG_USE_SPARSE_ARRAY 1
void create_zeng_reorder_table(uint n, uint num_indices, const uint* pIndices, crnlib::vector<uint>& remap_table, zeng_similarity_func pFunc, void* pContext, float similarity_func_weight);
#if CRN_ZENG_USE_SPARSE_ARRAY
#include "crn_sparse_array.h"
#endif
namespace crnlib {
#if CRN_ZENG_USE_SPARSE_ARRAY
typedef sparse_array<uint, 4> hist_type;
#else
typedef crnlib::vector<uint> hist_type;
#endif
typedef float (*zeng_similarity_func)(uint index_a, uint index_b, void* pContext);
void update_hist(hist_type& hist, int i, int j, int n);
void create_zeng_reorder_table(uint n, hist_type& hist, crnlib::vector<uint>& remap_table, zeng_similarity_func pFunc, void* pContext, float similarity_func_weight);
} // namespace crnlib