OpenJPH
Open-source implementation of JPEG2000 Part-15
Loading...
Searching...
No Matches
ojph_colour_avx2.cpp
Go to the documentation of this file.
1//***************************************************************************/
2// This software is released under the 2-Clause BSD license, included
3// below.
4//
5// Copyright (c) 2019, Aous Naman
6// Copyright (c) 2019, Kakadu Software Pty Ltd, Australia
7// Copyright (c) 2019, The University of New South Wales, Australia
8//
9// Redistribution and use in source and binary forms, with or without
10// modification, are permitted provided that the following conditions are
11// met:
12//
13// 1. Redistributions of source code must retain the above copyright
14// notice, this list of conditions and the following disclaimer.
15//
16// 2. Redistributions in binary form must reproduce the above copyright
17// notice, this list of conditions and the following disclaimer in the
18// documentation and/or other materials provided with the distribution.
19//
20// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
21// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
23// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
26// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
27// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
28// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
29// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
30// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31//***************************************************************************/
32// This file is part of the OpenJPH software implementation.
33// File: ojph_colour_avx2.cpp
34// Author: Aous Naman
35// Date: 11 October 2019
36//***************************************************************************/
37
38#include "ojph_arch.h"
39#if defined(OJPH_ARCH_I386) || defined(OJPH_ARCH_X86_64)
40
41#include <climits>
42#include <cmath>
43
44#include "ojph_defs.h"
45#include "ojph_mem.h"
46#include "ojph_colour.h"
47
48#include "ojph_params.h"
49#include "../codestream/ojph_params_local.h"
50
51#include <immintrin.h>
52
53namespace ojph {
54 namespace local {
55
57 // https://github.com/seung-lab/dijkstra3d/blob/master/libdivide.h
58 static inline
59 __m256i avx2_mm256_srai_epi64(__m256i a, int amt, __m256i m)
60 {
61 // note than m must be obtained using
62 // __m256i m = _mm256_set1_epi64x(1ULL << (63 - amt));
63 __m256i x = _mm256_srli_epi64(a, amt);
64 x = _mm256_xor_si256(x, m);
65 __m256i result = _mm256_sub_epi64(x, m);
66 return result;
67 }
68
70 void avx2_rev_convert(const line_buf *src_line,
71 const ui32 src_line_offset,
72 line_buf *dst_line,
73 const ui32 dst_line_offset,
74 si64 shift, ui32 width)
75 {
76 if (src_line->flags & line_buf::LFT_32BIT)
77 {
78 if (dst_line->flags & line_buf::LFT_32BIT)
79 {
80 const si32 *sp = src_line->i32 + src_line_offset;
81 si32 *dp = dst_line->i32 + dst_line_offset;
82 __m256i sh = _mm256_set1_epi32((si32)shift);
83 for (int i = (width + 7) >> 3; i > 0; --i, sp+=8, dp+=8)
84 {
85 __m256i s = _mm256_loadu_si256((__m256i*)sp);
86 s = _mm256_add_epi32(s, sh);
87 _mm256_storeu_si256((__m256i*)dp, s);
88 }
89 }
90 else
91 {
92 const si32 *sp = src_line->i32 + src_line_offset;
93 si64 *dp = dst_line->i64 + dst_line_offset;
94 __m256i sh = _mm256_set1_epi64x(shift);
95 for (int i = (width + 7) >> 3; i > 0; --i, sp+=8, dp+=8)
96 {
97 __m256i s, t;
98 s = _mm256_loadu_si256((__m256i*)sp);
99
100 t = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(s, 0));
101 t = _mm256_add_epi64(t, sh);
102 _mm256_storeu_si256((__m256i*)dp, t);
103
104 t = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(s, 1));
105 t = _mm256_add_epi64(t, sh);
106 _mm256_storeu_si256((__m256i*)dp + 1, t);
107 }
108 }
109 }
110 else
111 {
112 assert(src_line->flags | line_buf::LFT_64BIT);
113 assert(dst_line->flags | line_buf::LFT_32BIT);
114 const si64 *sp = src_line->i64 + src_line_offset;
115 si32 *dp = dst_line->i32 + dst_line_offset;
116 __m256i low_bits = _mm256_set_epi64x(0, (si64)ULLONG_MAX,
117 0, (si64)ULLONG_MAX);
118 __m256i sh = _mm256_set1_epi64x(shift);
119 for (int i = (width + 7) >> 3; i > 0; --i, sp+=8, dp+=8)
120 {
121 __m256i s, t;
122 s = _mm256_loadu_si256((__m256i*)sp);
123 s = _mm256_add_epi64(s, sh);
124
125 t = _mm256_shuffle_epi32(s, _MM_SHUFFLE(0, 0, 2, 0));
126 t = _mm256_and_si256(low_bits, t);
127
128 s = _mm256_loadu_si256((__m256i*)sp + 1);
129 s = _mm256_add_epi64(s, sh);
130
131 s = _mm256_shuffle_epi32(s, _MM_SHUFFLE(2, 0, 0, 0));
132 s = _mm256_andnot_si256(low_bits, s);
133
134 t = _mm256_or_si256(s, t);
135 t = _mm256_permute4x64_epi64(t, _MM_SHUFFLE(3, 1, 2, 0));
136 _mm256_storeu_si256((__m256i*)dp, t);
137 }
138 }
139 }
140
142 void avx2_rev_convert_nlt_type3(const line_buf *src_line,
143 const ui32 src_line_offset,
144 line_buf *dst_line,
145 const ui32 dst_line_offset,
146 si64 shift, ui32 width)
147 {
148 if (src_line->flags & line_buf::LFT_32BIT)
149 {
150 if (dst_line->flags & line_buf::LFT_32BIT)
151 {
152 const si32 *sp = src_line->i32 + src_line_offset;
153 si32 *dp = dst_line->i32 + dst_line_offset;
154 __m256i sh = _mm256_set1_epi32((si32)(-shift));
155 __m256i zero = _mm256_setzero_si256();
156 for (int i = (width + 7) >> 3; i > 0; --i, sp += 8, dp += 8)
157 {
158 __m256i s = _mm256_loadu_si256((__m256i*)sp);
159 __m256i c = _mm256_cmpgt_epi32(zero, s); // 0xFFFFFFFF for -ve val
160 __m256i v_m_sh = _mm256_sub_epi32(sh, s); // - shift - value
161 v_m_sh = _mm256_and_si256(c, v_m_sh); // keep only -shift-val
162 s = _mm256_andnot_si256(c, s); // keep only +ve or 0
163 s = _mm256_or_si256(s, v_m_sh); // combine
164 _mm256_storeu_si256((__m256i*)dp, s);
165 }
166 }
167 else
168 {
169 const si32 *sp = src_line->i32 + src_line_offset;
170 si64 *dp = dst_line->i64 + dst_line_offset;
171 __m256i sh = _mm256_set1_epi64x(-shift);
172 __m256i zero = _mm256_setzero_si256();
173 for (int i = (width + 7) >> 3; i > 0; --i, sp += 8, dp += 8)
174 {
175 __m256i s, t, u0, u1, c, v_m_sh;
176 s = _mm256_loadu_si256((__m256i*)sp);
177
178 t = _mm256_cmpgt_epi32(zero, s); // find -ve 32bit -1
179 u0 = _mm256_unpacklo_epi32(s, t); // correct 64bit data
180 c = _mm256_unpacklo_epi32(t, t); // 64bit -1 for -ve value
181
182 v_m_sh = _mm256_sub_epi64(sh, u0); // - shift - value
183 v_m_sh = _mm256_and_si256(c, v_m_sh); // keep only - shift - value
184 u0 = _mm256_andnot_si256(c, u0); // keep only +ve or 0
185 u0 = _mm256_or_si256(u0, v_m_sh); // combine
186
187 u1 = _mm256_unpackhi_epi32(s, t); // correct 64bit data
188 c = _mm256_unpackhi_epi32(t, t); // 64bit -1 for -ve value
189
190 v_m_sh = _mm256_sub_epi64(sh, u1); // - shift - value
191 v_m_sh = _mm256_and_si256(c, v_m_sh); // keep only - shift - value
192 u1 = _mm256_andnot_si256(c, u1); // keep only +ve or 0
193 u1 = _mm256_or_si256(u1, v_m_sh); // combine
194
195 t = _mm256_permute2x128_si256(u0, u1, (2 << 4) | 0);
196 _mm256_storeu_si256((__m256i*)dp, t);
197
198 t = _mm256_permute2x128_si256(u0, u1, (3 << 4) | 1);
199 _mm256_storeu_si256((__m256i*)dp + 1, t);
200 }
201 }
202 }
203 else
204 {
205 assert(src_line->flags | line_buf::LFT_64BIT);
206 assert(dst_line->flags | line_buf::LFT_32BIT);
207 const si64 *sp = src_line->i64 + src_line_offset;
208 si32 *dp = dst_line->i32 + dst_line_offset;
209 __m256i sh = _mm256_set1_epi64x(-shift);
210 __m256i zero = _mm256_setzero_si256();
211 __m256i half_mask = _mm256_set_epi64x(0, (si64)ULLONG_MAX,
212 0, (si64)ULLONG_MAX);
213 for (int i = (width + 7) >> 3; i > 0; --i, sp += 8, dp += 8)
214 {
215 // s for source, t for target, p for positive, n for negative,
216 // m for mask, and tm for temp
217 __m256i s, t, p, n, m, tm;
218 s = _mm256_loadu_si256((__m256i*)sp);
219
220 m = _mm256_cmpgt_epi64(zero, s); // 64b -1 for -ve value
221 tm = _mm256_sub_epi64(sh, s); // - shift - value
222 n = _mm256_and_si256(m, tm); // -ve
223 p = _mm256_andnot_si256(m, s); // +ve
224 tm = _mm256_or_si256(n, p);
225 tm = _mm256_shuffle_epi32(tm, _MM_SHUFFLE(0, 0, 2, 0));
226 t = _mm256_and_si256(half_mask, tm);
227
228 s = _mm256_loadu_si256((__m256i*)sp + 1);
229 m = _mm256_cmpgt_epi64(zero, s); // 64b -1 for -ve value
230 tm = _mm256_sub_epi64(sh, s); // - shift - value
231 n = _mm256_and_si256(m, tm); // -ve
232 p = _mm256_andnot_si256(m, s); // +ve
233 tm = _mm256_or_si256(n, p);
234 tm = _mm256_shuffle_epi32(tm, _MM_SHUFFLE(2, 0, 0, 0));
235 tm = _mm256_andnot_si256(half_mask, tm);
236
237 t = _mm256_or_si256(t, tm);
238 t = _mm256_permute4x64_epi64(t, _MM_SHUFFLE(3, 1, 2, 0));
239 _mm256_storeu_si256((__m256i*)dp, t);
240 }
241 }
242 }
243
245 static inline
246 __m256i ojph_mm256_max_ge_epi32(__m256i a, __m256i b, __m256 x, __m256 y)
247 {
248 // We must use _CMP_NLT_UQ or _CMP_GE_OQ, _CMP_GE_OS, or _CMP_NLT_US
249 // It is not clear to me which to use
250 __m256 ct = _mm256_cmp_ps(x, y, _CMP_NLT_UQ); // 0xFFFFFFFF for x >= y
251 __m256i c = _mm256_castps_si256(ct); // does not generate any code
252 __m256i d = _mm256_and_si256(c, a); // keep only a, where x >= y
253 __m256i e = _mm256_andnot_si256(c, b); // keep only b, where x < y
254 return _mm256_or_si256(d, e); // combine
255 }
256
258 static inline
259 __m256i ojph_mm256_min_lt_epi32(__m256i a, __m256i b, __m256 x, __m256 y)
260 {
261 // We must use _CMP_LT_OQ or _CMP_NGE_UQ, _CMP_LT_OS, or _CMP_NGE_US
262 // It is not clear to me which to use
263 __m256 ct = _mm256_cmp_ps(x, y, _CMP_NGE_UQ); // 0xFFFFFFFF for x < y
264 __m256i c = _mm256_castps_si256(ct); // does not generate any code
265 __m256i d = _mm256_and_si256(c, a); // keep only a, where x < y
266 __m256i e = _mm256_andnot_si256(c, b); // keep only b, where x >= y
267 return _mm256_or_si256(d, e); // combine
268 }
269
271 template<bool NLT_TYPE3>
272 static inline
273 void local_avx2_irv_convert_to_integer(const line_buf *src_line,
274 line_buf *dst_line, ui32 dst_line_offset,
275 ui32 bit_depth, bool is_signed, ui32 width)
276 {
277 assert((src_line->flags & line_buf::LFT_32BIT) &&
278 (src_line->flags & line_buf::LFT_INTEGER) == 0 &&
279 (dst_line->flags & line_buf::LFT_32BIT) &&
280 (dst_line->flags & line_buf::LFT_INTEGER));
281
282 assert(bit_depth <= 32);
283 const float* sp = src_line->f32;
284 si32* dp = dst_line->i32 + dst_line_offset;
285 // There is the possibility that converting to integer will
286 // exceed the dynamic range of 32bit integer; therefore, care must be
287 // exercised.
288 // We look if the floating point number is outside the half-closed
289 // interval [-0.5f, 0.5f). If so, we limit the resulting integer
290 // to the maximum/minimum that number supports.
291 si32 neg_limit = (si32)INT_MIN >> (32 - bit_depth);
292 __m256 mul = _mm256_set1_ps((float)(1ull << bit_depth));
293 __m256 fl_up_lim = _mm256_set1_ps(-(float)neg_limit); // val < upper
294 __m256 fl_low_lim = _mm256_set1_ps((float)neg_limit); // val >= lower
295 __m256i s32_up_lim = _mm256_set1_epi32(INT_MAX >> (32 - bit_depth));
296 __m256i s32_low_lim = _mm256_set1_epi32(INT_MIN >> (32 - bit_depth));
297
298 if (is_signed)
299 {
300 __m256i zero = _mm256_setzero_si256();
301 __m256i bias =
302 _mm256_set1_epi32(-(si32)((1ULL << (bit_depth - 1)) + 1));
303 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
304 __m256 t = _mm256_loadu_ps(sp);
305 t = _mm256_mul_ps(t, mul);
306 __m256i u = _mm256_cvtps_epi32(t);
307 u = ojph_mm256_max_ge_epi32(u, s32_low_lim, t, fl_low_lim);
308 u = ojph_mm256_min_lt_epi32(u, s32_up_lim, t, fl_up_lim);
309 if (NLT_TYPE3)
310 {
311 __m256i c = _mm256_cmpgt_epi32(zero, u); // 0xFFFFFFFF for -ve val
312 __m256i neg = _mm256_sub_epi32(bias, u); // -bias -value
313 neg = _mm256_and_si256(c, neg); // keep only - bias - val
314 u = _mm256_andnot_si256(c, u); // keep only +ve or 0
315 u = _mm256_or_si256(neg, u); // combine
316 }
317 _mm256_storeu_si256((__m256i*)dp, u);
318 }
319 }
320 else
321 {
322 __m256i half = _mm256_set1_epi32((si32)(1ULL << (bit_depth - 1)));
323 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
324 __m256 t = _mm256_loadu_ps(sp);
325 t = _mm256_mul_ps(t, mul);
326 __m256i u = _mm256_cvtps_epi32(t);
327 u = ojph_mm256_max_ge_epi32(u, s32_low_lim, t, fl_low_lim);
328 u = ojph_mm256_min_lt_epi32(u, s32_up_lim, t, fl_up_lim);
329 u = _mm256_add_epi32(u, half);
330 _mm256_storeu_si256((__m256i*)dp, u);
331 }
332 }
333 }
334
336 void avx2_irv_convert_to_integer(const line_buf *src_line,
337 line_buf *dst_line, ui32 dst_line_offset,
338 ui32 bit_depth, bool is_signed, ui32 width)
339 {
340 local_avx2_irv_convert_to_integer<false>(src_line, dst_line,
341 dst_line_offset, bit_depth, is_signed, width);
342 }
343
345 void avx2_irv_convert_to_integer_nlt_type3(const line_buf *src_line,
346 line_buf *dst_line, ui32 dst_line_offset,
347 ui32 bit_depth, bool is_signed, ui32 width)
348 {
349 local_avx2_irv_convert_to_integer<true>(src_line, dst_line,
350 dst_line_offset, bit_depth, is_signed, width);
351 }
352
354 template<int NLT_TYPE>
355 static inline
356 void local_avx2_irv_convert_to_integer_nlt2or4(const line_buf *src_line,
357 line_buf *dst_line, ui32 dst_line_offset,
358 ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec* rec)
359 {
360 assert((src_line->flags & line_buf::LFT_32BIT) &&
361 (src_line->flags & line_buf::LFT_INTEGER) == 0 &&
362 (dst_line->flags & line_buf::LFT_32BIT) &&
363 (dst_line->flags & line_buf::LFT_INTEGER));
364 ojph_unused(bit_depth);
365 ojph_unused(is_signed);
366
367 assert(rec->get_bit_depth() <= 32);
368 const float* sp = src_line->f32;
369 si32* dp = dst_line->i32 + dst_line_offset;
370
371 __m256 mul = _mm256_set1_ps(rec->multiplier);
372 __m256 d_min = _mm256_set1_ps(rec->fd_min);
373 __m256 d_max = _mm256_set1_ps(rec->fd_max);
374 __m256 delta = _mm256_set1_ps(rec->delta);
375 __m256 inv_delta = _mm256_set1_ps(rec->inv_delta);
376 const float* lut = rec->dec_points;
377
378 __m256 half_ps = _mm256_set1_ps(0.5f);
379 __m256i one = _mm256_set1_epi32(1);
380
381 if (rec->is_signed())
382 {
383 __m256 half =
384 _mm256_set1_ps((float)(1ULL << (rec->get_bit_depth() - 1)));
385 __m256i bias =
386 _mm256_set1_epi32(-(si32)((1ULL << (rec->get_bit_depth() - 1)) + 1));
387 __m256i zero = _mm256_setzero_si256();
388 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
389 __m256 t = _mm256_loadu_ps(sp);
390 t = _mm256_add_ps(t, half_ps); // convert to [0, 1]
391 t = _mm256_max_ps(t, d_min);
392 t = _mm256_min_ps(t, d_max);
393 __m256i k = _mm256_cvttps_epi32(
394 _mm256_mul_ps(_mm256_sub_ps(t, d_min), inv_delta));
395 __m256 d_k = _mm256_add_ps(d_min,
396 _mm256_mul_ps(_mm256_cvtepi32_ps(k), delta));
397 __m256 t_k = _mm256_i32gather_ps(lut, k, 4);
398 __m256 t_kp1 = _mm256_i32gather_ps(lut, _mm256_add_epi32(k, one), 4);
399 __m256 z = _mm256_add_ps(t_k,
400 _mm256_mul_ps(_mm256_mul_ps(_mm256_sub_ps(t, d_k), inv_delta),
401 _mm256_sub_ps(t_kp1, t_k)));
402 __m256i v =
403 _mm256_cvtps_epi32(_mm256_sub_ps(_mm256_mul_ps(z, mul), half));
404 if (NLT_TYPE == 4)
405 {
406 __m256i c = _mm256_cmpgt_epi32(zero, v); // 0xFFFFFFFF for -ve val
407 __m256i neg = _mm256_sub_epi32(bias, v); // - bias - value
408 neg = _mm256_and_si256(c, neg); // keep only - bias - val
409 v = _mm256_andnot_si256(c, v); // keep only +ve or 0
410 v = _mm256_or_si256(neg, v); // combine
411 }
412 _mm256_storeu_si256((__m256i*)dp, v);
413 }
414 }
415 else
416 {
417 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
418 __m256 t = _mm256_loadu_ps(sp);
419 t = _mm256_add_ps(t, half_ps); // convert to [0, 1]
420 t = _mm256_max_ps(t, d_min);
421 t = _mm256_min_ps(t, d_max);
422 __m256i k = _mm256_cvttps_epi32(
423 _mm256_mul_ps(_mm256_sub_ps(t, d_min), inv_delta));
424 __m256 d_k = _mm256_add_ps(d_min,
425 _mm256_mul_ps(_mm256_cvtepi32_ps(k), delta));
426 __m256 t_k = _mm256_i32gather_ps(lut, k, 4);
427 __m256 t_kp1 =
428 _mm256_i32gather_ps(lut, _mm256_add_epi32(k, one), 4);
429 __m256 z = _mm256_add_ps(t_k,
430 _mm256_mul_ps(_mm256_mul_ps(_mm256_sub_ps(t, d_k), inv_delta),
431 _mm256_sub_ps(t_kp1, t_k)));
432 __m256i v = _mm256_cvtps_epi32(_mm256_mul_ps(z, mul));
433 _mm256_storeu_si256((__m256i*)dp, v);
434 }
435 }
436 }
437
439 void avx2_irv_convert_to_integer_nlt(const line_buf *src_line,
440 line_buf *dst_line, ui32 dst_line_offset,
441 ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec* rec)
442 {
443 using nl = nlt_rec::nonlinearity;
444 if (rec->get_type() == nl::OJPH_NLT_LUT_STYLE_NLT)
445 local_avx2_irv_convert_to_integer_nlt2or4<2>(src_line, dst_line,
446 dst_line_offset, bit_depth, is_signed, width, rec);
447 else if (rec->get_type() == nl::OJPH_NLT_BINARY_COMPLEMENT_PLUS_LUT)
448 local_avx2_irv_convert_to_integer_nlt2or4<4>(src_line, dst_line,
449 dst_line_offset, bit_depth, is_signed, width, rec);
450 else
451 assert(0);
452 }
453
455 template<bool NLT_TYPE3>
456 static inline
457 void local_avx2_irv_convert_to_float(const line_buf *src_line,
458 ui32 src_line_offset, line_buf *dst_line,
459 ui32 bit_depth, bool is_signed, ui32 width)
460 {
461 assert((src_line->flags & line_buf::LFT_32BIT) &&
462 (src_line->flags & line_buf::LFT_INTEGER) &&
463 (dst_line->flags & line_buf::LFT_32BIT) &&
464 (dst_line->flags & line_buf::LFT_INTEGER) == 0);
465
466 assert(bit_depth <= 32);
467 __m256 mul = _mm256_set1_ps((float)(1.0 / (double)(1ULL << bit_depth)));
468
469 const si32* sp = src_line->i32 + src_line_offset;
470 float* dp = dst_line->f32;
471 if (is_signed)
472 {
473 __m256i zero = _mm256_setzero_si256();
474 __m256i bias =
475 _mm256_set1_epi32(-(si32)((1ULL << (bit_depth - 1)) + 1));
476 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
477 __m256i t = _mm256_loadu_si256((__m256i*)sp);
478 if (NLT_TYPE3)
479 {
480 __m256i c = _mm256_cmpgt_epi32(zero, t); // 0xFFFFFFFF for -ve val
481 __m256i neg = _mm256_sub_epi32(bias, t); // - bias - value
482 neg = _mm256_and_si256(c, neg); // keep only - bias - val
483 c = _mm256_andnot_si256(c, t); // keep only +ve or 0
484 t = _mm256_or_si256(neg, c); // combine
485 }
486 __m256 v = _mm256_cvtepi32_ps(t);
487 v = _mm256_mul_ps(v, mul);
488 _mm256_storeu_ps(dp, v);
489 }
490 }
491 else
492 {
493 __m256i half = _mm256_set1_epi32((si32)(1ULL << (bit_depth - 1)));
494 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
495 __m256i t = _mm256_loadu_si256((__m256i*)sp);
496 t = _mm256_sub_epi32(t, half);
497 __m256 v = _mm256_cvtepi32_ps(t);
498 v = _mm256_mul_ps(v, mul);
499 _mm256_storeu_ps(dp, v);
500 }
501 }
502 }
503
505 void avx2_irv_convert_to_float(const line_buf *src_line,
506 ui32 src_line_offset, line_buf *dst_line,
507 ui32 bit_depth, bool is_signed, ui32 width)
508 {
509 local_avx2_irv_convert_to_float<false>(src_line, src_line_offset,
510 dst_line, bit_depth, is_signed, width);
511 }
512
514 void avx2_irv_convert_to_float_nlt_type3(const line_buf *src_line,
515 ui32 src_line_offset, line_buf *dst_line,
516 ui32 bit_depth, bool is_signed, ui32 width)
517 {
518 local_avx2_irv_convert_to_float<true>(src_line, src_line_offset,
519 dst_line, bit_depth, is_signed, width);
520 }
521
523 template<int NLT_TYPE>
524 static inline
525 void local_avx2_irv_convert_to_float_nlt2or4(const line_buf *src_line,
526 ui32 src_line_offset, line_buf *dst_line,
527 ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec* rec)
528 {
529 ojph_unused(is_signed);
530 assert((src_line->flags & line_buf::LFT_32BIT) &&
531 (src_line->flags & line_buf::LFT_INTEGER) &&
532 (dst_line->flags & line_buf::LFT_32BIT) &&
533 (dst_line->flags & line_buf::LFT_INTEGER) == 0);
534
535 assert(bit_depth <= 32);
536 __m256 mul = _mm256_set1_ps((float)(1.0 / (double)(1ULL << bit_depth)));
537 __m256 d_min = _mm256_set1_ps(rec->ft_min);
538 __m256 d_max = _mm256_set1_ps(rec->ft_max);
539 __m256 delta = _mm256_set1_ps(rec->delta);
540 __m256 inv_delta = _mm256_set1_ps(rec->inv_delta);
541 const float* lut = rec->enc_points;
542
543 __m256 half_ps = _mm256_set1_ps(0.5f);
544 __m256i one = _mm256_set1_epi32(1);
545
546 const si32* sp = src_line->i32 + src_line_offset;
547 float* dp = dst_line->f32;
548 if (rec->is_signed())
549 {
550 __m256i bias =
551 _mm256_set1_epi32(-(si32)((1ULL << (rec->get_bit_depth() - 1)) + 1));
552 __m256i zero = _mm256_setzero_si256();
553 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
554 __m256i v = _mm256_loadu_si256((__m256i*)sp);
555 if (NLT_TYPE == 4)
556 {
557 __m256i c = _mm256_cmpgt_epi32(zero, v); // 0xFFFFFFFF for -ve val
558 __m256i neg = _mm256_sub_epi32(bias, v); // - bias - value
559 neg = _mm256_and_si256(c, neg); // keep only - bias - val
560 v = _mm256_andnot_si256(c, v); // keep only +ve or 0
561 v = _mm256_or_si256(neg, v); // combine
562 }
563 __m256 t = _mm256_add_ps( // convert to [0, 1]
564 _mm256_mul_ps(_mm256_cvtepi32_ps(v), mul), half_ps);
565 t = _mm256_max_ps(t, d_min);
566 t = _mm256_min_ps(t, d_max);
567 __m256i k = _mm256_cvttps_epi32(
568 _mm256_mul_ps(_mm256_sub_ps(t, d_min), inv_delta));
569 __m256 d_k = _mm256_add_ps(d_min,
570 _mm256_mul_ps(_mm256_cvtepi32_ps(k), delta));
571 __m256 t_k = _mm256_i32gather_ps(lut, k, 4);
572 __m256 t_kp1 =
573 _mm256_i32gather_ps(lut, _mm256_add_epi32(k, one), 4);
574 __m256 y = _mm256_add_ps(t_k,
575 _mm256_mul_ps(_mm256_mul_ps(_mm256_sub_ps(t, d_k), inv_delta),
576 _mm256_sub_ps(t_kp1, t_k)));
577 _mm256_storeu_ps(dp, _mm256_sub_ps(y, half_ps));
578 }
579 }
580 else
581 {
582 for (int i = (int)width; i > 0; i -= 8, sp += 8, dp += 8) {
583 __m256i v = _mm256_loadu_si256((__m256i*)sp);
584 __m256 t = _mm256_mul_ps(_mm256_cvtepi32_ps(v), mul); // in [0, 1]
585 t = _mm256_max_ps(t, d_min);
586 t = _mm256_min_ps(t, d_max);
587 __m256i k = _mm256_cvttps_epi32(
588 _mm256_mul_ps(_mm256_sub_ps(t, d_min), inv_delta));
589 __m256 d_k = _mm256_add_ps(d_min,
590 _mm256_mul_ps(_mm256_cvtepi32_ps(k), delta));
591 __m256 t_k = _mm256_i32gather_ps(lut, k, 4);
592 __m256 t_kp1 =
593 _mm256_i32gather_ps(lut, _mm256_add_epi32(k, one), 4);
594 __m256 y = _mm256_add_ps(t_k,
595 _mm256_mul_ps(_mm256_mul_ps(_mm256_sub_ps(t, d_k), inv_delta),
596 _mm256_sub_ps(t_kp1, t_k)));
597 _mm256_storeu_ps(dp, _mm256_sub_ps(y, half_ps));
598 }
599 }
600 }
601
603 void avx2_irv_convert_to_float_nlt(const line_buf *src_line,
604 ui32 src_line_offset, line_buf *dst_line,
605 ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec* rec)
606 {
607 using nl = nlt_rec::nonlinearity;
608 if (rec->get_type() == nl::OJPH_NLT_LUT_STYLE_NLT)
609 local_avx2_irv_convert_to_float_nlt2or4<2>(src_line,
610 src_line_offset, dst_line, bit_depth, is_signed, width, rec);
611 else if (rec->get_type() == nl::OJPH_NLT_BINARY_COMPLEMENT_PLUS_LUT)
612 local_avx2_irv_convert_to_float_nlt2or4<4>(src_line,
613 src_line_offset, dst_line, bit_depth, is_signed, width, rec);
614 else
615 assert(0);
616 }
617
618
620 void avx2_rct_forward(const line_buf *r,
621 const line_buf *g,
622 const line_buf *b,
623 line_buf *y, line_buf *cb, line_buf *cr,
624 ui32 repeat)
625 {
626 assert((y->flags & line_buf::LFT_INTEGER) &&
627 (cb->flags & line_buf::LFT_INTEGER) &&
628 (cr->flags & line_buf::LFT_INTEGER) &&
629 (r->flags & line_buf::LFT_INTEGER) &&
630 (g->flags & line_buf::LFT_INTEGER) &&
631 (b->flags & line_buf::LFT_INTEGER));
632
633 if (y->flags & line_buf::LFT_32BIT)
634 {
635 assert((y->flags & line_buf::LFT_32BIT) &&
636 (cb->flags & line_buf::LFT_32BIT) &&
637 (cr->flags & line_buf::LFT_32BIT) &&
638 (r->flags & line_buf::LFT_32BIT) &&
639 (g->flags & line_buf::LFT_32BIT) &&
640 (b->flags & line_buf::LFT_32BIT));
641 const si32 *rp = r->i32, * gp = g->i32, * bp = b->i32;
642 si32 *yp = y->i32, * cbp = cb->i32, * crp = cr->i32;
643 for (int i = (repeat + 7) >> 3; i > 0; --i)
644 {
645 __m256i mr = _mm256_load_si256((__m256i*)rp);
646 __m256i mg = _mm256_load_si256((__m256i*)gp);
647 __m256i mb = _mm256_load_si256((__m256i*)bp);
648 __m256i t = _mm256_add_epi32(mr, mb);
649 t = _mm256_add_epi32(t, _mm256_slli_epi32(mg, 1));
650 _mm256_store_si256((__m256i*)yp, _mm256_srai_epi32(t, 2));
651 t = _mm256_sub_epi32(mb, mg);
652 _mm256_store_si256((__m256i*)cbp, t);
653 t = _mm256_sub_epi32(mr, mg);
654 _mm256_store_si256((__m256i*)crp, t);
655
656 rp += 8; gp += 8; bp += 8;
657 yp += 8; cbp += 8; crp += 8;
658 }
659 }
660 else
661 {
662 assert((y->flags & line_buf::LFT_64BIT) &&
663 (cb->flags & line_buf::LFT_64BIT) &&
664 (cr->flags & line_buf::LFT_64BIT) &&
665 (r->flags & line_buf::LFT_32BIT) &&
666 (g->flags & line_buf::LFT_32BIT) &&
667 (b->flags & line_buf::LFT_32BIT));
668 __m256i v2 = _mm256_set1_epi64x(1ULL << (63 - 2));
669 const si32 *rp = r->i32, *gp = g->i32, *bp = b->i32;
670 si64 *yp = y->i64, *cbp = cb->i64, *crp = cr->i64;
671 for (int i = (repeat + 7) >> 3; i > 0; --i)
672 {
673 __m256i mr32 = _mm256_load_si256((__m256i*)rp);
674 __m256i mg32 = _mm256_load_si256((__m256i*)gp);
675 __m256i mb32 = _mm256_load_si256((__m256i*)bp);
676 __m256i mr, mg, mb, t;
677 mr = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mr32, 0));
678 mg = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mg32, 0));
679 mb = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mb32, 0));
680
681 t = _mm256_add_epi64(mr, mb);
682 t = _mm256_add_epi64(t, _mm256_slli_epi64(mg, 1));
683 _mm256_store_si256((__m256i*)yp, avx2_mm256_srai_epi64(t, 2, v2));
684 t = _mm256_sub_epi64(mb, mg);
685 _mm256_store_si256((__m256i*)cbp, t);
686 t = _mm256_sub_epi64(mr, mg);
687 _mm256_store_si256((__m256i*)crp, t);
688
689 yp += 4; cbp += 4; crp += 4;
690
691 mr = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mr32, 1));
692 mg = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mg32, 1));
693 mb = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(mb32, 1));
694
695 t = _mm256_add_epi64(mr, mb);
696 t = _mm256_add_epi64(t, _mm256_slli_epi64(mg, 1));
697 _mm256_store_si256((__m256i*)yp, avx2_mm256_srai_epi64(t, 2, v2));
698 t = _mm256_sub_epi64(mb, mg);
699 _mm256_store_si256((__m256i*)cbp, t);
700 t = _mm256_sub_epi64(mr, mg);
701 _mm256_store_si256((__m256i*)crp, t);
702
703 rp += 8; gp += 8; bp += 8;
704 yp += 4; cbp += 4; crp += 4;
705 }
706 }
707 }
708
710 void avx2_rct_backward(const line_buf *y,
711 const line_buf *cb,
712 const line_buf *cr,
713 line_buf *r, line_buf *g, line_buf *b,
714 ui32 repeat)
715 {
716 assert((y->flags & line_buf::LFT_INTEGER) &&
717 (cb->flags & line_buf::LFT_INTEGER) &&
718 (cr->flags & line_buf::LFT_INTEGER) &&
719 (r->flags & line_buf::LFT_INTEGER) &&
720 (g->flags & line_buf::LFT_INTEGER) &&
721 (b->flags & line_buf::LFT_INTEGER));
722
723 if (y->flags & line_buf::LFT_32BIT)
724 {
725 assert((y->flags & line_buf::LFT_32BIT) &&
726 (cb->flags & line_buf::LFT_32BIT) &&
727 (cr->flags & line_buf::LFT_32BIT) &&
728 (r->flags & line_buf::LFT_32BIT) &&
729 (g->flags & line_buf::LFT_32BIT) &&
730 (b->flags & line_buf::LFT_32BIT));
731 const si32 *yp = y->i32, *cbp = cb->i32, *crp = cr->i32;
732 si32 *rp = r->i32, *gp = g->i32, *bp = b->i32;
733 for (int i = (repeat + 7) >> 3; i > 0; --i)
734 {
735 __m256i my = _mm256_load_si256((__m256i*)yp);
736 __m256i mcb = _mm256_load_si256((__m256i*)cbp);
737 __m256i mcr = _mm256_load_si256((__m256i*)crp);
738
739 __m256i t = _mm256_add_epi32(mcb, mcr);
740 t = _mm256_sub_epi32(my, _mm256_srai_epi32(t, 2));
741 _mm256_store_si256((__m256i*)gp, t);
742 __m256i u = _mm256_add_epi32(mcb, t);
743 _mm256_store_si256((__m256i*)bp, u);
744 u = _mm256_add_epi32(mcr, t);
745 _mm256_store_si256((__m256i*)rp, u);
746
747 yp += 8; cbp += 8; crp += 8;
748 rp += 8; gp += 8; bp += 8;
749 }
750 }
751 else
752 {
753 assert((y->flags & line_buf::LFT_64BIT) &&
754 (cb->flags & line_buf::LFT_64BIT) &&
755 (cr->flags & line_buf::LFT_64BIT) &&
756 (r->flags & line_buf::LFT_32BIT) &&
757 (g->flags & line_buf::LFT_32BIT) &&
758 (b->flags & line_buf::LFT_32BIT));
759 __m256i v2 = _mm256_set1_epi64x(1ULL << (63 - 2));
760 __m256i low_bits = _mm256_set_epi64x(0, (si64)ULLONG_MAX,
761 0, (si64)ULLONG_MAX);
762 const si64 *yp = y->i64, *cbp = cb->i64, *crp = cr->i64;
763 si32 *rp = r->i32, *gp = g->i32, *bp = b->i32;
764 for (int i = (repeat + 7) >> 3; i > 0; --i)
765 {
766 __m256i my, mcb, mcr, tr, tg, tb;
767 my = _mm256_load_si256((__m256i*)yp);
768 mcb = _mm256_load_si256((__m256i*)cbp);
769 mcr = _mm256_load_si256((__m256i*)crp);
770
771 tg = _mm256_add_epi64(mcb, mcr);
772 tg = _mm256_sub_epi64(my, avx2_mm256_srai_epi64(tg, 2, v2));
773 tb = _mm256_add_epi64(mcb, tg);
774 tr = _mm256_add_epi64(mcr, tg);
775
776 __m256i mr, mg, mb;
777 mr = _mm256_shuffle_epi32(tr, _MM_SHUFFLE(0, 0, 2, 0));
778 mr = _mm256_and_si256(low_bits, mr);
779 mg = _mm256_shuffle_epi32(tg, _MM_SHUFFLE(0, 0, 2, 0));
780 mg = _mm256_and_si256(low_bits, mg);
781 mb = _mm256_shuffle_epi32(tb, _MM_SHUFFLE(0, 0, 2, 0));
782 mb = _mm256_and_si256(low_bits, mb);
783
784 yp += 4; cbp += 4; crp += 4;
785
786 my = _mm256_load_si256((__m256i*)yp);
787 mcb = _mm256_load_si256((__m256i*)cbp);
788 mcr = _mm256_load_si256((__m256i*)crp);
789
790 tg = _mm256_add_epi64(mcb, mcr);
791 tg = _mm256_sub_epi64(my, avx2_mm256_srai_epi64(tg, 2, v2));
792 tb = _mm256_add_epi64(mcb, tg);
793 tr = _mm256_add_epi64(mcr, tg);
794
795 tr = _mm256_shuffle_epi32(tr, _MM_SHUFFLE(2, 0, 0, 0));
796 tr = _mm256_andnot_si256(low_bits, tr);
797 mr = _mm256_or_si256(mr, tr);
798 mr = _mm256_permute4x64_epi64(mr, _MM_SHUFFLE(3, 1, 2, 0));
799
800 tg = _mm256_shuffle_epi32(tg, _MM_SHUFFLE(2, 0, 0, 0));
801 tg = _mm256_andnot_si256(low_bits, tg);
802 mg = _mm256_or_si256(mg, tg);
803 mg = _mm256_permute4x64_epi64(mg, _MM_SHUFFLE(3, 1, 2, 0));
804
805 tb = _mm256_shuffle_epi32(tb, _MM_SHUFFLE(2, 0, 0, 0));
806 tb = _mm256_andnot_si256(low_bits, tb);
807 mb = _mm256_or_si256(mb, tb);
808 mb = _mm256_permute4x64_epi64(mb, _MM_SHUFFLE(3, 1, 2, 0));
809
810 _mm256_store_si256((__m256i*)rp, mr);
811 _mm256_store_si256((__m256i*)gp, mg);
812 _mm256_store_si256((__m256i*)bp, mb);
813
814 yp += 4; cbp += 4; crp += 4;
815 rp += 8; gp += 8; bp += 8;
816 }
817 }
818 }
819
820 }
821}
822
823#endif
ojph::line_buf::LFT_INTEGER
@ LFT_INTEGER
Definition ojph_mem.h:170
ojph::line_buf::LFT_32BIT
@ LFT_32BIT
Definition ojph_mem.h:168
ojph::line_buf::LFT_64BIT
@ LFT_64BIT
Definition ojph_mem.h:169
ojph::local::avx2_rct_forward
void avx2_rct_forward(const line_buf *r, const line_buf *g, const line_buf *b, line_buf *y, line_buf *cb, line_buf *cr, ui32 repeat)
ojph::local::avx2_rct_backward
void avx2_rct_backward(const line_buf *y, const line_buf *cb, const line_buf *cr, line_buf *r, line_buf *g, line_buf *b, ui32 repeat)
ojph::local::avx2_rev_convert
void avx2_rev_convert(const line_buf *src_line, const ui32 src_line_offset, line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width)
ojph::local::avx2_irv_convert_to_float
void avx2_irv_convert_to_float(const line_buf *src_line, ui32 src_line_offset, line_buf *dst_line, ui32 bit_depth, bool is_signed, ui32 width)
ojph::local::avx2_irv_convert_to_integer_nlt
void avx2_irv_convert_to_integer_nlt(const line_buf *src_line, line_buf *dst_line, ui32 dst_line_offset, ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec *rec)
ojph::local::avx2_rev_convert_nlt_type3
void avx2_rev_convert_nlt_type3(const line_buf *src_line, const ui32 src_line_offset, line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width)
ojph::local::avx2_irv_convert_to_float_nlt
void avx2_irv_convert_to_float_nlt(const line_buf *src_line, ui32 src_line_offset, line_buf *dst_line, ui32 bit_depth, bool is_signed, ui32 width, const nlt_rec *rec)
ojph::local::avx2_irv_convert_to_integer
void avx2_irv_convert_to_integer(const line_buf *src_line, line_buf *dst_line, ui32 dst_line_offset, ui32 bit_depth, bool is_signed, ui32 width)
ojph::local::avx2_irv_convert_to_float_nlt_type3
void avx2_irv_convert_to_float_nlt_type3(const line_buf *src_line, ui32 src_line_offset, line_buf *dst_line, ui32 bit_depth, bool is_signed, ui32 width)
ojph::local::avx2_irv_convert_to_integer_nlt_type3
void avx2_irv_convert_to_integer_nlt_type3(const line_buf *src_line, line_buf *dst_line, ui32 dst_line_offset, ui32 bit_depth, bool is_signed, ui32 width)
ojph::si64
int64_t si64
Definition ojph_defs.h:57
ojph::si32
int32_t si32
Definition ojph_defs.h:55
ojph::ui32
uint32_t ui32
Definition ojph_defs.h:54
ojph_arch.h
ojph_colour.h
ojph_defs.h
ojph_unused
#define ojph_unused(x)
Definition ojph_defs.h:78
ojph_mem.h
ojph_params.h
ojph_params_local.h
ojph::local::nlt_rec
Definition ojph_params_local.h:840
ojph::local::nlt_rec::nonlinearity
ojph::param_nlt::nonlinearity nonlinearity
Definition ojph_params_local.h:841