From 2a584248766f34ec3b89184023ba8b1cf48c6587 Mon Sep 17 00:00:00 2001 From: "Jeff Roberts (Bellevue)" Date: Wed, 19 Jun 2024 10:41:02 -0700 Subject: [PATCH] 2.09 resize - fix defines for GCC arm 32 --- stb_image_resize2.h | 293 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 244 insertions(+), 49 deletions(-) diff --git a/stb_image_resize2.h b/stb_image_resize2.h index f48c509d5..86e66d281 100644 --- a/stb_image_resize2.h +++ b/stb_image_resize2.h @@ -1,4 +1,4 @@ -/* stb_image_resize2 - v2.07 - public domain image resizing +/* stb_image_resize2 - v2.09 - public domain image resizing by Jeff Roberts (v2) and Jorge L Rodriguez http://github.com/nothings/stb @@ -320,7 +320,7 @@ CONTRIBUTORS Jeff Roberts: 2.0 implementation, optimizations, SIMD - Martins Mozeiko: NEON simd, WASM simd, clang and GCC whisperer. + Martins Mozeiko: NEON simd, WASM simd, clang and GCC whisperer Fabian Giesen: half float and srgb converters Sean Barrett: API design, optimizations Jorge L Rodriguez: Original 1.0 implementation @@ -328,19 +328,24 @@ Nathan Reed: warning fixes for 1.0 REVISIONS + 2.09 (2024-06-19) fix the defines for 32-bit ARM GCC builds (was selecting + hardware half floats). + 2.08 (2024-06-10) fix for RGB->BGR three channel flips and add SIMD (thanks + to Ryan Salsbury), fix for sub-rect resizes, use the + pragmas to control unrolling when they are available. 2.07 (2024-05-24) fix for slow final split during threaded conversions of very wide scanlines when downsampling (caused by extra input converting), fix for wide scanline resamples with many splits (int overflow), fix GCC warning. 2.06 (2024-02-10) fix for identical width/height 3x or more down-scaling - undersampling a single row on rare resize ratios (about 1%) - 2.05 (2024-02-07) fix for 2 pixel to 1 pixel resizes with wrap (thanks Aras) - fix for output callback (thanks Julien Koenen) + undersampling a single row on rare resize ratios (about 1%). + 2.05 (2024-02-07) fix for 2 pixel to 1 pixel resizes with wrap (thanks Aras), + fix for output callback (thanks Julien Koenen). 2.04 (2023-11-17) fix for rare AVX bug, shadowed symbol (thanks Nikola Smiljanic). 2.03 (2023-11-01) ASAN and TSAN warnings fixed, minor tweaks. 2.00 (2023-10-10) mostly new source: new api, optimizations, simd, vertical-first, etc - (2x-5x faster without simd, 4x-12x faster with simd) - (in some cases, 20x to 40x faster - resizing to very small for example) + 2x-5x faster without simd, 4x-12x faster with simd, + in some cases, 20x to 40x faster esp resizing large to very small. 0.96 (2019-03-04) fixed warnings 0.95 (2017-07-23) fixed warnings 0.94 (2017-03-18) fixed warnings @@ -410,13 +415,13 @@ typedef uint64_t stbir_uint64; #endif #endif -#if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) || defined(_M_ARM) || (__ARM_NEON_FP & 4) != 0 && __ARM_FP16_FORMAT_IEEE != 0 +#if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) || ((__ARM_NEON_FP & 4) != 0) || defined(__ARM_NEON__) #ifndef STBIR_NEON #define STBIR_NEON #endif #endif -#if defined(_M_ARM) +#if defined(_M_ARM) || defined(__arm__) #ifdef STBIR_USE_FMA #undef STBIR_USE_FMA // no FMA for 32-bit arm on MSVC #endif @@ -1094,10 +1099,11 @@ struct stbir__info #define stbir__small_float ((float)1 / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20)) // min/max friendly -#define STBIR_CLAMP(x, xmin, xmax) do { \ +#define STBIR_CLAMP(x, xmin, xmax) for(;;) { \ if ( (x) < (xmin) ) (x) = (xmin); \ if ( (x) > (xmax) ) (x) = (xmax); \ -} while (0) + break; \ +} static stbir__inline int stbir__min(int a, int b) { @@ -1194,19 +1200,35 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS 4 // when threading, what is the minimum number of scanlines for a split? #endif -// restrict pointers for the output pointers +// restrict pointers for the output pointers, other loop and unroll control #if defined( _MSC_VER ) && !defined(__clang__) #define STBIR_STREAMOUT_PTR( star ) star __restrict #define STBIR_NO_UNROLL( ptr ) __assume(ptr) // this oddly keeps msvc from unrolling a loop -#elif defined( __clang__ ) + #if _MSC_VER >= 1900 + #define STBIR_NO_UNROLL_LOOP_START __pragma(loop( no_vector )) + #else + #define STBIR_NO_UNROLL_LOOP_START + #endif +#elif defined( __clang__ ) #define STBIR_STREAMOUT_PTR( star ) star __restrict__ - #define STBIR_NO_UNROLL( ptr ) __asm__ (""::"r"(ptr)) -#elif defined( __GNUC__ ) + #define STBIR_NO_UNROLL( ptr ) __asm__ (""::"r"(ptr)) + #if ( __clang_major__ >= 4 ) || ( ( __clang_major__ >= 3 ) && ( __clang_minor__ >= 5 ) ) + #define STBIR_NO_UNROLL_LOOP_START _Pragma("clang loop unroll(disable)") _Pragma("clang loop vectorize(disable)") + #else + #define STBIR_NO_UNROLL_LOOP_START + #endif +#elif defined( __GNUC__ ) #define STBIR_STREAMOUT_PTR( star ) star __restrict__ #define STBIR_NO_UNROLL( ptr ) __asm__ (""::"r"(ptr)) + #if __GNUC__ >= 14 + #define STBIR_NO_UNROLL_LOOP_START _Pragma("GCC unroll 0") _Pragma("GCC novector") + #else + #define STBIR_NO_UNROLL_LOOP_START + #endif #else #define STBIR_STREAMOUT_PTR( star ) star #define STBIR_NO_UNROLL( ptr ) + #define STBIR_NO_UNROLL_LOOP_START #endif #ifdef STBIR_NO_SIMD // force simd off for whatever reason @@ -1758,11 +1780,19 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) ((stbir_uint64)(4*b+0)<<32) | ((stbir_uint64)(4*b+1)<<40) | ((stbir_uint64)(4*b+2)<<48) | ((stbir_uint64)(4*b+3)<<56)), \ vcreate_u8( (4*c+0) | ((4*c+1)<<8) | ((4*c+2)<<16) | ((4*c+3)<<24) | \ ((stbir_uint64)(4*d+0)<<32) | ((stbir_uint64)(4*d+1)<<40) | ((stbir_uint64)(4*d+2)<<48) | ((stbir_uint64)(4*d+3)<<56) ) ) + + static stbir__inline uint8x16x2_t stbir_make16x2(float32x4_t rega,float32x4_t regb) + { + uint8x16x2_t r = { vreinterpretq_u8_f32(rega), vreinterpretq_u8_f32(regb) }; + return r; + } #else #define stbir_make16(a,b,c,d) (uint8x16_t){4*a+0,4*a+1,4*a+2,4*a+3,4*b+0,4*b+1,4*b+2,4*b+3,4*c+0,4*c+1,4*c+2,4*c+3,4*d+0,4*d+1,4*d+2,4*d+3} + #define stbir_make16x2(a,b) (uint8x16x2_t){{vreinterpretq_u8_f32(a),vreinterpretq_u8_f32(b)}} #endif #define stbir__simdf_swiz( reg, one, two, three, four ) vreinterpretq_f32_u8( vqtbl1q_u8( vreinterpretq_u8_f32(reg), stbir_make16(one, two, three, four) ) ) + #define stbir__simdf_swiz2( rega, regb, one, two, three, four ) vreinterpretq_f32_u8( vqtbl2q_u8( stbir_make16x2(rega,regb), stbir_make16(one, two, three, four) ) ) #define stbir__simdi_16madd( out, reg0, reg1 ) \ { \ @@ -2146,7 +2176,7 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #endif -#if defined(STBIR_NEON) && !defined(_M_ARM) +#if defined(STBIR_NEON) && !defined(_M_ARM) && !defined(__arm__) #if defined( _MSC_VER ) && !defined(__clang__) typedef __int16 stbir__FP16; @@ -2163,7 +2193,7 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #endif -#if !defined(STBIR_NEON) && !defined(STBIR_FP16C) || defined(STBIR_NEON) && defined(_M_ARM) +#if (!defined(STBIR_NEON) && !defined(STBIR_FP16C)) || (defined(STBIR_NEON) && defined(_M_ARM)) || (defined(STBIR_NEON) && defined(__arm__)) // Fabian's half float routines, see: https://gist.github.com/rygorous/2156668 @@ -2390,7 +2420,7 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) stbir__simdi_store( output,final ); } -#elif defined(STBIR_WASM) || (defined(STBIR_NEON) && defined(_MSC_VER) && defined(_M_ARM)) // WASM or 32-bit ARM on MSVC/clang +#elif defined(STBIR_WASM) || (defined(STBIR_NEON) && (defined(_MSC_VER) || defined(_M_ARM) || defined(__arm__))) // WASM or 32-bit ARM on MSVC/clang static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input) { @@ -2436,7 +2466,7 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) return vget_lane_f16(vcvt_f16_f32(vdupq_n_f32(f)), 0).n16_u16[0]; } -#elif defined(STBIR_NEON) // 64-bit ARM +#elif defined(STBIR_NEON) && ( defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) ) // 64-bit ARM static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input) { @@ -2514,6 +2544,7 @@ static const STBIR__SIMDI_CONST(STBIR_topscale, 0x02000000); // Adding this switch saves about 5K on clang which is Captain Unroll the 3rd. #define STBIR_SIMD_STREAMOUT_PTR( star ) STBIR_STREAMOUT_PTR( star ) #define STBIR_SIMD_NO_UNROLL(ptr) STBIR_NO_UNROLL(ptr) +#define STBIR_SIMD_NO_UNROLL_LOOP_START STBIR_NO_UNROLL_LOOP_START #ifdef STBIR_MEMCPY #undef STBIR_MEMCPY @@ -2536,6 +2567,7 @@ static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes ) { if ( bytes ) { + STBIR_SIMD_NO_UNROLL_LOOP_START do { STBIR_SIMD_NO_UNROLL(d); @@ -2552,6 +2584,7 @@ static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes ) stbir__simdf_store( d, x ); d = (char*)( ( ( (size_t)d ) + 16 ) & ~15 ); + STBIR_SIMD_NO_UNROLL_LOOP_START for(;;) { STBIR_SIMD_NO_UNROLL(d); @@ -2584,6 +2617,7 @@ static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes ) stbir__simdfX_store( d + 12*stbir__simdfX_float_count, x3 ); d = (char*)( ( ( (size_t)d ) + (16*stbir__simdfX_float_count) ) & ~((16*stbir__simdfX_float_count)-1) ); + STBIR_SIMD_NO_UNROLL_LOOP_START for(;;) { STBIR_SIMD_NO_UNROLL(d); @@ -2620,6 +2654,7 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte if ( ofs_to_dest >= 16 ) // is the overlap more than 16 away? { char STBIR_SIMD_STREAMOUT_PTR( * ) s_end16 = ((char*) src) + (bytes&~15); + STBIR_SIMD_NO_UNROLL_LOOP_START do { stbir__simdf x; @@ -2646,6 +2681,7 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte // when in scalar mode, we let unrolling happen, so this macro just does the __restrict #define STBIR_SIMD_STREAMOUT_PTR( star ) STBIR_STREAMOUT_PTR( star ) #define STBIR_SIMD_NO_UNROLL(ptr) +#define STBIR_SIMD_NO_UNROLL_LOOP_START #endif // SSE2 @@ -2757,7 +2793,7 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte #ifndef STBIR_SIMD -// memcpy that is specically intentionally overlapping (src is smaller then dest, so can be +// memcpy that is specifically intentionally overlapping (src is smaller then dest, so can be // a normal forward copy, bytes is divisible by 4 and bytes is greater than or equal to // the diff between dest and src) static void stbir_overlapping_memcpy( void * dest, void const * src, size_t bytes ) @@ -2769,6 +2805,7 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte if ( ofs_to_dest >= 8 ) // is the overlap more than 8 away? { char STBIR_SIMD_STREAMOUT_PTR( * ) s_end8 = ((char*) src) + (bytes&~7); + STBIR_NO_UNROLL_LOOP_START do { STBIR_NO_UNROLL(sd); @@ -2780,6 +2817,7 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte return; } + STBIR_NO_UNROLL_LOOP_START do { STBIR_NO_UNROLL(sd); @@ -2884,13 +2922,6 @@ static float stbir__filter_mitchell(float x, float s, void * user_data) return (0.0f); } -static float stbir__support_zero(float s, void * user_data) -{ - STBIR__UNUSED(s); - STBIR__UNUSED(user_data); - return 0; -} - static float stbir__support_zeropoint5(float s, void * user_data) { STBIR__UNUSED(s); @@ -3598,6 +3629,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* switch( widest ) { case 1: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_1( pc, coeffs ); ++pc; @@ -3605,6 +3637,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 2: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_2( pc, coeffs ); pc += 2; @@ -3612,6 +3645,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 3: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_2( pc, coeffs ); STBIR_MOVE_1( pc+2, coeffs+2 ); @@ -3620,6 +3654,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 4: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_4( pc, coeffs ); pc += 4; @@ -3627,6 +3662,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 5: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_4( pc, coeffs ); STBIR_MOVE_1( pc+4, coeffs+4 ); @@ -3635,6 +3671,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 6: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_4( pc, coeffs ); STBIR_MOVE_2( pc+4, coeffs+4 ); @@ -3643,6 +3680,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 7: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_4( pc, coeffs ); STBIR_MOVE_2( pc+4, coeffs+4 ); @@ -3652,6 +3690,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 8: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_4( pc, coeffs ); STBIR_MOVE_4( pc+4, coeffs+4 ); @@ -3660,6 +3699,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 9: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_4( pc, coeffs ); STBIR_MOVE_4( pc+4, coeffs+4 ); @@ -3669,6 +3709,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 10: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_4( pc, coeffs ); STBIR_MOVE_4( pc+4, coeffs+4 ); @@ -3678,6 +3719,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 11: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_4( pc, coeffs ); STBIR_MOVE_4( pc+4, coeffs+4 ); @@ -3688,6 +3730,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; case 12: + STBIR_NO_UNROLL_LOOP_START do { STBIR_MOVE_4( pc, coeffs ); STBIR_MOVE_4( pc+4, coeffs+4 ); @@ -3697,6 +3740,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* } while ( pc < pc_end ); break; default: + STBIR_NO_UNROLL_LOOP_START do { float * copy_end = pc + widest - 4; float * c = coeffs; @@ -3707,6 +3751,7 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* c += 4; } while ( pc <= copy_end ); copy_end += 4; + STBIR_NO_UNROLL_LOOP_START while ( pc < copy_end ) { STBIR_MOVE_1( pc, c ); @@ -4017,6 +4062,7 @@ static void stbir__fancy_alpha_weight_4ch( float * out_buffer, int width_times_c #ifdef STBIR_SIMD8 decode += 16; + STBIR_NO_UNROLL_LOOP_START while ( decode <= end_decode ) { stbir__simdf8 d0,d1,a0,a1,p0,p1; @@ -4041,6 +4087,7 @@ static void stbir__fancy_alpha_weight_4ch( float * out_buffer, int width_times_c decode -= 16; #else decode += 8; + STBIR_NO_UNROLL_LOOP_START while ( decode <= end_decode ) { stbir__simdf d0,a0,d1,a1,p0,p1; @@ -4063,12 +4110,14 @@ static void stbir__fancy_alpha_weight_4ch( float * out_buffer, int width_times_c // might be one last odd pixel #ifdef STBIR_SIMD8 + STBIR_NO_UNROLL_LOOP_START while ( decode < end_decode ) #else if ( decode < end_decode ) #endif { stbir__simdf d,a,p; + STBIR_NO_UNROLL(decode); stbir__simdf_load( d, decode ); stbir__simdf_0123to3333( a, d ); stbir__simdf_mult( p, a, d ); @@ -4110,6 +4159,7 @@ static void stbir__fancy_alpha_weight_2ch( float * out_buffer, int width_times_c decode += 8; if ( decode <= end_decode ) { + STBIR_NO_UNROLL_LOOP_START do { #ifdef STBIR_SIMD8 stbir__simdf8 d0,a0,p0; @@ -4153,6 +4203,7 @@ static void stbir__fancy_alpha_weight_2ch( float * out_buffer, int width_times_c decode -= 8; #endif + STBIR_SIMD_NO_UNROLL_LOOP_START while( decode < end_decode ) { float x = decode[0], y = decode[1]; @@ -4173,6 +4224,7 @@ static void stbir__fancy_alpha_unweight_4ch( float * encode_buffer, int width_ti // fancy RGBA is stored internally as R G B A Rpm Gpm Bpm + STBIR_SIMD_NO_UNROLL_LOOP_START do { float alpha = input[3]; #ifdef STBIR_SIMD @@ -4240,6 +4292,7 @@ static void stbir__simple_alpha_weight_4ch( float * decode_buffer, int width_tim #ifdef STBIR_SIMD { decode += 2 * stbir__simdfX_float_count; + STBIR_NO_UNROLL_LOOP_START while ( decode <= end_decode ) { stbir__simdfX d0,a0,d1,a1; @@ -4258,6 +4311,7 @@ static void stbir__simple_alpha_weight_4ch( float * decode_buffer, int width_tim // few last pixels remnants #ifdef STBIR_SIMD8 + STBIR_NO_UNROLL_LOOP_START while ( decode < end_decode ) #else if ( decode < end_decode ) @@ -4293,6 +4347,7 @@ static void stbir__simple_alpha_weight_2ch( float * decode_buffer, int width_tim #ifdef STBIR_SIMD decode += 2 * stbir__simdfX_float_count; + STBIR_NO_UNROLL_LOOP_START while ( decode <= end_decode ) { stbir__simdfX d0,a0,d1,a1; @@ -4310,6 +4365,7 @@ static void stbir__simple_alpha_weight_2ch( float * decode_buffer, int width_tim decode -= 2 * stbir__simdfX_float_count; #endif + STBIR_SIMD_NO_UNROLL_LOOP_START while( decode < end_decode ) { float alpha = decode[1]; @@ -4324,6 +4380,7 @@ static void stbir__simple_alpha_unweight_4ch( float * encode_buffer, int width_t float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer; float const * end_output = encode_buffer + width_times_channels; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float alpha = encode[3]; @@ -4371,9 +4428,77 @@ static void stbir__simple_flip_3ch( float * decode_buffer, int width_times_chann float STBIR_STREAMOUT_PTR(*) decode = decode_buffer; float const * end_decode = decode_buffer + width_times_channels; - decode += 12; +#ifdef STBIR_SIMD + #ifdef stbir__simdf_swiz2 // do we have two argument swizzles? + end_decode -= 12; + STBIR_NO_UNROLL_LOOP_START + while( decode <= end_decode ) + { + // on arm64 8 instructions, no overlapping stores + stbir__simdf a,b,c,na,nb; + STBIR_SIMD_NO_UNROLL(decode); + stbir__simdf_load( a, decode ); + stbir__simdf_load( b, decode+4 ); + stbir__simdf_load( c, decode+8 ); + + na = stbir__simdf_swiz2( a, b, 2, 1, 0, 5 ); + b = stbir__simdf_swiz2( a, b, 4, 3, 6, 7 ); + nb = stbir__simdf_swiz2( b, c, 0, 1, 4, 3 ); + c = stbir__simdf_swiz2( b, c, 2, 7, 6, 5 ); + + stbir__simdf_store( decode, na ); + stbir__simdf_store( decode+4, nb ); + stbir__simdf_store( decode+8, c ); + decode += 12; + } + end_decode += 12; + #else + end_decode -= 24; + STBIR_NO_UNROLL_LOOP_START + while( decode <= end_decode ) + { + // 26 instructions on x64 + stbir__simdf a,b,c,d,e,f,g; + float i21, i23; + STBIR_SIMD_NO_UNROLL(decode); + stbir__simdf_load( a, decode ); + stbir__simdf_load( b, decode+3 ); + stbir__simdf_load( c, decode+6 ); + stbir__simdf_load( d, decode+9 ); + stbir__simdf_load( e, decode+12 ); + stbir__simdf_load( f, decode+15 ); + stbir__simdf_load( g, decode+18 ); + + a = stbir__simdf_swiz( a, 2, 1, 0, 3 ); + b = stbir__simdf_swiz( b, 2, 1, 0, 3 ); + c = stbir__simdf_swiz( c, 2, 1, 0, 3 ); + d = stbir__simdf_swiz( d, 2, 1, 0, 3 ); + e = stbir__simdf_swiz( e, 2, 1, 0, 3 ); + f = stbir__simdf_swiz( f, 2, 1, 0, 3 ); + g = stbir__simdf_swiz( g, 2, 1, 0, 3 ); + + // stores overlap, need to be in order, + stbir__simdf_store( decode, a ); + i21 = decode[21]; + stbir__simdf_store( decode+3, b ); + i23 = decode[23]; + stbir__simdf_store( decode+6, c ); + stbir__simdf_store( decode+9, d ); + stbir__simdf_store( decode+12, e ); + stbir__simdf_store( decode+15, f ); + stbir__simdf_store( decode+18, g ); + decode[21] = i23; + decode[23] = i21; + decode += 24; + } + end_decode += 24; + #endif +#else + end_decode -= 12; + STBIR_NO_UNROLL_LOOP_START while( decode <= end_decode ) { + // 16 instructions float t0,t1,t2,t3; STBIR_NO_UNROLL(decode); t0 = decode[0]; t1 = decode[3]; t2 = decode[6]; t3 = decode[9]; @@ -4381,8 +4506,10 @@ static void stbir__simple_flip_3ch( float * decode_buffer, int width_times_chann decode[2] = t0; decode[5] = t1; decode[8] = t2; decode[11] = t3; decode += 12; } - decode -= 12; + end_decode += 12; +#endif + STBIR_NO_UNROLL_LOOP_START while( decode < end_decode ) { float t = decode[0]; @@ -7071,36 +7198,27 @@ static stbir__info * stbir__alloc_internal_mem_and_build_samplers( stbir__sample info->ring_buffer_num_entries = conservative_split_output_size; STBIR_ASSERT( info->ring_buffer_num_entries <= info->alloc_ring_buffer_num_entries ); - // a few of the horizontal gather functions read one dword past the end (but mask it out), so put in a normal value so no snans or denormals accidentally sneak in + // a few of the horizontal gather functions read past the end of the decode (but mask it out), + // so put in normal values so no snans or denormals accidentally sneak in (also, in the ring + // buffer for vertical first) for( i = 0 ; i < splits ; i++ ) { - int width, ofs; - - // find the right most span - if ( info->scanline_extents.spans[0].n1 > info->scanline_extents.spans[1].n1 ) - width = info->scanline_extents.spans[0].n1 - info->scanline_extents.spans[0].n0; - else - width = info->scanline_extents.spans[1].n1 - info->scanline_extents.spans[1].n0; + int t, ofs, start; - // this calc finds the exact end of the decoded scanline for all filter modes. - // usually this is just the width * effective channels. But we have to account - // for the area to the left of the scanline for wrap filtering and alignment, this - // is stored as a negative value in info->scanline_extents.conservative.n0. Next, - // we need to skip the exact size of the right hand size filter area (again for - // wrap mode), this is in info->scanline_extents.edge_sizes[1]). - ofs = ( width + 1 - info->scanline_extents.conservative.n0 + info->scanline_extents.edge_sizes[1] ) * effective_channels; + ofs = decode_buffer_size / 4; + start = ofs - 4; + if ( start < 0 ) start = 0; - // place a known, but numerically valid value in the decode buffer - info->split_info[i].decode_buffer[ ofs ] = 9999.0f; + for( t = start ; t < ofs; t++ ) + info->split_info[i].decode_buffer[ t ] = 9999.0f; - // if vertical filtering first, place a known, but numerically valid value in the all - // of the ring buffer accumulators if ( vertical_first ) { int j; for( j = 0; j < info->ring_buffer_num_entries ; j++ ) { - stbir__get_ring_buffer_entry( info, info->split_info + i, j )[ ofs ] = 9999.0f; + for( t = start ; t < ofs; t++ ) + stbir__get_ring_buffer_entry( info, info->split_info + i, j )[ t ] = 9999.0f; } } } @@ -8079,6 +8197,7 @@ static void STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * deco if ( width_times_channels >= 16 ) { decode_end -= 16; + STBIR_NO_UNROLL_LOOP_START for(;;) { #ifdef STBIR_SIMD8 @@ -8134,6 +8253,7 @@ static void STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * deco // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while( decode <= decode_end ) { STBIR_SIMD_NO_UNROLL(decode); @@ -8149,6 +8269,7 @@ static void STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * deco // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( decode < decode_end ) { STBIR_NO_UNROLL(decode); @@ -8175,6 +8296,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outpu { float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; end_output -= stbir__simdfX_float_count*2; + STBIR_NO_UNROLL_LOOP_START for(;;) { stbir__simdfX e0, e1; @@ -8206,6 +8328,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outpu // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four output += 4; + STBIR_NO_UNROLL_LOOP_START while( output <= end_output ) { stbir__simdf e0; @@ -8224,6 +8347,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outpu // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { stbir__simdf e0; @@ -8260,6 +8384,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outpu // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { float f; @@ -8289,6 +8414,7 @@ static void STBIR__CODER_NAME(stbir__decode_uint8_linear)( float * decodep, int if ( width_times_channels >= 16 ) { decode_end -= 16; + STBIR_NO_UNROLL_LOOP_START for(;;) { #ifdef STBIR_SIMD8 @@ -8338,6 +8464,7 @@ static void STBIR__CODER_NAME(stbir__decode_uint8_linear)( float * decodep, int // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while( decode <= decode_end ) { STBIR_SIMD_NO_UNROLL(decode); @@ -8353,6 +8480,7 @@ static void STBIR__CODER_NAME(stbir__decode_uint8_linear)( float * decodep, int // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( decode < decode_end ) { STBIR_NO_UNROLL(decode); @@ -8379,6 +8507,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear )( void * outputp, int { float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; end_output -= stbir__simdfX_float_count*2; + STBIR_SIMD_NO_UNROLL_LOOP_START for(;;) { stbir__simdfX e0, e1; @@ -8410,6 +8539,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear )( void * outputp, int // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four output += 4; + STBIR_NO_UNROLL_LOOP_START while( output <= end_output ) { stbir__simdf e0; @@ -8448,6 +8578,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear )( void * outputp, int // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { float f; @@ -8488,6 +8619,7 @@ static void STBIR__CODER_NAME(stbir__decode_uint8_srgb)( float * decodep, int wi // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( decode < decode_end ) { STBIR_NO_UNROLL(decode); @@ -8578,6 +8710,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb )( void * outputp, int w { float const * end_encode_m16 = encode + width_times_channels - 16; end_output -= 16; + STBIR_SIMD_NO_UNROLL_LOOP_START for(;;) { stbir__simdf f0, f1, f2, f3; @@ -8616,6 +8749,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb )( void * outputp, int w // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four output += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while ( output <= end_output ) { STBIR_SIMD_NO_UNROLL(encode); @@ -8633,6 +8767,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb )( void * outputp, int w // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { STBIR_NO_UNROLL(encode); @@ -8678,6 +8813,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb4_linearalpha )( void * o { float const * end_encode_m16 = encode + width_times_channels - 16; end_output -= 16; + STBIR_SIMD_NO_UNROLL_LOOP_START for(;;) { stbir__simdf f0, f1, f2, f3; @@ -8713,6 +8849,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb4_linearalpha )( void * o } #endif + STBIR_SIMD_NO_UNROLL_LOOP_START do { float f; STBIR_SIMD_NO_UNROLL(encode); @@ -8768,6 +8905,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb2_linearalpha )( void * o { float const * end_encode_m16 = encode + width_times_channels - 16; end_output -= 16; + STBIR_SIMD_NO_UNROLL_LOOP_START for(;;) { stbir__simdf f0, f1, f2, f3; @@ -8801,6 +8939,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb2_linearalpha )( void * o } #endif + STBIR_SIMD_NO_UNROLL_LOOP_START do { float f; STBIR_SIMD_NO_UNROLL(encode); @@ -8829,6 +8968,7 @@ static void STBIR__CODER_NAME(stbir__decode_uint16_linear_scaled)( float * decod if ( width_times_channels >= 8 ) { decode_end -= 8; + STBIR_NO_UNROLL_LOOP_START for(;;) { #ifdef STBIR_SIMD8 @@ -8872,6 +9012,7 @@ static void STBIR__CODER_NAME(stbir__decode_uint16_linear_scaled)( float * decod // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while( decode <= decode_end ) { STBIR_SIMD_NO_UNROLL(decode); @@ -8887,6 +9028,7 @@ static void STBIR__CODER_NAME(stbir__decode_uint16_linear_scaled)( float * decod // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( decode < decode_end ) { STBIR_NO_UNROLL(decode); @@ -8915,6 +9057,7 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * output { float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; end_output -= stbir__simdfX_float_count*2; + STBIR_SIMD_NO_UNROLL_LOOP_START for(;;) { stbir__simdfX e0, e1; @@ -8942,6 +9085,7 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * output // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four output += 4; + STBIR_NO_UNROLL_LOOP_START while( output <= end_output ) { stbir__simdf e; @@ -8960,6 +9104,7 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * output // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { stbir__simdf e; @@ -8981,6 +9126,7 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * output // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four output += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while( output <= end_output ) { float f; @@ -8997,6 +9143,7 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * output // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { float f; @@ -9026,6 +9173,7 @@ static void STBIR__CODER_NAME(stbir__decode_uint16_linear)( float * decodep, int if ( width_times_channels >= 8 ) { decode_end -= 8; + STBIR_NO_UNROLL_LOOP_START for(;;) { #ifdef STBIR_SIMD8 @@ -9066,6 +9214,7 @@ static void STBIR__CODER_NAME(stbir__decode_uint16_linear)( float * decodep, int // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while( decode <= decode_end ) { STBIR_SIMD_NO_UNROLL(decode); @@ -9081,6 +9230,7 @@ static void STBIR__CODER_NAME(stbir__decode_uint16_linear)( float * decodep, int // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( decode < decode_end ) { STBIR_NO_UNROLL(decode); @@ -9108,6 +9258,7 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, int { float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; end_output -= stbir__simdfX_float_count*2; + STBIR_SIMD_NO_UNROLL_LOOP_START for(;;) { stbir__simdfX e0, e1; @@ -9135,6 +9286,7 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, int // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four output += 4; + STBIR_NO_UNROLL_LOOP_START while( output <= end_output ) { stbir__simdf e; @@ -9156,6 +9308,7 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, int // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four output += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while( output <= end_output ) { float f; @@ -9174,6 +9327,7 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, int // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { float f; @@ -9202,6 +9356,7 @@ static void STBIR__CODER_NAME(stbir__decode_half_float_linear)( float * decodep, { stbir__FP16 const * end_input_m8 = input + width_times_channels - 8; decode_end -= 8; + STBIR_NO_UNROLL_LOOP_START for(;;) { STBIR_NO_UNROLL(decode); @@ -9243,6 +9398,7 @@ static void STBIR__CODER_NAME(stbir__decode_half_float_linear)( float * decodep, // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while( decode <= decode_end ) { STBIR_SIMD_NO_UNROLL(decode); @@ -9258,6 +9414,7 @@ static void STBIR__CODER_NAME(stbir__decode_half_float_linear)( float * decodep, // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( decode < decode_end ) { STBIR_NO_UNROLL(decode); @@ -9284,6 +9441,7 @@ static void STBIR__CODER_NAME( stbir__encode_half_float_linear )( void * outputp { float const * end_encode_m8 = encode + width_times_channels - 8; end_output -= 8; + STBIR_SIMD_NO_UNROLL_LOOP_START for(;;) { STBIR_SIMD_NO_UNROLL(encode); @@ -9324,6 +9482,7 @@ static void STBIR__CODER_NAME( stbir__encode_half_float_linear )( void * outputp // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four output += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while( output <= end_output ) { STBIR_SIMD_NO_UNROLL(output); @@ -9339,6 +9498,7 @@ static void STBIR__CODER_NAME( stbir__encode_half_float_linear )( void * outputp // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { STBIR_NO_UNROLL(output); @@ -9367,6 +9527,7 @@ static void STBIR__CODER_NAME(stbir__decode_float_linear)( float * decodep, int { float const * end_input_m16 = input + width_times_channels - 16; decode_end -= 16; + STBIR_NO_UNROLL_LOOP_START for(;;) { STBIR_NO_UNROLL(decode); @@ -9415,6 +9576,7 @@ static void STBIR__CODER_NAME(stbir__decode_float_linear)( float * decodep, int // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while( decode <= decode_end ) { STBIR_SIMD_NO_UNROLL(decode); @@ -9430,6 +9592,7 @@ static void STBIR__CODER_NAME(stbir__decode_float_linear)( float * decodep, int // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( decode < decode_end ) { STBIR_NO_UNROLL(decode); @@ -9489,6 +9652,7 @@ static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int { float const * end_encode_m8 = encode + width_times_channels - ( stbir__simdfX_float_count * 2 ); end_output -= ( stbir__simdfX_float_count * 2 ); + STBIR_SIMD_NO_UNROLL_LOOP_START for(;;) { stbir__simdfX e0, e1; @@ -9522,6 +9686,7 @@ static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four output += 4; + STBIR_NO_UNROLL_LOOP_START while( output <= end_output ) { stbir__simdf e0; @@ -9546,6 +9711,7 @@ static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int // try to do blocks of 4 when you can #if stbir__coder_min_num != 3 // doesn't divide cleanly by four output += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START while( output <= end_output ) { float e; @@ -9565,6 +9731,7 @@ static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int // do the remnants #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { float e; @@ -9675,6 +9842,7 @@ static void STBIR_chans( stbir__vertical_scatter_with_,_coeffs)( float ** output stbIF5(stbir__simdfX c5 = stbir__simdf_frepX( c5s ); ) stbIF6(stbir__simdfX c6 = stbir__simdf_frepX( c6s ); ) stbIF7(stbir__simdfX c7 = stbir__simdf_frepX( c7s ); ) + STBIR_SIMD_NO_UNROLL_LOOP_START while ( ( (char*)input_end - (char*) input ) >= (16*stbir__simdfX_float_count) ) { stbir__simdfX o0, o1, o2, o3, r0, r1, r2, r3; @@ -9729,6 +9897,7 @@ static void STBIR_chans( stbir__vertical_scatter_with_,_coeffs)( float ** output input += (4*stbir__simdfX_float_count); stbIF0( output0 += (4*stbir__simdfX_float_count); ) stbIF1( output1 += (4*stbir__simdfX_float_count); ) stbIF2( output2 += (4*stbir__simdfX_float_count); ) stbIF3( output3 += (4*stbir__simdfX_float_count); ) stbIF4( output4 += (4*stbir__simdfX_float_count); ) stbIF5( output5 += (4*stbir__simdfX_float_count); ) stbIF6( output6 += (4*stbir__simdfX_float_count); ) stbIF7( output7 += (4*stbir__simdfX_float_count); ) } + STBIR_SIMD_NO_UNROLL_LOOP_START while ( ( (char*)input_end - (char*) input ) >= 16 ) { stbir__simdf o0, r0; @@ -9761,6 +9930,7 @@ static void STBIR_chans( stbir__vertical_scatter_with_,_coeffs)( float ** output } } #else + STBIR_NO_UNROLL_LOOP_START while ( ( (char*)input_end - (char*) input ) >= 16 ) { float r0, r1, r2, r3; @@ -9792,6 +9962,7 @@ static void STBIR_chans( stbir__vertical_scatter_with_,_coeffs)( float ** output stbIF0( output0 += 4; ) stbIF1( output1 += 4; ) stbIF2( output2 += 4; ) stbIF3( output3 += 4; ) stbIF4( output4 += 4; ) stbIF5( output5 += 4; ) stbIF6( output6 += 4; ) stbIF7( output7 += 4; ) } #endif + STBIR_NO_UNROLL_LOOP_START while ( input < input_end ) { float r = input[0]; @@ -9855,6 +10026,7 @@ static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, stbIF6(stbir__simdfX c6 = stbir__simdf_frepX( c6s ); ) stbIF7(stbir__simdfX c7 = stbir__simdf_frepX( c7s ); ) + STBIR_SIMD_NO_UNROLL_LOOP_START while ( ( (char*)input0_end - (char*) input0 ) >= (16*stbir__simdfX_float_count) ) { stbir__simdfX o0, o1, o2, o3, r0, r1, r2, r3; @@ -9899,6 +10071,7 @@ static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, stbIF0( input0 += (4*stbir__simdfX_float_count); ) stbIF1( input1 += (4*stbir__simdfX_float_count); ) stbIF2( input2 += (4*stbir__simdfX_float_count); ) stbIF3( input3 += (4*stbir__simdfX_float_count); ) stbIF4( input4 += (4*stbir__simdfX_float_count); ) stbIF5( input5 += (4*stbir__simdfX_float_count); ) stbIF6( input6 += (4*stbir__simdfX_float_count); ) stbIF7( input7 += (4*stbir__simdfX_float_count); ) } + STBIR_SIMD_NO_UNROLL_LOOP_START while ( ( (char*)input0_end - (char*) input0 ) >= 16 ) { stbir__simdf o0, r0; @@ -9923,6 +10096,7 @@ static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, } } #else + STBIR_NO_UNROLL_LOOP_START while ( ( (char*)input0_end - (char*) input0 ) >= 16 ) { float o0, o1, o2, o3; @@ -9944,6 +10118,7 @@ static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, stbIF0( input0 += 4; ) stbIF1( input1 += 4; ) stbIF2( input2 += 4; ) stbIF3( input3 += 4; ) stbIF4( input4 += 4; ) stbIF5( input5 += 4; ) stbIF6( input6 += 4; ) stbIF7( input7 += 4; ) } #endif + STBIR_NO_UNROLL_LOOP_START while ( input0 < input0_end ) { float o0; @@ -10036,6 +10211,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_1_coeff)( floa { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10048,6 +10224,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_2_coeffs)( flo { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10060,6 +10237,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_3_coeffs)( flo { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10072,6 +10250,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_4_coeffs)( flo { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10084,6 +10263,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_5_coeffs)( flo { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10097,6 +10277,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_6_coeffs)( flo { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10111,6 +10292,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_7_coeffs)( flo float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; stbir__3_coeff_setup(); + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10125,6 +10307,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_8_coeffs)( flo { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10138,6 +10321,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_9_coeffs)( flo { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10152,6 +10336,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_10_coeffs)( fl { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10167,6 +10352,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_11_coeffs)( fl float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; stbir__3_coeff_setup(); + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10181,6 +10367,7 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_12_coeffs)( fl { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; float const * hc = horizontal_coefficients; @@ -10195,12 +10382,14 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod0 { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 4 + 3 ) >> 2; float const * hc = horizontal_coefficients; stbir__4_coeff_start(); + STBIR_SIMD_NO_UNROLL_LOOP_START do { hc += 4; decode += STBIR__horizontal_channels * 4; @@ -10215,12 +10404,14 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod1 { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 5 + 3 ) >> 2; float const * hc = horizontal_coefficients; stbir__4_coeff_start(); + STBIR_SIMD_NO_UNROLL_LOOP_START do { hc += 4; decode += STBIR__horizontal_channels * 4; @@ -10236,12 +10427,14 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod2 { float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 6 + 3 ) >> 2; float const * hc = horizontal_coefficients; stbir__4_coeff_start(); + STBIR_SIMD_NO_UNROLL_LOOP_START do { hc += 4; decode += STBIR__horizontal_channels * 4; @@ -10259,12 +10452,14 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod3 float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; stbir__3_coeff_setup(); + STBIR_SIMD_NO_UNROLL_LOOP_START do { float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 7 + 3 ) >> 2; float const * hc = horizontal_coefficients; stbir__4_coeff_start(); + STBIR_SIMD_NO_UNROLL_LOOP_START do { hc += 4; decode += STBIR__horizontal_channels * 4;