diff --git a/benchmarks/BM_ramp.cpp b/benchmarks/BM_ramp.cpp index 2a056392..ef2b5379 100644 --- a/benchmarks/BM_ramp.cpp +++ b/benchmarks/BM_ramp.cpp @@ -30,7 +30,8 @@ static void LinearScalar(benchmark::State& state) { for (auto _ : state) { auto value = dist(gen); - sfz::linearRamp(absl::MakeSpan(output), 0.0f, value); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, false); + sfz::linearRamp(absl::MakeSpan(output), 0.0f, value); } } @@ -42,7 +43,8 @@ static void LinearSIMD(benchmark::State& state) { for (auto _ : state) { auto value = dist(gen); - sfz::linearRamp(absl::MakeSpan(output), 0.0f, value); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, true); + sfz::linearRamp(absl::MakeSpan(output), 0.0f, value); } } static void LinearScalarUnaligned(benchmark::State& state) { @@ -53,7 +55,8 @@ static void LinearScalarUnaligned(benchmark::State& state) { for (auto _ : state) { auto value = dist(gen); - sfz::linearRamp(absl::MakeSpan(output).subspan(1), 0.0f, value); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, false); + sfz::linearRamp(absl::MakeSpan(output).subspan(1), 0.0f, value); } } @@ -65,7 +68,8 @@ static void LinearSIMDUnaligned(benchmark::State& state) { for (auto _ : state) { auto value = dist(gen); - sfz::linearRamp(absl::MakeSpan(output).subspan(1), 0.0f, value); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, true); + sfz::linearRamp(absl::MakeSpan(output).subspan(1), 0.0f, value); } } @@ -77,7 +81,8 @@ static void MulScalar(benchmark::State& state) { for (auto _ : state) { auto value = dist(gen); - sfz::multiplicativeRamp(absl::MakeSpan(output), 1.0f, value); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplicativeRamp, false); + sfz::multiplicativeRamp(absl::MakeSpan(output), 1.0f, value); } } @@ -89,7 +94,8 @@ static void MulSIMD(benchmark::State& state) { for (auto _ : state) { auto value = dist(gen); - sfz::multiplicativeRamp(absl::MakeSpan(output), 1.0f, value); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplicativeRamp, true); + sfz::multiplicativeRamp(absl::MakeSpan(output), 1.0f, value); } } static void MulScalarUnaligned(benchmark::State& state) { @@ -100,7 +106,8 @@ static void MulScalarUnaligned(benchmark::State& state) { for (auto _ : state) { auto value = dist(gen); - sfz::multiplicativeRamp(absl::MakeSpan(output).subspan(1), 1.0f, value); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplicativeRamp, false); + sfz::multiplicativeRamp(absl::MakeSpan(output).subspan(1), 1.0f, value); } } @@ -112,7 +119,8 @@ static void MulSIMDUnaligned(benchmark::State& state) { for (auto _ : state) { auto value = dist(gen); - sfz::multiplicativeRamp(absl::MakeSpan(output).subspan(1), 1.0f, value); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplicativeRamp, true); + sfz::multiplicativeRamp(absl::MakeSpan(output).subspan(1), 1.0f, value); } } diff --git a/src/sfizz/SIMDHelpers.cpp b/src/sfizz/SIMDHelpers.cpp index 14c6275b..bcb56197 100644 --- a/src/sfizz/SIMDHelpers.cpp +++ b/src/sfizz/SIMDHelpers.cpp @@ -269,4 +269,74 @@ void multiplyAdd(float gain, const float* input, float* output, unsigned *output++ += gain * (*input++); } +template <> +float linearRamp(float* output, float start, float step, unsigned size) noexcept +{ + const auto sentinel = output + size; + + if (getSIMDOpStatus(SIMDOps::linearRamp)) { +#if SFIZZ_CPU_FAMILY_X86_64 || SFIZZ_CPU_FAMILY_I386 + if (cpuInfo.has_sse()) { + const auto* lastAligned = prevAligned(sentinel); + while (unaligned(output) && output < lastAligned){ + *output++ = start; + start += step; + } + + auto mmStart = _mm_set1_ps(start - step); + auto mmStep = _mm_set_ps(step + step + step + step, step + step + step, step + step, step); + while (output < lastAligned) { + mmStart = _mm_add_ps(mmStart, mmStep); + _mm_store_ps(output, mmStart); + mmStart = _mm_shuffle_ps(mmStart, mmStart, _MM_SHUFFLE(3, 3, 3, 3)); + incrementAll<4>( output); + } + start = _mm_cvtss_f32(mmStart) + step; + // fallthrough from lastAligned to sentinel + } +#endif + } + + while (output < sentinel) { + *output++ = start; + start += step; + } + return start; +} + +template <> +float multiplicativeRamp(float* output, float start, float step, unsigned size) noexcept +{ + const auto sentinel = output + size; + + if (getSIMDOpStatus(SIMDOps::linearRamp)) { +#if SFIZZ_CPU_FAMILY_X86_64 || SFIZZ_CPU_FAMILY_I386 + if (cpuInfo.has_sse()) { + const auto* lastAligned = prevAligned(sentinel); + while (unaligned(output) && output < lastAligned){ + *output++ = start; + start *= step; + } + + auto mmStart = _mm_set1_ps(start / step); + auto mmStep = _mm_set_ps(step * step * step * step, step * step * step, step * step, step); + while (output < lastAligned) { + mmStart = _mm_mul_ps(mmStart, mmStep); + _mm_store_ps(output, mmStart); + mmStart = _mm_shuffle_ps(mmStart, mmStart, _MM_SHUFFLE(3, 3, 3, 3)); + incrementAll<4>( output); + } + start = _mm_cvtss_f32(mmStart) * step; + // fallthrough from lastAligned to sentinel + } +#endif + } + + while (output < sentinel) { + *output++ = start; + start *= step; + } + return start; +} + } diff --git a/src/sfizz/SIMDHelpers.h b/src/sfizz/SIMDHelpers.h index b190293f..ef9fd2c7 100644 --- a/src/sfizz/SIMDHelpers.h +++ b/src/sfizz/SIMDHelpers.h @@ -339,32 +339,34 @@ void multiplyAdd(T gain, absl::Span input, absl::Span output) noexce multiplyAdd(gain, input.data(), output.data(), minSpanSize(input, output)); } -namespace _internals { - template - inline void snippetRampLinear(T*& output, T& value, T step) - { - *output++ = value; - value += step; - } -} - /** * @brief Compute a linear ramp blockwise between 2 values * * @tparam T the underlying type - * @tparam SIMD use the SIMD version or the scalar version * @param output The destination span * @param start * @param step + * @param size * @return T */ -template +template +T linearRamp(T* output, T start, T step, unsigned size) noexcept +{ + const auto sentinel = output + size; + while (output < sentinel) { + *output++ = start; + start += step; + } + return start; +} + +template <> +float linearRamp(float* output, float start, float step, unsigned size) noexcept; + +template T linearRamp(absl::Span output, T start, T step) noexcept { - auto* out = output.begin(); - while (out < output.end()) - _internals::snippetRampLinear(out, start, step); - return start; + return linearRamp(output.data(), start, step, output.size()); } namespace _internals { @@ -380,26 +382,31 @@ namespace _internals { * @brief Compute a multiplicative ramp blockwise between 2 values * * @tparam T the underlying type - * @tparam SIMD use the SIMD version or the scalar version * @param output The destination span * @param start * @param step * @return T */ -template -T multiplicativeRamp(absl::Span output, T start, T step) noexcept +template +T multiplicativeRamp(T* output, T start, T step, unsigned size) noexcept { - auto* out = output.begin(); - while (out < output.end()) - _internals::snippetRampMultiplicative(out, start, step); + const auto sentinel = output + size; + while (output < sentinel) { + *output++ = start; + start *= step; + } return start; } template <> -float linearRamp(absl::Span output, float start, float step) noexcept; +float multiplicativeRamp(float* output, float start, float step, unsigned size) noexcept; -template <> -float multiplicativeRamp(absl::Span output, float start, float step) noexcept; +template +T multiplicativeRamp(absl::Span output, T start, T step) noexcept +{ + return multiplicativeRamp(output.data(), start, step, output.size()); + +} namespace _internals { template diff --git a/src/sfizz/SIMDSSE.cpp b/src/sfizz/SIMDSSE.cpp index d9be6492..a1258e85 100644 --- a/src/sfizz/SIMDSSE.cpp +++ b/src/sfizz/SIMDSSE.cpp @@ -16,56 +16,7 @@ constexpr uintptr_t TypeAlignment = 4; -template <> -float sfz::linearRamp(absl::Span output, float value, float step) noexcept -{ - auto* out = output.begin(); - const auto* lastAligned = prevAligned(output.end()); - while (unaligned(out) && out < lastAligned) - _internals::snippetRampLinear(out, value, step); - - auto mmValue = _mm_set1_ps(value - step); - auto mmStep = _mm_set_ps(step + step + step + step, step + step + step, step + step, step); - - while (out < lastAligned) { - mmValue = _mm_add_ps(mmValue, mmStep); - _mm_store_ps(out, mmValue); - mmValue = _mm_shuffle_ps(mmValue, mmValue, _MM_SHUFFLE(3, 3, 3, 3)); - out += TypeAlignment; - } - - value = _mm_cvtss_f32(mmValue) + step; - - while (out < output.end()) - _internals::snippetRampLinear(out, value, step); - return value; -} - -template <> -float sfz::multiplicativeRamp(absl::Span output, float value, float step) noexcept -{ - auto* out = output.begin(); - const auto* lastAligned = prevAligned(output.end()); - - while (unaligned(out) && out < lastAligned) - _internals::snippetRampMultiplicative(out, value, step); - - auto mmValue = _mm_set1_ps(value / step); - auto mmStep = _mm_set_ps(step * step * step * step, step * step * step, step * step, step); - - while (out < lastAligned) { - mmValue = _mm_mul_ps(mmValue, mmStep); - _mm_store_ps(out, mmValue); - mmValue = _mm_shuffle_ps(mmValue, mmValue, _MM_SHUFFLE(3, 3, 3, 3)); - out += TypeAlignment; - } - - value = _mm_cvtss_f32(mmValue) * step; - while (out < output.end()) - _internals::snippetRampMultiplicative(out, value, step); - return value; -} template <> void sfz::add(absl::Span input, absl::Span output) noexcept diff --git a/tests/SIMDHelpersT.cpp b/tests/SIMDHelpersT.cpp index bd864e52..6e67ee78 100644 --- a/tests/SIMDHelpersT.cpp +++ b/tests/SIMDHelpersT.cpp @@ -356,7 +356,8 @@ TEST_CASE("[Helpers] Linear Ramp") const float v { fillValue }; std::array output; std::array expected { start, start + v, start + v + v, start + v + v + v, start + v + v + v + v, start + v + v + v + v + v }; - sfz::linearRamp(absl::MakeSpan(output), start, v); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, false); + sfz::linearRamp(absl::MakeSpan(output), start, v); REQUIRE(output == expected); } @@ -366,7 +367,8 @@ TEST_CASE("[Helpers] Linear Ramp (SIMD)") const float v { fillValue }; std::array output; std::array expected { start, start + v, start + v + v, start + v + v + v, start + v + v + v + v, start + v + v + v + v + v }; - sfz::linearRamp(absl::MakeSpan(output), start, v); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, true); + sfz::linearRamp(absl::MakeSpan(output), start, v); REQUIRE(approxEqual(output, expected)); } @@ -375,8 +377,10 @@ TEST_CASE("[Helpers] Linear Ramp (SIMD vs scalar)") const float start { 0.0f }; std::vector outputScalar(bigBufferSize); std::vector outputSIMD(bigBufferSize); - sfz::linearRamp(absl::MakeSpan(outputScalar), start, fillValue); - sfz::linearRamp(absl::MakeSpan(outputSIMD), start, fillValue); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, false); + sfz::linearRamp(absl::MakeSpan(outputScalar), start, fillValue); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, true); + sfz::linearRamp(absl::MakeSpan(outputSIMD), start, fillValue); REQUIRE(approxEqual(outputScalar, outputSIMD)); } @@ -385,8 +389,10 @@ TEST_CASE("[Helpers] Linear Ramp unaligned (SIMD vs scalar)") const float start { 0.0f }; std::vector outputScalar(bigBufferSize); std::vector outputSIMD(bigBufferSize); - sfz::linearRamp(absl::MakeSpan(outputScalar).subspan(1), start, fillValue); - sfz::linearRamp(absl::MakeSpan(outputSIMD).subspan(1), start, fillValue); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, false); + sfz::linearRamp(absl::MakeSpan(outputScalar).subspan(1), start, fillValue); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, true); + sfz::linearRamp(absl::MakeSpan(outputSIMD).subspan(1), start, fillValue); REQUIRE(approxEqual(outputScalar, outputSIMD)); } @@ -396,7 +402,8 @@ TEST_CASE("[Helpers] Multiplicative Ramp") const float v { fillValue }; std::array output; std::array expected { start, start * v, start * v * v, start * v * v * v, start * v * v * v * v, start * v * v * v * v * v }; - sfz::multiplicativeRamp(absl::MakeSpan(output), start, v); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplicativeRamp, false); + sfz::multiplicativeRamp(absl::MakeSpan(output), start, v); REQUIRE(approxEqual(output, expected)); } @@ -406,7 +413,8 @@ TEST_CASE("[Helpers] Multiplicative Ramp (SIMD)") const float v { fillValue }; std::array output; std::array expected { start, start * v, start * v * v, start * v * v * v, start * v * v * v * v, start * v * v * v * v * v }; - sfz::multiplicativeRamp(absl::MakeSpan(output), start, v); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplicativeRamp, true); + sfz::multiplicativeRamp(absl::MakeSpan(output), start, v); REQUIRE(approxEqual(output, expected)); } @@ -415,8 +423,10 @@ TEST_CASE("[Helpers] Multiplicative Ramp (SIMD vs scalar)") const float start { 1.0f }; std::vector outputScalar(bigBufferSize); std::vector outputSIMD(bigBufferSize); - sfz::multiplicativeRamp(absl::MakeSpan(outputScalar), start, fillValue); - sfz::multiplicativeRamp(absl::MakeSpan(outputSIMD), start, fillValue); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplicativeRamp, false); + sfz::multiplicativeRamp(absl::MakeSpan(outputScalar), start, fillValue); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplicativeRamp, true); + sfz::multiplicativeRamp(absl::MakeSpan(outputSIMD), start, fillValue); REQUIRE(approxEqual(outputScalar, outputSIMD)); } @@ -425,8 +435,10 @@ TEST_CASE("[Helpers] Multiplicative Ramp unaligned (SIMD vs scalar)") const float start { 1.0f }; std::vector outputScalar(bigBufferSize); std::vector outputSIMD(bigBufferSize); - sfz::multiplicativeRamp(absl::MakeSpan(outputScalar).subspan(1), start, fillValue); - sfz::multiplicativeRamp(absl::MakeSpan(outputSIMD).subspan(1), start, fillValue); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplicativeRamp, false); + sfz::multiplicativeRamp(absl::MakeSpan(outputScalar).subspan(1), start, fillValue); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplicativeRamp, true); + sfz::multiplicativeRamp(absl::MakeSpan(outputSIMD).subspan(1), start, fillValue); REQUIRE(approxEqual(outputScalar, outputSIMD)); } @@ -666,6 +678,7 @@ TEST_CASE("[Helpers] Cumulative sum (SIMD vs Scalar)") std::vector input(bigBufferSize); std::vector outputScalar(bigBufferSize); std::vector outputSIMD(bigBufferSize); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, true); sfz::linearRamp(absl::MakeSpan(input), 0.0f, 0.1f); sfz::cumsum(input, absl::MakeSpan(outputScalar)); sfz::cumsum(input, absl::MakeSpan(outputSIMD)); @@ -686,6 +699,7 @@ TEST_CASE("[Helpers] Diff (SIMD vs Scalar)") std::vector input(bigBufferSize); std::vector outputScalar(bigBufferSize); std::vector outputSIMD(bigBufferSize); + sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, true); sfz::linearRamp(absl::MakeSpan(input), 0.0f, 0.1f); sfz::diff(input, absl::MakeSpan(outputScalar)); sfz::diff(input, absl::MakeSpan(outputSIMD));