diff --git a/benchmarks/BM_multiplyAdd.cpp b/benchmarks/BM_multiplyAdd.cpp index 4bb01e3b..a85e28ab 100644 --- a/benchmarks/BM_multiplyAdd.cpp +++ b/benchmarks/BM_multiplyAdd.cpp @@ -46,28 +46,32 @@ BENCHMARK_DEFINE_F(MultiplyAdd, Straight)(benchmark::State& state) { BENCHMARK_DEFINE_F(MultiplyAdd, Scalar)(benchmark::State& state) { for (auto _ : state) { - sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, false); + sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); } } BENCHMARK_DEFINE_F(MultiplyAdd, SIMD)(benchmark::State& state) { for (auto _ : state) { - sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, true); + sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); } } BENCHMARK_DEFINE_F(MultiplyAdd, Scalar_Unaligned)(benchmark::State& state) { for (auto _ : state) { - sfz::multiplyAdd(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, false); + sfz::multiplyAdd(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); } } BENCHMARK_DEFINE_F(MultiplyAdd, SIMD_Unaligned)(benchmark::State& state) { for (auto _ : state) { - sfz::multiplyAdd(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, true); + sfz::multiplyAdd(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); } } diff --git a/benchmarks/BM_multiplyAddFixedGain.cpp b/benchmarks/BM_multiplyAddFixedGain.cpp index e05635ba..97a4001a 100644 --- a/benchmarks/BM_multiplyAddFixedGain.cpp +++ b/benchmarks/BM_multiplyAddFixedGain.cpp @@ -48,7 +48,8 @@ BENCHMARK_DEFINE_F(MultiplyAddFixedGain, Scalar) (benchmark::State& state) { for (auto _ : state) { - sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, false); + sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); } } @@ -56,7 +57,8 @@ BENCHMARK_DEFINE_F(MultiplyAddFixedGain, SIMD) (benchmark::State& state) { for (auto _ : state) { - sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, true); + sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); } } @@ -64,7 +66,8 @@ BENCHMARK_DEFINE_F(MultiplyAddFixedGain, Scalar_Unaligned) (benchmark::State& state) { for (auto _ : state) { - sfz::multiplyAdd(gain, absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, false); + sfz::multiplyAdd(gain, absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); } } @@ -72,7 +75,8 @@ BENCHMARK_DEFINE_F(MultiplyAddFixedGain, SIMD_Unaligned) (benchmark::State& state) { for (auto _ : state) { - sfz::multiplyAdd(gain, absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, true); + sfz::multiplyAdd(gain, absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); } } diff --git a/src/sfizz/SIMDHelpers.cpp b/src/sfizz/SIMDHelpers.cpp index 3aba1f84..5be73cac 100644 --- a/src/sfizz/SIMDHelpers.cpp +++ b/src/sfizz/SIMDHelpers.cpp @@ -17,10 +17,10 @@ static cpuid::cpuinfo cpuInfo; void resetSIMDStatus() { - simdStatus[static_cast(SIMDOps::writeInterleaved)] = true; - simdStatus[static_cast(SIMDOps::readInterleaved)] = true; + simdStatus[static_cast(SIMDOps::writeInterleaved)] = false; + simdStatus[static_cast(SIMDOps::readInterleaved)] = false; simdStatus[static_cast(SIMDOps::fill)] = true; - simdStatus[static_cast(SIMDOps::gain)] = false; + simdStatus[static_cast(SIMDOps::gain)] = true; simdStatus[static_cast(SIMDOps::divide)] = false; simdStatus[static_cast(SIMDOps::mathfuns)] = false; simdStatus[static_cast(SIMDOps::loopingSFZIndex)] = true; @@ -136,7 +136,7 @@ void writeInterleaved(const float* inputLeft, const float* inputRight, float* ou tickWrite(output, inputLeft, inputRight); } -template<> +template <> void applyGain(float gain, const float* input, float* output, unsigned size) noexcept { const auto sentinel = output + size; @@ -162,7 +162,7 @@ void applyGain(float gain, const float* input, float* output, unsigned si *output++ = gain * (*input++); } -template<> +template <> void applyGain(const float* gain, const float* input, float* output, unsigned size) noexcept { const auto sentinel = output + size; @@ -214,4 +214,59 @@ void sfz::divide(const float* input, const float* divisor, float* output, *output++ = (*input++) / (*divisor++); } +template <> +void sfz::multiplyAdd(const float* gain, const float* input, float* output, unsigned size) noexcept +{ + const auto sentinel = output + size; + + if (getSIMDOpStatus(SIMDOps::multiplyAdd)) { +#if SFIZZ_CPU_FAMILY_X86_64 || SFIZZ_CPU_FAMILY_I386 + if (cpuInfo.has_sse()) { + const auto* lastAligned = prevAligned(sentinel); + while (unaligned(input, output) && output < lastAligned) + *output++ += (*gain++) * (*input++); + + while (output < lastAligned) { + auto mmOut = _mm_load_ps(output); + mmOut = _mm_add_ps(_mm_mul_ps(_mm_load_ps(gain), _mm_load_ps(input)), mmOut); + _mm_store_ps(output, mmOut); + incrementAll<4>(gain, input, output); + } + // fallthrough from lastAligned to sentinel + } +#endif + } + + while (output < sentinel) + *output++ += (*gain++) * (*input++); +} + +template <> +void sfz::multiplyAdd(float gain, const float* input, float* output, unsigned size) noexcept +{ + const auto sentinel = output + size; + + if (getSIMDOpStatus(SIMDOps::multiplyAdd)) { +#if SFIZZ_CPU_FAMILY_X86_64 || SFIZZ_CPU_FAMILY_I386 + if (cpuInfo.has_sse()) { + const auto* lastAligned = prevAligned(sentinel); + while (unaligned(input, output) && output < lastAligned) + *output++ += gain * (*input++); + + auto mmGain = _mm_set1_ps(gain); + while (output < lastAligned) { + auto mmOut = _mm_load_ps(output); + mmOut = _mm_add_ps(_mm_mul_ps(mmGain, _mm_load_ps(input)), mmOut); + _mm_store_ps(output, mmOut); + incrementAll<4>(input, output); + } + // fallthrough from lastAligned to sentinel + } +#endif + } + + while (output < sentinel) + *output++ += gain * (*input++); +} + } diff --git a/src/sfizz/SIMDHelpers.h b/src/sfizz/SIMDHelpers.h index 8a722109..b190293f 100644 --- a/src/sfizz/SIMDHelpers.h +++ b/src/sfizz/SIMDHelpers.h @@ -285,61 +285,60 @@ void divide(absl::Span output, absl::Span divisor) noexcept divide(output.data(), divisor.data(), output.data(), minSpanSize(divisor, output)); } -namespace _internals { - template - inline void snippetMultiplyAdd(const T*& gain, const T*& input, T*& output) - { +/** + * @brief Applies a gain to the input and add it on the output + * + * @tparam T the underlying type + * @param gain + * @param input + * @param output + * @param size + */ +template +void multiplyAdd(const T* gain, const T* input, T* output, unsigned size) noexcept +{ + const auto sentinel = output + size; + while (output < sentinel) *output++ += (*gain++) * (*input++); - } +} - template - inline void snippetMultiplyAdd(const T gain, const T*& input, T*& output) - { - *output++ += gain * (*input++); - } +template <> +void multiplyAdd(const float* gain, const float* input, float* output, unsigned size) noexcept; + +template +void multiplyAdd(absl::Span gain, absl::Span input, absl::Span output) noexcept +{ + CHECK_SPAN_SIZES(gain, input, output); + multiplyAdd(gain.data(), input.data(), output.data(), minSpanSize(gain, input, output)); } /** * @brief Applies a gain to the input and add it on the output * - * The output size will be the minimum of the gain span, input span and output span sizes. - * * @tparam T the underlying type - * @tparam SIMD use the SIMD version or the scalar version * @param gain * @param input * @param output + * @param size */ -template -void multiplyAdd(absl::Span gain, absl::Span input, absl::Span output) noexcept +template +void multiplyAdd(T gain, const T* input, T* output, unsigned size) noexcept { - CHECK(gain.size() == input.size()); - CHECK(input.size() <= output.size()); - auto* in = input.begin(); - auto* g = gain.begin(); - auto* out = output.begin(); - auto* sentinel = out + std::min(gain.size(), std::min(output.size(), input.size())); - while (out < sentinel) - _internals::snippetMultiplyAdd(g, in, out); + const auto sentinel = output + size; + while (output < sentinel) + *output++ += gain * (*input++); } template <> -void multiplyAdd(absl::Span gain, absl::Span input, absl::Span output) noexcept; +void multiplyAdd(float gain, const float* input, float* output, unsigned size) noexcept; -template -void multiplyAdd(const T gain, absl::Span input, absl::Span output) noexcept +template +void multiplyAdd(T gain, absl::Span input, absl::Span output) noexcept { - // CHECK(input.size() <= output.size()); - auto* in = input.begin(); - auto* out = output.begin(); - auto* sentinel = out + std::min(output.size(), input.size()); - while (out < sentinel) - _internals::snippetMultiplyAdd(gain, in, out); + CHECK_SPAN_SIZES(input, output); + multiplyAdd(gain, input.data(), output.data(), minSpanSize(input, output)); } -template <> -void multiplyAdd(const float gain, absl::Span input, absl::Span output) noexcept; - namespace _internals { template inline void snippetRampLinear(T*& output, T& value, T step) diff --git a/src/sfizz/SIMDSSE.cpp b/src/sfizz/SIMDSSE.cpp index 0f4a807c..d9be6492 100644 --- a/src/sfizz/SIMDSSE.cpp +++ b/src/sfizz/SIMDSSE.cpp @@ -16,52 +16,6 @@ constexpr uintptr_t TypeAlignment = 4; -template <> -void sfz::multiplyAdd(absl::Span gain, absl::Span input, absl::Span output) noexcept -{ - auto* in = input.begin(); - auto* out = output.begin(); - auto* g = gain.begin(); - const auto size = std::min(output.size(), std::min(input.size(), gain.size())); - const auto* lastAligned = prevAligned(output.begin() + size); - - while (unaligned(out, in, g) && out < lastAligned) - _internals::snippetMultiplyAdd(g, in, out); - - while (out < lastAligned) { - auto mmOut = _mm_load_ps(out); - mmOut = _mm_add_ps(_mm_mul_ps(_mm_load_ps(g), _mm_load_ps(in)), mmOut); - _mm_store_ps(out, mmOut); - incrementAll(g, in, out); - } - - while (out < output.end()) - _internals::snippetMultiplyAdd(g, in, out); -} - -template <> -void sfz::multiplyAdd(const float gain, absl::Span input, absl::Span output) noexcept -{ - auto* in = input.begin(); - auto* out = output.begin(); - const auto size = std::min(output.size(), input.size()); - const auto* lastAligned = prevAligned(output.begin() + size); - - while (unaligned(out, in) && out < lastAligned) - _internals::snippetMultiplyAdd(gain, in, out); - - auto mmGain = _mm_set1_ps(gain); - while (out < lastAligned) { - auto mmOut = _mm_load_ps(out); - mmOut = _mm_add_ps(_mm_mul_ps(mmGain, _mm_load_ps(in)), mmOut); - _mm_store_ps(out, mmOut); - incrementAll(in, out); - } - - while (out < output.end()) - _internals::snippetMultiplyAdd(gain, in, out); -} - template <> float sfz::linearRamp(absl::Span output, float value, float step) noexcept { diff --git a/tests/SIMDHelpersT.cpp b/tests/SIMDHelpersT.cpp index 5b547dcf..bd864e52 100644 --- a/tests/SIMDHelpersT.cpp +++ b/tests/SIMDHelpersT.cpp @@ -462,13 +462,25 @@ TEST_CASE("[Helpers] Add (SIMD vs scalar)") REQUIRE(approxEqual(outputScalar, outputSIMD)); } +TEST_CASE("[Helpers] MultiplyAdd (Scalar)") +{ + std::array gain { 0.0f, 0.1f, 0.2f, 0.3f, 0.4f }; + std::array input { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; + std::array output { 5.0f, 4.0f, 3.0f, 2.0f, 1.0f }; + std::array expected { 5.0f, 4.2f, 3.6f, 3.2f, 3.0f }; + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, false); + sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); + REQUIRE(output == expected); +} + TEST_CASE("[Helpers] MultiplyAdd (SIMD)") { std::array gain { 0.0f, 0.1f, 0.2f, 0.3f, 0.4f }; std::array input { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; std::array output { 5.0f, 4.0f, 3.0f, 2.0f, 1.0f }; std::array expected { 5.0f, 4.2f, 3.6f, 3.2f, 3.0f }; - sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, true); + sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); REQUIRE(output == expected); } @@ -483,18 +495,32 @@ TEST_CASE("[Helpers] MultiplyAdd (SIMD vs scalar)") absl::c_iota(outputScalar, 0.0f); absl::c_iota(outputSIMD, 0.0f); - sfz::multiplyAdd(gain, input, absl::MakeSpan(outputScalar)); - sfz::multiplyAdd(gain, input, absl::MakeSpan(outputSIMD)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, false); + sfz::multiplyAdd(gain, input, absl::MakeSpan(outputScalar)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, true); + sfz::multiplyAdd(gain, input, absl::MakeSpan(outputSIMD)); REQUIRE(approxEqual(outputScalar, outputSIMD)); } +TEST_CASE("[Helpers] MultiplyAdd fixed gain (Scalar)") +{ + float gain = 0.3f; + std::array input { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; + std::array output { 5.0f, 4.0f, 3.0f, 2.0f, 1.0f }; + std::array expected { 5.3f, 4.6f, 3.9f, 3.2f, 2.5f }; + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, false); + sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); + REQUIRE(output == expected); +} + TEST_CASE("[Helpers] MultiplyAdd fixed gain (SIMD)") { float gain = 0.3f; std::array input { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; std::array output { 5.0f, 4.0f, 3.0f, 2.0f, 1.0f }; std::array expected { 5.3f, 4.6f, 3.9f, 3.2f, 2.5f }; - sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, true); + sfz::multiplyAdd(gain, input, absl::MakeSpan(output)); REQUIRE(output == expected); } @@ -508,8 +534,10 @@ TEST_CASE("[Helpers] MultiplyAdd fixed gain (SIMD vs scalar)") absl::c_iota(outputScalar, 0.0f); absl::c_iota(outputSIMD, 0.0f); - sfz::multiplyAdd(gain, input, absl::MakeSpan(outputScalar)); - sfz::multiplyAdd(gain, input, absl::MakeSpan(outputSIMD)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, false); + sfz::multiplyAdd(gain, input, absl::MakeSpan(outputScalar)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyAdd, true); + sfz::multiplyAdd(gain, input, absl::MakeSpan(outputSIMD)); REQUIRE(approxEqual(outputScalar, outputSIMD)); }