From 48b8011354c0f71bff295d0aca4e6c55eab5f1bf Mon Sep 17 00:00:00 2001 From: Paul Ferrand Date: Tue, 8 Sep 2020 23:00:19 +0200 Subject: [PATCH] Add SIMD helpers targeted for the mod matrix and plug them in On my machine now the previous code path was probably vectorized but this way we have the structure to handle platforms where it's not --- benchmarks/BM_multiplyMul.cpp | 83 ++++++++++++++++++++++++ benchmarks/BM_multiplyMulFixedGain.cpp | 88 ++++++++++++++++++++++++++ benchmarks/CMakeLists.txt | 2 + src/sfizz/SIMDHelpers.cpp | 20 ++++++ src/sfizz/SIMDHelpers.h | 52 +++++++++++++++ src/sfizz/modulations/ModMatrix.cpp | 8 +-- src/sfizz/simd/HelpersSSE.cpp | 43 +++++++++++++ src/sfizz/simd/HelpersSSE.h | 2 + src/sfizz/simd/HelpersScalar.h | 16 +++++ tests/SIMDHelpersT.cpp | 79 +++++++++++++++++++++++ 10 files changed, 388 insertions(+), 5 deletions(-) create mode 100644 benchmarks/BM_multiplyMul.cpp create mode 100644 benchmarks/BM_multiplyMulFixedGain.cpp diff --git a/benchmarks/BM_multiplyMul.cpp b/benchmarks/BM_multiplyMul.cpp new file mode 100644 index 00000000..d002fba0 --- /dev/null +++ b/benchmarks/BM_multiplyMul.cpp @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: BSD-2-Clause + +// This code is part of the sfizz library and is licensed under a BSD 2-clause +// license. You should have receive a LICENSE.md file along with the code. +// If not, contact the sfizz maintainers at https://github.com/sfztools/sfizz + +#include "SIMDHelpers.h" +#include +#include +#include +#include +#include +#include + +class MultiplyMul : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State& state) { + std::random_device rd { }; + std::mt19937 gen { rd() }; + std::uniform_real_distribution dist { 0.1f, 1.0f }; + input = std::vector(state.range(0)); + output = std::vector(state.range(0)); + gain = std::vector(state.range(0)); + std::fill(output.begin(), output.end(), 2.0f ); + std::generate(gain.begin(), gain.end(), [&]() { return dist(gen); }); + std::generate(input.begin(), input.end(), [&]() { return dist(gen); }); + } + + void TearDown(const ::benchmark::State& /* state */) { + + } + + std::vector gain; + std::vector input; + std::vector output; +}; + +BENCHMARK_DEFINE_F(MultiplyMul, Straight)(benchmark::State& state) { + for (auto _ : state) + { + for (int i = 0; i < state.range(0); ++i) + output[i] *= gain[i] * input[i]; + } +} + +BENCHMARK_DEFINE_F(MultiplyMul, Scalar)(benchmark::State& state) { + for (auto _ : state) + { + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul, false); + sfz::multiplyMul(gain, input, absl::MakeSpan(output)); + } +} + +BENCHMARK_DEFINE_F(MultiplyMul, SIMD)(benchmark::State& state) { + for (auto _ : state) + { + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul, true); + sfz::multiplyMul(gain, input, absl::MakeSpan(output)); + } +} + +BENCHMARK_DEFINE_F(MultiplyMul, Scalar_Unaligned)(benchmark::State& state) { + for (auto _ : state) + { + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul, false); + sfz::multiplyMul(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + } +} + +BENCHMARK_DEFINE_F(MultiplyMul, SIMD_Unaligned)(benchmark::State& state) { + for (auto _ : state) + { + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul, true); + sfz::multiplyMul(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + } +} + +BENCHMARK_REGISTER_F(MultiplyMul, Straight)->RangeMultiplier(4)->Range(1 << 2, 1 << 12); +BENCHMARK_REGISTER_F(MultiplyMul, Scalar)->RangeMultiplier(4)->Range(1 << 2, 1 << 12); +BENCHMARK_REGISTER_F(MultiplyMul, SIMD)->RangeMultiplier(4)->Range(1 << 2, 1 << 12); +BENCHMARK_REGISTER_F(MultiplyMul, Scalar_Unaligned)->RangeMultiplier(4)->Range(1 << 2, 1 << 12); +BENCHMARK_REGISTER_F(MultiplyMul, SIMD_Unaligned)->RangeMultiplier(4)->Range(1 << 2, 1 << 12); +BENCHMARK_MAIN(); diff --git a/benchmarks/BM_multiplyMulFixedGain.cpp b/benchmarks/BM_multiplyMulFixedGain.cpp new file mode 100644 index 00000000..56988bef --- /dev/null +++ b/benchmarks/BM_multiplyMulFixedGain.cpp @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BSD-2-Clause + +// This code is part of the sfizz library and is licensed under a BSD 2-clause +// license. You should have receive a LICENSE.md file along with the code. +// If not, contact the sfizz maintainers at https://github.com/sfztools/sfizz + +#include "SIMDHelpers.h" +#include +#include +#include +#include +#include +#include + +class MultiplyMulFixedGain : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State& state) + { + std::random_device rd {}; + std::mt19937 gen { rd() }; + std::uniform_real_distribution dist { 0.1f, 1.0f }; + input = std::vector(state.range(0)); + output = std::vector(state.range(0)); + gain = dist(gen); + std::fill(output.begin(), output.end(), 2.0f); + std::generate(input.begin(), input.end(), [&]() { return dist(gen); }); + } + + void TearDown(const ::benchmark::State& /* state */) + { + } + + float gain = {}; + std::vector input; + std::vector output; +}; + +BENCHMARK_DEFINE_F(MultiplyMulFixedGain, Straight) +(benchmark::State& state) +{ + for (auto _ : state) { + for (int i = 0; i < state.range(0); ++i) + output[i] *= gain * input[i]; + } +} + +BENCHMARK_DEFINE_F(MultiplyMulFixedGain, Scalar) +(benchmark::State& state) +{ + for (auto _ : state) { + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul1, false); + sfz::multiplyMul1(gain, input, absl::MakeSpan(output)); + } +} + +BENCHMARK_DEFINE_F(MultiplyMulFixedGain, SIMD) +(benchmark::State& state) +{ + for (auto _ : state) { + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul1, true); + sfz::multiplyMul1(gain, input, absl::MakeSpan(output)); + } +} + +BENCHMARK_DEFINE_F(MultiplyMulFixedGain, Scalar_Unaligned) +(benchmark::State& state) +{ + for (auto _ : state) { + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul1, false); + sfz::multiplyMul1(gain, absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + } +} + +BENCHMARK_DEFINE_F(MultiplyMulFixedGain, SIMD_Unaligned) +(benchmark::State& state) +{ + for (auto _ : state) { + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul1, true); + sfz::multiplyMul1(gain, absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + } +} + +BENCHMARK_REGISTER_F(MultiplyMulFixedGain, Straight)->RangeMultiplier(4)->Range(1 << 2, 1 << 12); +BENCHMARK_REGISTER_F(MultiplyMulFixedGain, Scalar)->RangeMultiplier(4)->Range(1 << 2, 1 << 12); +BENCHMARK_REGISTER_F(MultiplyMulFixedGain, SIMD)->RangeMultiplier(4)->Range(1 << 2, 1 << 12); +BENCHMARK_REGISTER_F(MultiplyMulFixedGain, Scalar_Unaligned)->RangeMultiplier(4)->Range(1 << 2, 1 << 12); +BENCHMARK_REGISTER_F(MultiplyMulFixedGain, SIMD_Unaligned)->RangeMultiplier(4)->Range(1 << 2, 1 << 12); +BENCHMARK_MAIN(); diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index cba645f3..0ecd4494 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -49,6 +49,8 @@ target_link_libraries(bm_ADSR PRIVATE sfizz::sfizz) sfizz_add_benchmark(bm_add BM_add.cpp) sfizz_add_benchmark(bm_multiplyAdd BM_multiplyAdd.cpp) sfizz_add_benchmark(bm_multiplyAddFixedGain BM_multiplyAddFixedGain.cpp) +sfizz_add_benchmark(bm_multiplyMul BM_multiplyMul.cpp) +sfizz_add_benchmark(bm_multiplyMulFixedGain BM_multiplyMulFixedGain.cpp) sfizz_add_benchmark(bm_subtract BM_subtract.cpp) sfizz_add_benchmark(bm_copy BM_copy.cpp) sfizz_add_benchmark(bm_mean BM_mean.cpp) diff --git a/src/sfizz/SIMDHelpers.cpp b/src/sfizz/SIMDHelpers.cpp index c0287772..f34124a4 100644 --- a/src/sfizz/SIMDHelpers.cpp +++ b/src/sfizz/SIMDHelpers.cpp @@ -29,6 +29,8 @@ struct SIMDDispatch { decltype(÷Scalar) divide = ÷Scalar; decltype(&multiplyAddScalar) multiplyAdd = &multiplyAddScalar; decltype(&multiplyAdd1Scalar) multiplyAdd1 = &multiplyAdd1Scalar; + decltype(&multiplyMulScalar) multiplyMul = &multiplyMulScalar; + decltype(&multiplyMul1Scalar) multiplyMul1 = &multiplyMul1Scalar; decltype(&linearRampScalar) linearRamp = &linearRampScalar; decltype(&multiplicativeRampScalar) multiplicativeRamp = &multiplicativeRampScalar; decltype(&addScalar) add = &addScalar; @@ -81,6 +83,8 @@ void SIMDDispatch::setStatus(SIMDOps op, bool enable) SIMD_OP(subtract1) SIMD_OP(multiplyAdd) SIMD_OP(multiplyAdd1) + SIMD_OP(multiplyMul) + SIMD_OP(multiplyMul1) SIMD_OP(copy) SIMD_OP(cumsum) SIMD_OP(diff) @@ -118,6 +122,8 @@ void SIMDDispatch::setStatus(SIMDOps op, bool enable) SIMD_OP(subtract1) SIMD_OP(multiplyAdd) SIMD_OP(multiplyAdd1) + SIMD_OP(multiplyMul) + SIMD_OP(multiplyMul1) SIMD_OP(copy) SIMD_OP(cumsum) SIMD_OP(diff) @@ -158,6 +164,8 @@ void SIMDDispatch::resetStatus() setStatus(SIMDOps::subtract1, false); setStatus(SIMDOps::multiplyAdd, false); setStatus(SIMDOps::multiplyAdd1, false); + setStatus(SIMDOps::multiplyMul, false); + setStatus(SIMDOps::multiplyMul1, false); setStatus(SIMDOps::copy, false); setStatus(SIMDOps::cumsum, true); setStatus(SIMDOps::diff, false); @@ -243,6 +251,18 @@ void multiplyAdd1(float gain, const float* input, float* output, unsigned return simdDispatch().multiplyAdd1(gain, input, output, size); } +template <> +void multiplyMul(const float* gain, const float* input, float* output, unsigned size) noexcept +{ + return simdDispatch().multiplyMul(gain, input, output, size); +} + +template <> +void multiplyMul1(float gain, const float* input, float* output, unsigned size) noexcept +{ + return simdDispatch().multiplyMul1(gain, input, output, size); +} + template <> float linearRamp(float* output, float start, float step, unsigned size) noexcept { diff --git a/src/sfizz/SIMDHelpers.h b/src/sfizz/SIMDHelpers.h index ae34f732..08371613 100644 --- a/src/sfizz/SIMDHelpers.h +++ b/src/sfizz/SIMDHelpers.h @@ -52,6 +52,8 @@ enum class SIMDOps { subtract1, multiplyAdd, multiplyAdd1, + multiplyMul, + multiplyMul1, copy, cumsum, diff, @@ -322,6 +324,56 @@ void multiplyAdd1(T gain, absl::Span input, absl::Span output) noexc multiplyAdd1(gain, input.data(), output.data(), minSpanSize(input, output)); } +/** + * @brief Applies a gain to the input and multiply the output with it + * + * @tparam T the underlying type + * @param gain + * @param input + * @param output + * @param size + */ +template +void multiplyMul(const T* gain, const T* input, T* output, unsigned size) noexcept +{ + multiplyMulScalar(gain, input, output, size); +} + +template <> +void multiplyMul(const float* gain, const float* input, float* output, unsigned size) noexcept; + +template +void multiplyMul(absl::Span gain, absl::Span input, absl::Span output) noexcept +{ + CHECK_SPAN_SIZES(gain, input, output); + multiplyMul(gain.data(), input.data(), output.data(), minSpanSize(gain, input, output)); +} + +/** + * @brief Applies a fixed gain to the input and multiply the output with it + * + * @tparam T the underlying type + * @param gain + * @param input + * @param output + * @param size + */ +template +void multiplyMul1(T gain, const T* input, T* output, unsigned size) noexcept +{ + multiplyMul1Scalar(gain, input, output, size); +} + +template <> +void multiplyMul1(float gain, const float* input, float* output, unsigned size) noexcept; + +template +void multiplyMul1(T gain, absl::Span input, absl::Span output) noexcept +{ + CHECK_SPAN_SIZES(input, output); + multiplyMul1(gain, input.data(), output.data(), minSpanSize(input, output)); +} + /** * @brief Compute a linear ramp blockwise between 2 values * diff --git a/src/sfizz/modulations/ModMatrix.cpp b/src/sfizz/modulations/ModMatrix.cpp index 643f168d..a8c1f8fe 100644 --- a/src/sfizz/modulations/ModMatrix.cpp +++ b/src/sfizz/modulations/ModMatrix.cpp @@ -355,16 +355,14 @@ float* ModMatrix::getModulation(TargetId targetId) } else { if (targetFlags & kModIsMultiplicative) { - for (uint32_t i = 0; i < numFrames; ++i) - buffer[i] *= sourceDepth * sourceBuffer[i]; + multiplyMul1(sourceDepth, sourceBuffer, buffer); } else if (targetFlags & kModIsPercentMultiplicative) { - for (uint32_t i = 0; i < numFrames; ++i) - buffer[i] *= (0.01f * sourceDepth) * sourceBuffer[i]; + multiplyMul1(0.01f * sourceDepth, sourceBuffer, buffer); } else { ASSERT(targetFlags & kModIsAdditive); - sfz::multiplyAdd1(sourceDepth, sourceBuffer, buffer); + multiplyAdd1(sourceDepth, sourceBuffer, buffer); } } } diff --git a/src/sfizz/simd/HelpersSSE.cpp b/src/sfizz/simd/HelpersSSE.cpp index d17288e7..4f8daf25 100644 --- a/src/sfizz/simd/HelpersSSE.cpp +++ b/src/sfizz/simd/HelpersSSE.cpp @@ -183,6 +183,49 @@ void multiplyAdd1SSE(float gain, const float* input, float* output, unsigned siz *output++ += gain * (*input++); } +void multiplyMulSSE(const float* gain, const float* input, float* output, unsigned size) noexcept +{ + const auto sentinel = output + size; + +#if SFIZZ_HAVE_SSE2 + const auto* lastAligned = prevAligned(sentinel); + while (unaligned(input, output) && output < lastAligned) + *output++ *= (*gain++) * (*input++); + + while (output < lastAligned) { + auto mmOut = _mm_load_ps(output); + mmOut = _mm_mul_ps(_mm_mul_ps(_mm_load_ps(gain), _mm_load_ps(input)), mmOut); + _mm_store_ps(output, mmOut); + incrementAll(gain, input, output); + } +#endif + + while (output < sentinel) + *output++ *= (*gain++) * (*input++); +} + +void multiplyMul1SSE(float gain, const float* input, float* output, unsigned size) noexcept +{ + const auto sentinel = output + size; + +#if SFIZZ_HAVE_SSE2 + const auto* lastAligned = prevAligned(sentinel); + while (unaligned(input, output) && output < lastAligned) + *output++ *= gain * (*input++); + + auto mmGain = _mm_set1_ps(gain); + while (output < lastAligned) { + auto mmOut = _mm_load_ps(output); + mmOut = _mm_mul_ps(_mm_mul_ps(mmGain, _mm_load_ps(input)), mmOut); + _mm_store_ps(output, mmOut); + incrementAll(input, output); + } +#endif + + while (output < sentinel) + *output++ *= gain * (*input++); +} + float linearRampSSE(float* output, float start, float step, unsigned size) noexcept { const auto sentinel = output + size; diff --git a/src/sfizz/simd/HelpersSSE.h b/src/sfizz/simd/HelpersSSE.h index 5046d914..fded8048 100644 --- a/src/sfizz/simd/HelpersSSE.h +++ b/src/sfizz/simd/HelpersSSE.h @@ -14,6 +14,8 @@ void gain1SSE(float gain, const float* input, float* output, unsigned size) noex void divideSSE(const float* input, const float* divisor, float* output, unsigned size) noexcept; void multiplyAddSSE(const float* gain, const float* input, float* output, unsigned size) noexcept; void multiplyAdd1SSE(float gain, const float* input, float* output, unsigned size) noexcept; +void multiplyMulSSE(const float* gain, const float* input, float* output, unsigned size) noexcept; +void multiplyMul1SSE(float gain, const float* input, float* output, unsigned size) noexcept; float linearRampSSE(float* output, float start, float step, unsigned size) noexcept; float multiplicativeRampSSE(float* output, float start, float step, unsigned size) noexcept; void addSSE(const float* input, float* output, unsigned size) noexcept; diff --git a/src/sfizz/simd/HelpersScalar.h b/src/sfizz/simd/HelpersScalar.h index 871a9ff6..33c4af3f 100644 --- a/src/sfizz/simd/HelpersScalar.h +++ b/src/sfizz/simd/HelpersScalar.h @@ -67,6 +67,22 @@ inline void multiplyAdd1Scalar(T gain, const T* input, T* output, unsigned size) *output++ += gain * (*input++); } +template +inline void multiplyMulScalar(const T* gain, const T* input, T* output, unsigned size) noexcept +{ + const auto sentinel = output + size; + while (output < sentinel) + *output++ *= (*gain++) * (*input++); +} + +template +inline void multiplyMul1Scalar(T gain, const T* input, T* output, unsigned size) noexcept +{ + const auto sentinel = output + size; + while (output < sentinel) + *output++ *= gain * (*input++); +} + template T linearRampScalar(T* output, T start, T step, unsigned size) noexcept { diff --git a/tests/SIMDHelpersT.cpp b/tests/SIMDHelpersT.cpp index f0ef8277..27502881 100644 --- a/tests/SIMDHelpersT.cpp +++ b/tests/SIMDHelpersT.cpp @@ -517,6 +517,7 @@ TEST_CASE("[Helpers] MultiplyAdd (SIMD)") REQUIRE(output == expected); } + TEST_CASE("[Helpers] MultiplyAdd (SIMD vs scalar)") { std::vector gain(bigBufferSize); @@ -574,6 +575,84 @@ TEST_CASE("[Helpers] MultiplyAdd fixed gain (SIMD vs scalar)") REQUIRE(approxEqual(outputScalar, outputSIMD)); } +TEST_CASE("[Helpers] MultiplyMul (Scalar)") +{ + std::array gain { 0.0f, 0.1f, 0.2f, 0.3f, 0.4f }; + std::array input { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; + std::array output { 5.0f, 4.0f, 3.0f, 2.0f, 1.0f }; + std::array expected { 0.0f, 0.8f, 1.8f, 2.4f, 2.0f }; + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul, true); + sfz::multiplyMul(gain, input, absl::MakeSpan(output)); + REQUIRE(approxEqual(output, expected)); +} + +TEST_CASE("[Helpers] MultiplyMul (SIMD)") +{ + std::array gain { 0.0f, 0.1f, 0.2f, 0.3f, 0.4f }; + std::array input { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; + std::array output { 5.0f, 4.0f, 3.0f, 2.0f, 1.0f }; + std::array expected { 0.0f, 0.8f, 1.8f, 2.4f, 2.0f }; + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul, true); + sfz::multiplyMul(gain, input, absl::MakeSpan(output)); + REQUIRE(approxEqual(output, expected)); +} + +TEST_CASE("[Helpers] MultiplyMul (SIMD vs Scalar)") +{ + std::vector gain(bigBufferSize); + std::vector input(bigBufferSize); + std::vector outputScalar(bigBufferSize); + std::vector outputSIMD(bigBufferSize); + absl::c_iota(gain, 0.0f); + absl::c_iota(input, 0.0f); + absl::c_iota(outputScalar, 0.0f); + absl::c_iota(outputSIMD, 0.0f); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul, false); + sfz::multiplyMul(gain, input, absl::MakeSpan(outputScalar)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul, true); + sfz::multiplyMul(gain, input, absl::MakeSpan(outputSIMD)); + REQUIRE(approxEqual(outputScalar, outputSIMD)); +} + +TEST_CASE("[Helpers] MultiplyMul fixed gain (Scalar)") +{ + float gain = 0.3f; + std::array input { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; + std::array output { 5.0f, 4.0f, 3.0f, 2.0f, 1.0f }; + std::array expected { 1.5f, 2.4f, 2.7f, 2.4f, 1.5f }; + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul1, false); + sfz::multiplyMul1(gain, input, absl::MakeSpan(output)); + REQUIRE(output == expected); +} + +TEST_CASE("[Helpers] MultiplyMul fixed gain (SIMD)") +{ + float gain = 0.3f; + std::array input { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; + std::array output { 5.0f, 4.0f, 3.0f, 2.0f, 1.0f }; + std::array expected { 1.5f, 2.4f, 2.7f, 2.4f, 1.5f }; + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul1, true); + sfz::multiplyMul1(gain, input, absl::MakeSpan(output)); + REQUIRE(output == expected); +} + +TEST_CASE("[Helpers] MultiplyMul fixed gain (SIMD vs scalar)") +{ + float gain = 0.3f; + std::vector input(bigBufferSize); + std::vector outputScalar(bigBufferSize); + std::vector outputSIMD(bigBufferSize); + absl::c_iota(input, 0.0f); + absl::c_iota(outputScalar, 0.0f); + absl::c_iota(outputSIMD, 0.0f); + + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul1, false); + sfz::multiplyMul1(gain, input, absl::MakeSpan(outputScalar)); + sfz::setSIMDOpStatus(sfz::SIMDOps::multiplyMul1, true); + sfz::multiplyMul1(gain, input, absl::MakeSpan(outputSIMD)); + REQUIRE(approxEqual(outputScalar, outputSIMD)); +} + TEST_CASE("[Helpers] Subtract") { std::array input { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };