From 1ab1ab802b5f3bdce68d02eec509080f9ae630ed Mon Sep 17 00:00:00 2001 From: Paul Ferrand Date: Sat, 30 May 2020 22:17:33 +0200 Subject: [PATCH] Moved applyGain to the new mode --- benchmarks/BM_gain.cpp | 12 ++-- src/sfizz/SIMDHelpers.cpp | 59 +++++++++++++++-- src/sfizz/SIMDHelpers.h | 134 ++++++++++++++++++-------------------- src/sfizz/SIMDSSE.cpp | 41 ------------ tests/SIMDHelpersT.cpp | 24 ++++--- 5 files changed, 140 insertions(+), 130 deletions(-) diff --git a/benchmarks/BM_gain.cpp b/benchmarks/BM_gain.cpp index cb355f1f..3a753389 100644 --- a/benchmarks/BM_gain.cpp +++ b/benchmarks/BM_gain.cpp @@ -66,14 +66,14 @@ BENCHMARK_DEFINE_F(GainSingle, Straight)(benchmark::State& state) { BENCHMARK_DEFINE_F(GainSingle, Scalar)(benchmark::State& state) { for (auto _ : state) { - sfz::applyGain(gain, input, absl::MakeSpan(output)); + sfz::applyGain(gain, input, absl::MakeSpan(output)); } } BENCHMARK_DEFINE_F(GainSingle, SIMD)(benchmark::State& state) { for (auto _ : state) { - sfz::applyGain(gain, input, absl::MakeSpan(output)); + sfz::applyGain(gain, input, absl::MakeSpan(output)); } } @@ -88,28 +88,28 @@ BENCHMARK_DEFINE_F(GainArray, Straight)(benchmark::State& state) { BENCHMARK_DEFINE_F(GainArray, Scalar)(benchmark::State& state) { for (auto _ : state) { - sfz::applyGain(gain, input, absl::MakeSpan(output)); + sfz::applyGain(gain, input, absl::MakeSpan(output)); } } BENCHMARK_DEFINE_F(GainArray, SIMD)(benchmark::State& state) { for (auto _ : state) { - sfz::applyGain(gain, input, absl::MakeSpan(output)); + sfz::applyGain(gain, input, absl::MakeSpan(output)); } } BENCHMARK_DEFINE_F(GainArray, Scalar_Unaligned)(benchmark::State& state) { for (auto _ : state) { - sfz::applyGain(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + sfz::applyGain(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); } } BENCHMARK_DEFINE_F(GainArray, SIMD_Unaligned)(benchmark::State& state) { for (auto _ : state) { - sfz::applyGain(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + sfz::applyGain(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); } } diff --git a/src/sfizz/SIMDHelpers.cpp b/src/sfizz/SIMDHelpers.cpp index 5308f601..a186280a 100644 --- a/src/sfizz/SIMDHelpers.cpp +++ b/src/sfizz/SIMDHelpers.cpp @@ -13,6 +13,7 @@ namespace sfz { static std::array(SIMDOps::_sentinel)> simdStatus; static bool simdStatusInitialized = false; +static cpuid::cpuinfo cpuInfo; void resetSIMDStatus() { @@ -58,7 +59,7 @@ bool getSIMDOpStatus(SIMDOps op) constexpr uintptr_t TypeAlignment = 4; -template +template inline void tickRead(const T*& input, T*& outputLeft, T*& outputRight) { *outputLeft++ = *input++; @@ -75,7 +76,7 @@ inline void tickWrite(T*& output, const T*& inputLeft, const T*& inputRight) void readInterleaved(const float* input, float* outputLeft, float* outputRight, unsigned inputSize) noexcept { const auto sentinel = input + inputSize - 1; - cpuid::cpuinfo cpuInfo; + if (getSIMDOpStatus(SIMDOps::readInterleaved)) { #if SFIZZ_CPU_FAMILY_X86_64 || SFIZZ_CPU_FAMILY_I386 if (cpuInfo.has_sse()) { @@ -109,8 +110,7 @@ void writeInterleaved(const float* inputLeft, const float* inputRight, float* ou { const auto sentinel = output + outputSize - 1; - cpuid::cpuinfo cpuInfo; - if (getSIMDOpStatus(SIMDOps::readInterleaved)) { + if (getSIMDOpStatus(SIMDOps::writeInterleaved)) { #if SFIZZ_CPU_FAMILY_X86_64 || SFIZZ_CPU_FAMILY_I386 if (cpuInfo.has_sse()) { const auto* lastAligned = prevAligned(output + outputSize - 4); @@ -136,5 +136,56 @@ void writeInterleaved(const float* inputLeft, const float* inputRight, float* ou tickWrite(output, inputLeft, inputRight); } +template<> +void applyGain(float gain, const float* input, float* output, unsigned size) noexcept +{ + const auto sentinel = output + size; + + if (getSIMDOpStatus(SIMDOps::gain)) { +#if SFIZZ_CPU_FAMILY_X86_64 || SFIZZ_CPU_FAMILY_I386 + if (cpuInfo.has_sse()) { + const auto* lastAligned = prevAligned(sentinel); + const auto mmGain = _mm_set_ps1(gain); + while (unaligned(input, output) && output < lastAligned) + *output++ = gain * (*input++); + + while (output < lastAligned) { + _mm_store_ps(output, _mm_mul_ps(mmGain, _mm_load_ps(input))); + incrementAll<4>(input, output); + } + // fallthrough from lastAligned to sentinel + } +#endif + } + + while (output < sentinel) + *output++ = gain * (*input++); } +template<> +void applyGain(const float* gain, const float* input, float* output, unsigned size) noexcept +{ + const auto sentinel = output + size; + + if (getSIMDOpStatus(SIMDOps::gain)) { +#if SFIZZ_CPU_FAMILY_X86_64 || SFIZZ_CPU_FAMILY_I386 + if (cpuInfo.has_sse()) { + const auto* lastAligned = prevAligned(sentinel); + + while (unaligned(input, output) && output < lastAligned) + *output++ = (*gain++) * (*input++); + + while (output < lastAligned) { + _mm_store_ps(output, _mm_mul_ps(_mm_load_ps(gain), _mm_load_ps(input))); + incrementAll<4>(gain, input, output); + } + // fallthrough from lastAligned to sentinel + } +#endif + } + + while (output < sentinel) + *output++ = (*gain++) * (*input++); +} + +} diff --git a/src/sfizz/SIMDHelpers.h b/src/sfizz/SIMDHelpers.h index 4be2cee5..b93f3123 100644 --- a/src/sfizz/SIMDHelpers.h +++ b/src/sfizz/SIMDHelpers.h @@ -112,14 +112,13 @@ void readInterleaved(const float* input, float* outputLeft, float* outputRight, inline void readInterleaved(absl::Span input, absl::Span outputLeft, absl::Span outputRight) noexcept { - // The size of the output is not big enough for the input... - CHECK(outputLeft.size() >= input.size() / 2); - CHECK(outputRight.size() >= input.size() / 2); + // Something is fishy with the sizes + CHECK(outputLeft.size() == input.size() / 2); + CHECK(outputRight.size() == input.size() / 2); const auto size = min(input.size(), 2 * outputLeft.size(), 2 * outputRight.size()); readInterleaved(input.data(), outputLeft.data(), outputRight.data(), size); } - /** * @brief Write a pair of left and right stereo input into a single buffer interleaved. * @@ -132,9 +131,9 @@ void writeInterleaved(const float* inputLeft, const float* inputRight, float* ou inline void writeInterleaved(absl::Span inputLeft, absl::Span inputRight, absl::Span output) noexcept { - // Not enough data in the inputs - CHECK(inputLeft.size() >= output.size() / 2); - CHECK(inputRight.size() >= output.size() / 2); + // Something is fishy with the sizes + CHECK(inputLeft.size() == output.size() / 2); + CHECK(inputRight.size() == output.size() / 2); const auto size = min(output.size(), 2 * inputLeft.size(), 2 * inputRight.size()); writeInterleaved(inputLeft.data(), inputRight.data(), output.data(), size); } @@ -152,103 +151,96 @@ void fill(absl::Span output, T value) noexcept absl::c_fill(output, value); } -namespace _internals { - template - inline void snippetGain(T gain, const T*& input, T*& output) - { - *output++ = gain * (*input++); - } -} - /** * @brief Applies a scalar gain to the input * - * The output size will be the minimum of the input span and output span size. - * - * @tparam T the underlying type - * @tparam SIMD use the SIMD version or the scalar version * @param gain the gain to apply * @param input * @param output + * @param size */ -template -void applyGain(T gain, absl::Span input, absl::Span output) noexcept +template +void applyGain(T gain, const T* input, T* output, unsigned size) noexcept { - CHECK(input.size() <= output.size()); - auto* in = input.begin(); - auto* out = output.begin(); - auto* sentinel = out + std::min(output.size(), input.size()); - while (out < sentinel) - _internals::snippetGain(gain, in, out); + const auto sentinel = output + size; + while (output < sentinel) + *output++ = gain * (*input++); } -namespace _internals { - template - inline void snippetGainSpan(const T*& gain, const T*& input, T*& output) - { - *output++ = (*gain++) * (*input++); - } +template<> +void applyGain(float gain, const float* input, float* output, unsigned size) noexcept; + +template +inline void applyGain(T gain, absl::Span input, absl::Span output) noexcept +{ + CHECK_SPAN_SIZES(input, output); + applyGain(gain, input.data(), output.data(), minSpanSize(input, output)); } /** - * @brief Applies a vector gain to an input stap + * @brief Applies a scalar gain inplace * - * The output size will be the minimum of the gain, input span and output span size. + * @param gain the gain to apply + * @param array + * @param size + */ +template +inline void applyGain(float gain, float* array, unsigned size) noexcept +{ + applyGain(gain, array, array, size); +} + +template +inline void applyGain(float gain, absl::Span array) noexcept +{ + applyGain(gain, array.data(), array.data(), array.size()); +} + +/** + * @brief Applies a vector gain to an input span * - * @tparam T the underlying type - * @tparam SIMD use the SIMD version or the scalar version * @param gain * @param input * @param output + * @param size */ -template -void applyGain(absl::Span gain, absl::Span input, absl::Span output) noexcept +template +void applyGain(const T* gain, const T* input, T* output, unsigned size) noexcept { - CHECK(gain.size() == input.size()); - CHECK(input.size() <= output.size()); - auto* in = input.begin(); - auto* g = gain.begin(); - auto* out = output.begin(); - auto* sentinel = out + std::min(gain.size(), std::min(output.size(), input.size())); - while (out < sentinel) - _internals::snippetGainSpan(g, in, out); + const auto sentinel = output + size; + while (output < sentinel) + *output++ = (*gain++) * (*input++); } -/** - * @brief Applies a scalar gain in-place on a span - * - * @tparam T the underlying type - * @tparam SIMD use the SIMD version or the scalar version - * @param gain - * @param output - */ -template -void applyGain(T gain, absl::Span output) noexcept +template<> +void applyGain(const float* gain, const float* input, float* output, unsigned size) noexcept; + +template +inline void applyGain(absl::Span gain, absl::Span input, absl::Span output) noexcept { - applyGain(gain, output, output); + CHECK_SPAN_SIZES(gain, input, output); + applyGain(gain.data(), input.data(), output.data(), minSpanSize(gain, input, output)); } /** * @brief Applies a vector gain in-place on a span * - * The output size will be the minimum of the gain span and output span size. - * - * @tparam T the underlying type - * @tparam SIMD use the SIMD version or the scalar version * @param gain - * @param output + * @param array + * @param size */ -template -void applyGain(absl::Span gain, absl::Span output) noexcept +template +inline void applyGain(const T* gain, T* array, unsigned size) noexcept { - applyGain(gain, output, output); + applyGain(gain, array, array, size); } -template <> -void applyGain(float gain, absl::Span input, absl::Span output) noexcept; - -template <> -void applyGain(absl::Span gain, absl::Span input, absl::Span output) noexcept; +template +inline void applyGain(absl::Span gain, absl::Span array) noexcept +{ + CHECK_SPAN_SIZES(gain, array); + applyGain(gain.data(), array.data(), array.data(), minSpanSize(gain, array)); +} namespace _internals { template diff --git a/src/sfizz/SIMDSSE.cpp b/src/sfizz/SIMDSSE.cpp index 0c2f1a8a..5b893b80 100644 --- a/src/sfizz/SIMDSSE.cpp +++ b/src/sfizz/SIMDSSE.cpp @@ -16,47 +16,6 @@ constexpr uintptr_t TypeAlignment = 4; -template <> -void sfz::applyGain(float gain, absl::Span input, absl::Span output) noexcept -{ - auto* in = input.begin(); - auto* out = output.begin(); - const auto size = std::min(output.size(), input.size()); - const auto* lastAligned = prevAligned(output.begin() + size); - const auto mmGain = _mm_set_ps1(gain); - - while (unaligned(out, in) && out < lastAligned) - *out++ = gain * (*in++); - - while (out < lastAligned) { - _mm_store_ps(out, _mm_mul_ps(mmGain, _mm_load_ps(in))); - incrementAll(out, in); - } - - while (out < output.end()) - *out++ = gain * (*in++); -} - -template <> -void sfz::applyGain(absl::Span gain, absl::Span input, absl::Span output) noexcept -{ - auto* in = input.begin(); - auto* out = output.begin(); - auto* g = gain.begin(); - const auto size = std::min(output.size(), std::min(input.size(), gain.size())); - const auto* lastAligned = prevAligned(output.begin() + size); - - while (unaligned(out, in, g) && out < lastAligned) - _internals::snippetGainSpan(g, in, out); - - while (out < lastAligned) { - _mm_store_ps(out, _mm_mul_ps(_mm_load_ps(g), _mm_load_ps(in))); - incrementAll(g, in, out); - } - - while (out < output.end()) - _internals::snippetGainSpan(g, in, out); -} template <> diff --git a/tests/SIMDHelpersT.cpp b/tests/SIMDHelpersT.cpp index 20b57866..5b547dcf 100644 --- a/tests/SIMDHelpersT.cpp +++ b/tests/SIMDHelpersT.cpp @@ -275,7 +275,8 @@ TEST_CASE("[Helpers] Gain, single") std::array input { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }; std::array output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; std::array expected { fillValue, fillValue, fillValue, fillValue, fillValue }; - sfz::applyGain(fillValue, input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::gain, false); + sfz::applyGain(fillValue, input, absl::MakeSpan(output)); REQUIRE(output == expected); } @@ -283,7 +284,8 @@ TEST_CASE("[Helpers] Gain, single and inplace") { std::array buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }; std::array expected { fillValue, fillValue, fillValue, fillValue, fillValue }; - sfz::applyGain(fillValue, buffer, absl::MakeSpan(buffer)); + sfz::setSIMDOpStatus(sfz::SIMDOps::gain, false); + sfz::applyGain(fillValue, buffer, absl::MakeSpan(buffer)); REQUIRE(buffer == expected); } @@ -293,7 +295,8 @@ TEST_CASE("[Helpers] Gain, spans") std::array gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; std::array output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; std::array expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; - sfz::applyGain(gain, input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::gain, false); + sfz::applyGain(gain, input, absl::MakeSpan(output)); REQUIRE(output == expected); } @@ -302,7 +305,8 @@ TEST_CASE("[Helpers] Gain, spans and inplace") std::array buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }; std::array gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; std::array expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; - sfz::applyGain(gain, buffer, absl::MakeSpan(buffer)); + sfz::setSIMDOpStatus(sfz::SIMDOps::gain, false); + sfz::applyGain(gain, buffer, absl::MakeSpan(buffer)); REQUIRE(buffer == expected); } @@ -311,7 +315,8 @@ TEST_CASE("[Helpers] Gain, single (SIMD)") std::array input { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }; std::array output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; std::array expected { fillValue, fillValue, fillValue, fillValue, fillValue }; - sfz::applyGain(fillValue, input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::gain, true); + sfz::applyGain(fillValue, input, absl::MakeSpan(output)); REQUIRE(output == expected); } @@ -319,7 +324,8 @@ TEST_CASE("[Helpers] Gain, single and inplace (SIMD)") { std::array buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }; std::array expected { fillValue, fillValue, fillValue, fillValue, fillValue }; - sfz::applyGain(fillValue, buffer, absl::MakeSpan(buffer)); + sfz::setSIMDOpStatus(sfz::SIMDOps::gain, true); + sfz::applyGain(fillValue, buffer, absl::MakeSpan(buffer)); REQUIRE(buffer == expected); } @@ -329,7 +335,8 @@ TEST_CASE("[Helpers] Gain, spans (SIMD)") std::array gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; std::array output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; std::array expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; - sfz::applyGain(gain, input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::gain, true); + sfz::applyGain(gain, input, absl::MakeSpan(output)); REQUIRE(output == expected); } @@ -338,7 +345,8 @@ TEST_CASE("[Helpers] Gain, spans and inplace (SIMD)") std::array buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f }; std::array gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; std::array expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; - sfz::applyGain(gain, buffer, absl::MakeSpan(buffer)); + sfz::setSIMDOpStatus(sfz::SIMDOps::gain, true); + sfz::applyGain(gain, buffer, absl::MakeSpan(buffer)); REQUIRE(buffer == expected); }