From a83cd99040c8797f30c3eb89b839987c0fd28d8f Mon Sep 17 00:00:00 2001 From: Paul Ferrand Date: Sun, 31 May 2020 12:40:18 +0200 Subject: [PATCH] Move cumsum to the new format --- benchmarks/BM_cumsum.cpp | 12 +++++++---- benchmarks/BM_diff.cpp | 2 +- benchmarks/BM_envelopes.cpp | 2 +- src/sfizz/SIMDHelpers.cpp | 42 ++++++++++++++++++++++++++++++++++++- src/sfizz/SIMDHelpers.h | 34 +++++++++++++++++------------- src/sfizz/SIMDSSE.cpp | 31 --------------------------- tests/SIMDHelpersT.cpp | 9 +++++--- 7 files changed, 77 insertions(+), 55 deletions(-) diff --git a/benchmarks/BM_cumsum.cpp b/benchmarks/BM_cumsum.cpp index c8626848..9df02b76 100644 --- a/benchmarks/BM_cumsum.cpp +++ b/benchmarks/BM_cumsum.cpp @@ -35,28 +35,32 @@ public: BENCHMARK_DEFINE_F(CumArray, Sum_Scalar)(benchmark::State& state) { for (auto _ : state) { - sfz::cumsum(input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::cumsum, false); + sfz::cumsum(input, absl::MakeSpan(output)); } } BENCHMARK_DEFINE_F(CumArray, Sum_SIMD)(benchmark::State& state) { for (auto _ : state) { - sfz::cumsum(input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::cumsum, true); + sfz::cumsum(input, absl::MakeSpan(output)); } } BENCHMARK_DEFINE_F(CumArray, Sum_Scalar_Unaligned)(benchmark::State& state) { for (auto _ : state) { - sfz::cumsum(absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + sfz::setSIMDOpStatus(sfz::SIMDOps::cumsum, false); + sfz::cumsum(absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); } } BENCHMARK_DEFINE_F(CumArray, Sum_SIMD_Unaligned)(benchmark::State& state) { for (auto _ : state) { - sfz::cumsum(absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); + sfz::setSIMDOpStatus(sfz::SIMDOps::cumsum, true); + sfz::cumsum(absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1)); } } diff --git a/benchmarks/BM_diff.cpp b/benchmarks/BM_diff.cpp index af50441d..911556bb 100644 --- a/benchmarks/BM_diff.cpp +++ b/benchmarks/BM_diff.cpp @@ -22,7 +22,7 @@ public: input = std::vector(state.range(0)); output = std::vector(state.range(0)); std::generate(input.begin(), input.end(), [&]() { return dist(gen); }); - sfz::cumsum(input, absl::MakeSpan(input)); + sfz::cumsum(input, absl::MakeSpan(input)); } void TearDown(const ::benchmark::State& /* state */) { diff --git a/benchmarks/BM_envelopes.cpp b/benchmarks/BM_envelopes.cpp index 1fa3ae91..f58218a8 100644 --- a/benchmarks/BM_envelopes.cpp +++ b/benchmarks/BM_envelopes.cpp @@ -20,7 +20,7 @@ public: input = std::vector(state.range(0)); output = std::vector(state.range(0)); std::generate(input.begin(), input.end(), [&]() { return dist(gen); }); - sfz::cumsum(input, absl::MakeSpan(input)); + sfz::cumsum(input, absl::MakeSpan(input)); } void TearDown(const ::benchmark::State& /* state */) diff --git a/src/sfizz/SIMDHelpers.cpp b/src/sfizz/SIMDHelpers.cpp index 24c9a882..4dbff456 100644 --- a/src/sfizz/SIMDHelpers.cpp +++ b/src/sfizz/SIMDHelpers.cpp @@ -546,7 +546,7 @@ float meanSquared(const float* vector, unsigned size) noexcept #endif } - while (vector < sentinel){ + while (vector < sentinel) { result += (*vector) * (*vector); vector++; } @@ -554,4 +554,44 @@ float meanSquared(const float* vector, unsigned size) noexcept return result / static_cast(size); } +template <> +void cumsum(const float* input, float* output, unsigned size) noexcept +{ + if (size == 0) + return; + + const auto sentinel = output + size; + *output++ = *input++; + + if (getSIMDOpStatus(SIMDOps::cumsum)) { +#if SFIZZ_CPU_FAMILY_X86_64 || SFIZZ_CPU_FAMILY_I386 + if (cpuInfo.has_sse()) { + const auto* lastAligned = prevAligned(sentinel); + + while (unaligned(input, output) && output < lastAligned) { + *output = *(output - 1) + *input++; + output++; + } + + auto mmOutput = _mm_set_ps1(*(output - 1)); + while (output < lastAligned) { + auto mmOffset = _mm_load_ps(input); + mmOffset = _mm_add_ps(mmOffset, _mm_castsi128_ps(_mm_slli_si128(_mm_castps_si128(mmOffset), 4))); + mmOffset = _mm_add_ps(mmOffset, _mm_shuffle_ps(_mm_setzero_ps(), mmOffset, _MM_SHUFFLE(1, 0, 0, 0))); + mmOutput = _mm_add_ps(mmOutput, mmOffset); + _mm_store_ps(output, mmOutput); + mmOutput = _mm_shuffle_ps(mmOutput, mmOutput, _MM_SHUFFLE(3, 3, 3, 3)); + incrementAll<4>(input, output); + } + // fallthrough from lastAligned to sentinel + } +#endif + } + + while (output < sentinel) { + *output = *(output - 1) + *input++; + output++; + } +} + } diff --git a/src/sfizz/SIMDHelpers.h b/src/sfizz/SIMDHelpers.h index 69d3bb42..509766f6 100644 --- a/src/sfizz/SIMDHelpers.h +++ b/src/sfizz/SIMDHelpers.h @@ -606,28 +606,34 @@ namespace _internals { * The output size will be the minimum of the input span and output span sizes. * * @tparam T the underlying type - * @tparam SIMD use the SIMD version or the scalar version - * @param vector - * @return T + * @param input + * @param output + * @param size */ -template -void cumsum(absl::Span input, absl::Span output) noexcept +template +void cumsum(const T* input, T* output, unsigned size) noexcept { - CHECK(output.size() >= input.size()); - if (input.size() == 0) + if (size == 0) return; - auto out = output.data(); - auto in = input.data(); - const auto sentinel = in + std::min(input.size(), output.size()); + const auto sentinel = output + size; - *out++ = *in++; - while (in < sentinel) - _internals::snippetCumsum(in, out); + *output++ = *input++; + while (output < sentinel) { + *output = *(output - 1) + *input++; + output++; + } } template <> -void cumsum(absl::Span input, absl::Span output) noexcept; +void cumsum(const float* input, float* output, unsigned size) noexcept; + +template +void cumsum(absl::Span input, absl::Span output) noexcept +{ + CHECK_SPAN_SIZES(input, output); + cumsum(input.data(), output.data(), minSpanSize(input, output)); +} // FIXME: This should go away once the changes from the resampler are in diff --git a/src/sfizz/SIMDSSE.cpp b/src/sfizz/SIMDSSE.cpp index ca29d7ba..8e04801d 100644 --- a/src/sfizz/SIMDSSE.cpp +++ b/src/sfizz/SIMDSSE.cpp @@ -16,37 +16,6 @@ constexpr uintptr_t TypeAlignment = 4; -template <> -void sfz::cumsum(absl::Span input, absl::Span output) noexcept -{ - CHECK(output.size() >= input.size()); - if (input.size() == 0) - return; - - auto out = output.data(); - auto in = input.data(); - const auto sentinel = in + std::min(input.size(), output.size()); - const auto lastAligned = prevAligned(sentinel); - - *out++ = *in++; - while (unaligned(in, out) && in < lastAligned) - _internals::snippetCumsum(in, out); - - auto mmOutput = _mm_set_ps1(*(out - 1)); - while (in < lastAligned) { - auto mmOffset = _mm_load_ps(in); - mmOffset = _mm_add_ps(mmOffset, _mm_castsi128_ps(_mm_slli_si128(_mm_castps_si128(mmOffset), 4))); - mmOffset = _mm_add_ps(mmOffset, _mm_shuffle_ps(_mm_setzero_ps(), mmOffset, _MM_SHUFFLE(1, 0, 0, 0))); - mmOutput = _mm_add_ps(mmOutput, mmOffset); - _mm_store_ps(out, mmOutput); - mmOutput = _mm_shuffle_ps(mmOutput, mmOutput, _MM_SHUFFLE(3, 3, 3, 3)); - incrementAll(in, out); - } - - while (in < sentinel) - _internals::snippetCumsum(in, out); -} - template <> void sfz::diff(absl::Span input, absl::Span output) noexcept { diff --git a/tests/SIMDHelpersT.cpp b/tests/SIMDHelpersT.cpp index 0eefbd34..15003d9a 100644 --- a/tests/SIMDHelpersT.cpp +++ b/tests/SIMDHelpersT.cpp @@ -696,7 +696,8 @@ TEST_CASE("[Helpers] Cumulative sum") std::array input { 1.1f, 1.2f, 1.3f, 1.4f, 1.5f, 1.6f }; // 1.1 2.3 3.6 5.0f 6.5 8.1 std::array output; std::array expected { 1.1f, 2.3f, 3.6f, 5.0f, 6.5f, 8.1f }; - sfz::cumsum(input, absl::MakeSpan(output)); + sfz::setSIMDOpStatus(sfz::SIMDOps::cumsum, false); + sfz::cumsum(input, absl::MakeSpan(output)); REQUIRE(approxEqual(output, expected)); } @@ -707,8 +708,10 @@ TEST_CASE("[Helpers] Cumulative sum (SIMD vs Scalar)") std::vector outputSIMD(bigBufferSize); sfz::setSIMDOpStatus(sfz::SIMDOps::linearRamp, true); sfz::linearRamp(absl::MakeSpan(input), 0.0f, 0.1f); - sfz::cumsum(input, absl::MakeSpan(outputScalar)); - sfz::cumsum(input, absl::MakeSpan(outputSIMD)); + sfz::setSIMDOpStatus(sfz::SIMDOps::cumsum, false); + sfz::cumsum(input, absl::MakeSpan(outputScalar)); + sfz::setSIMDOpStatus(sfz::SIMDOps::cumsum, true); + sfz::cumsum(input, absl::MakeSpan(outputSIMD)); REQUIRE(approxEqual(outputScalar, outputSIMD)); }