sam/sfizz
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Lots of template work

Browse source
This commit is contained in:
paul 2019-08-21 23:00:06 +02:00
parent 8e5c6650aa
commit d2e55cf4b2
10 changed files with 463 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
CMakeLists.txt
View file

@ -178,4 +178,7 @@ add_executable(bm_mathfuns benchmarks/BM_mathfuns.cpp sources/SIMDSSE.cpp)
target_link_libraries(bm_mathfuns benchmark absl::span absl::span) target_link_libraries(bm_mathfuns benchmark absl::span absl::span)
if (UNIX) if (UNIX)
# target_compile_options(bm_math_loops PRIVATE -fopenmp) # target_compile_options(bm_math_loops PRIVATE -fopenmp)
endif() endif()
add_executable(bm_gain benchmarks/BM_gain.cpp sources/SIMDSSE.cpp)
target_link_libraries(bm_gain benchmark absl::span)

28
benchmarks/BM_fill.cpp
View file

@ -22,7 +22,17 @@ static void Fill_float(benchmark::State& state) {
std::mt19937 gen { rd() }; std::mt19937 gen { rd() };
std::uniform_real_distribution<float> dist { 1, 2 }; std::uniform_real_distribution<float> dist { 1, 2 };
for (auto _ : state) { for (auto _ : state) {
fill<float, false>(buffer, dist(gen)); fill<float, false>(absl::MakeSpan(buffer), dist(gen));
}
}
static void Fill_float_unaligned(benchmark::State& state) {
Buffer<float> buffer (state.range(0));
std::random_device rd { };
std::mt19937 gen { rd() };
std::uniform_real_distribution<float> dist { 1, 2 };
for (auto _ : state) {
fill<float, false>(absl::MakeSpan(buffer).subspan(1), dist(gen));
} }
} }
@ -32,7 +42,17 @@ static void Fill_float_SSE(benchmark::State& state) {
std::mt19937 gen { rd() }; std::mt19937 gen { rd() };
std::uniform_real_distribution<float> dist { 1, 2 }; std::uniform_real_distribution<float> dist { 1, 2 };
for (auto _ : state) { for (auto _ : state) {
fill<float, true>(buffer, dist(gen)); fill<float, true>(absl::MakeSpan(buffer), dist(gen));
}
}
static void Fill_float_SSE_unaligned(benchmark::State& state) {
Buffer<float> buffer (state.range(0));
std::random_device rd { };
std::mt19937 gen { rd() };
std::uniform_real_distribution<float> dist { 1, 2 };
for (auto _ : state) {
fill<float, true>(absl::MakeSpan(buffer).subspan(1), dist(gen));
} }
} }
@ -42,12 +62,14 @@ static void Fill_double(benchmark::State& state) {
std::mt19937 gen { rd() }; std::mt19937 gen { rd() };
std::uniform_real_distribution<double> dist { 1, 2 }; std::uniform_real_distribution<double> dist { 1, 2 };
for (auto _ : state) { for (auto _ : state) {
fill<double>(buffer, dist(gen)); fill<double>(absl::MakeSpan(buffer), dist(gen));
} }
} }
BENCHMARK(Dummy)->Range((2<<6), (2<<16)); BENCHMARK(Dummy)->Range((2<<6), (2<<16));
BENCHMARK(Fill_float)->Range((2<<6), (2<<16)); BENCHMARK(Fill_float)->Range((2<<6), (2<<16));
BENCHMARK(Fill_float_SSE)->Range((2<<6), (2<<16)); BENCHMARK(Fill_float_SSE)->Range((2<<6), (2<<16));
BENCHMARK(Fill_float_unaligned)->Range((2<<6), (2<<16));
BENCHMARK(Fill_float_SSE_unaligned)->Range((2<<6), (2<<16));
BENCHMARK(Fill_double)->Range((2<<6), (2<<16)); BENCHMARK(Fill_double)->Range((2<<6), (2<<16));
BENCHMARK_MAIN(); BENCHMARK_MAIN();

120
benchmarks/BM_gain.cpp Normal file
View file

@ -0,0 +1,120 @@
#include <benchmark/benchmark.h>
#include <random>
#include <numeric>
#include <vector>
#include <cmath>
#include <iostream>
#include "../sources/SIMDHelpers.h"
class GainSingle : public benchmark::Fixture {
public:
void SetUp(const ::benchmark::State& state) {
std::random_device rd { };
std::mt19937 gen { rd() };
std::uniform_real_distribution<float> dist { 0, 1 };
input = std::vector<float>(state.range(0));
output = std::vector<float>(state.range(0));
gain = dist(gen);
std::generate(input.begin(), input.end(), [&]() { return dist(gen); });
}
void TearDown(const ::benchmark::State& state [[maybe_unused]]) {
}
float gain;
std::vector<float> input;
std::vector<float> output;
};
class GainArray : public benchmark::Fixture {
public:
void SetUp(const ::benchmark::State& state) {
std::random_device rd { };
std::mt19937 gen { rd() };
std::uniform_real_distribution<float> dist { 0, 1 };
input = std::vector<float>(state.range(0));
output = std::vector<float>(state.range(0));
gain = std::vector<float>(state.range(0));
std::generate(gain.begin(), gain.end(), [&]() { return dist(gen); });
std::generate(input.begin(), input.end(), [&]() { return dist(gen); });
}
void TearDown(const ::benchmark::State& state [[maybe_unused]]) {
}
std::vector<float> gain;
std::vector<float> input;
std::vector<float> output;
};
BENCHMARK_DEFINE_F(GainSingle, Straight)(benchmark::State& state) {
for (auto _ : state)
{
for (int i = 0; i < state.range(0); ++i)
output[i] = gain * input[i];
}
}
BENCHMARK_DEFINE_F(GainSingle, Scalar)(benchmark::State& state) {
for (auto _ : state)
{
applyGain<float, false>(gain, input, absl::MakeSpan(output));
}
}
BENCHMARK_DEFINE_F(GainSingle, SIMD)(benchmark::State& state) {
for (auto _ : state)
{
applyGain<float, true>(gain, input, absl::MakeSpan(output));
}
}
BENCHMARK_DEFINE_F(GainArray, Straight)(benchmark::State& state) {
for (auto _ : state)
{
for (int i = 0; i < state.range(0); ++i)
output[i] = gain[i] * input[i];
}
}
BENCHMARK_DEFINE_F(GainArray, Scalar)(benchmark::State& state) {
for (auto _ : state)
{
applyGain<float, false>(gain, input, absl::MakeSpan(output));
}
}
BENCHMARK_DEFINE_F(GainArray, SIMD)(benchmark::State& state) {
for (auto _ : state)
{
applyGain<float, true>(gain, input, absl::MakeSpan(output));
}
}
BENCHMARK_DEFINE_F(GainArray, Scalar_Unaligned)(benchmark::State& state) {
for (auto _ : state)
{
applyGain<float, false>(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1));
}
}
BENCHMARK_DEFINE_F(GainArray, SIMD_Unaligned)(benchmark::State& state) {
for (auto _ : state)
{
applyGain<float, true>(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1));
}
}
BENCHMARK_REGISTER_F(GainSingle, Straight)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
BENCHMARK_REGISTER_F(GainSingle, Scalar)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
BENCHMARK_REGISTER_F(GainSingle, SIMD)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
BENCHMARK_REGISTER_F(GainArray, Straight)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
BENCHMARK_REGISTER_F(GainArray, Scalar)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
BENCHMARK_REGISTER_F(GainArray, SIMD)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
BENCHMARK_REGISTER_F(GainArray, Scalar_Unaligned)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
BENCHMARK_REGISTER_F(GainArray, SIMD_Unaligned)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
BENCHMARK_MAIN();

26
benchmarks/BM_readInterleaved.cpp
View file

@ -33,7 +33,7 @@ static void Scalar_Unaligned(benchmark::State& state) {
Buffer<float> outputRight (state.range(0)); Buffer<float> outputRight (state.range(0));
std::iota(input.begin(), input.end(), 1.0f); std::iota(input.begin(), input.end(), 1.0f);
for (auto _ : state) { for (auto _ : state) {
readInterleaved<float, false>(absl::MakeSpan(input).subspan(1), absl::MakeSpan(outputLeft).subspan(1), absl::MakeSpan(outputRight).subspan(1)); readInterleaved<float, false>(absl::MakeSpan(input).subspan(2), absl::MakeSpan(outputLeft), absl::MakeSpan(outputRight));
} }
} }
@ -43,7 +43,27 @@ static void SSE_Unaligned(benchmark::State& state) {
Buffer<float> outputRight (state.range(0)); Buffer<float> outputRight (state.range(0));
std::iota(input.begin(), input.end(), 1.0f); std::iota(input.begin(), input.end(), 1.0f);
for (auto _ : state) { for (auto _ : state) {
readInterleaved<float, true>(absl::MakeSpan(input).subspan(1), absl::MakeSpan(outputLeft).subspan(1), absl::MakeSpan(outputRight).subspan(1)); readInterleaved<float, true>(absl::MakeSpan(input).subspan(2), absl::MakeSpan(outputLeft), absl::MakeSpan(outputRight));
}
}
static void Scalar_Unaligned_2(benchmark::State& state) {
Buffer<float> input (state.range(0) * 2);
Buffer<float> outputLeft (state.range(0));
Buffer<float> outputRight (state.range(0));
std::iota(input.begin(), input.end(), 1.0f);
for (auto _ : state) {
readInterleaved<float, false>(absl::MakeSpan(input).subspan(2), absl::MakeSpan(outputLeft).subspan(1), absl::MakeSpan(outputRight).subspan(3));
}
}
static void SSE_Unaligned_2(benchmark::State& state) {
Buffer<float> input (state.range(0) * 2);
Buffer<float> outputLeft (state.range(0));
Buffer<float> outputRight (state.range(0));
std::iota(input.begin(), input.end(), 1.0f);
for (auto _ : state) {
readInterleaved<float, true>(absl::MakeSpan(input).subspan(2), absl::MakeSpan(outputLeft).subspan(1), absl::MakeSpan(outputRight).subspan(3));
} }
} }
@ -51,4 +71,6 @@ BENCHMARK(Scalar)->Range((8<<10), (8<<20));
BENCHMARK(SSE)->Range((8<<10), (8<<20)); BENCHMARK(SSE)->Range((8<<10), (8<<20));
BENCHMARK(Scalar_Unaligned)->Range((8<<10), (8<<20)); BENCHMARK(Scalar_Unaligned)->Range((8<<10), (8<<20));
BENCHMARK(SSE_Unaligned)->Range((8<<10), (8<<20)); BENCHMARK(SSE_Unaligned)->Range((8<<10), (8<<20));
BENCHMARK(Scalar_Unaligned_2)->Range((8<<10), (8<<20));
BENCHMARK(SSE_Unaligned_2)->Range((8<<10), (8<<20));
BENCHMARK_MAIN(); BENCHMARK_MAIN();

30
benchmarks/BM_writeInterleaved.cpp
View file

@ -36,7 +36,7 @@ static void Unaligned_Interleaved_Write(benchmark::State& state) {
std::iota(inputLeft.begin(), inputLeft.end(), 1.0f); std::iota(inputLeft.begin(), inputLeft.end(), 1.0f);
std::iota(inputRight.begin(), inputRight.end(), 1.0f); std::iota(inputRight.begin(), inputRight.end(), 1.0f);
for (auto _ : state) { for (auto _ : state) {
writeInterleaved<float, false>(absl::MakeSpan(inputLeft).subspan(1) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(1)); writeInterleaved<float, false>(absl::MakeSpan(inputLeft).subspan(1) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(2));
benchmark::DoNotOptimize(output); benchmark::DoNotOptimize(output);
} }
} }
@ -48,7 +48,31 @@ static void Unaligned_Interleaved_Write_SSE(benchmark::State& state) {
std::iota(inputLeft.begin(), inputLeft.end(), 1.0f); std::iota(inputLeft.begin(), inputLeft.end(), 1.0f);
std::iota(inputRight.begin(), inputRight.end(), 1.0f); std::iota(inputRight.begin(), inputRight.end(), 1.0f);
for (auto _ : state) { for (auto _ : state) {
writeInterleaved<float, true>(absl::MakeSpan(inputLeft).subspan(1) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(1)); writeInterleaved<float, true>(absl::MakeSpan(inputLeft).subspan(1) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(2));
benchmark::DoNotOptimize(output);
}
}
static void Unaligned_Interleaved_Write_2(benchmark::State& state) {
Buffer<float> inputLeft (state.range(0));
Buffer<float> inputRight (state.range(0));
Buffer<float> output (state.range(0) * 2);
std::iota(inputLeft.begin(), inputLeft.end(), 1.0f);
std::iota(inputRight.begin(), inputRight.end(), 1.0f);
for (auto _ : state) {
writeInterleaved<float, false>(absl::MakeSpan(inputLeft) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(2));
benchmark::DoNotOptimize(output);
}
}
static void Unaligned_Interleaved_Write_SSE_2(benchmark::State& state) {
Buffer<float> inputLeft (state.range(0));
Buffer<float> inputRight (state.range(0));
Buffer<float> output (state.range(0) * 2);
std::iota(inputLeft.begin(), inputLeft.end(), 1.0f);
std::iota(inputRight.begin(), inputRight.end(), 1.0f);
for (auto _ : state) {
writeInterleaved<float, true>(absl::MakeSpan(inputLeft) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(2));
benchmark::DoNotOptimize(output); benchmark::DoNotOptimize(output);
} }
} }
@ -57,4 +81,6 @@ BENCHMARK(Interleaved_Write)->Range((8<<10), (8<<20));
BENCHMARK(Interleaved_Write_SSE)->Range((8<<10), (8<<20)); BENCHMARK(Interleaved_Write_SSE)->Range((8<<10), (8<<20));
BENCHMARK(Unaligned_Interleaved_Write)->Range((8<<10), (8<<20)); BENCHMARK(Unaligned_Interleaved_Write)->Range((8<<10), (8<<20));
BENCHMARK(Unaligned_Interleaved_Write_SSE)->Range((8<<10), (8<<20)); BENCHMARK(Unaligned_Interleaved_Write_SSE)->Range((8<<10), (8<<20));
BENCHMARK(Unaligned_Interleaved_Write_2)->Range((8<<10), (8<<20));
BENCHMARK(Unaligned_Interleaved_Write_SSE_2)->Range((8<<10), (8<<20));
BENCHMARK_MAIN(); BENCHMARK_MAIN();

2
sources/Globals.h
View file

@ -26,6 +26,8 @@ namespace SIMDConfig
constexpr bool writeInterleaved { true }; constexpr bool writeInterleaved { true };
constexpr bool readInterleaved { true }; constexpr bool readInterleaved { true };
constexpr bool fill { false }; constexpr bool fill { false };
constexpr bool gain { false };
constexpr bool mathfuns { false };
#if USE_SIMD #if USE_SIMD
constexpr bool useSIMD { true }; constexpr bool useSIMD { true };
#else #else

12
sources/SIMDDummy.cpp
View file

@ -56,4 +56,16 @@ template<>
void cos<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept void cos<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept
{ {
cos<float, false>(input, output); cos<float, false>(input, output);
}
template<>
void applyGain<float, true>(float gain, absl::Span<const float> input, absl::Span<float> output) noexcept
{
applyGain<float, false>(gain, input, output);
}
template<>
void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept
{
applyGain<float, false>(gain, input, output);
} }

58
sources/SIMDHelpers.h
View file

@ -1,6 +1,5 @@
#include "absl/types/span.h"
#include "Globals.h" #include "Globals.h"
#include <absl/types/span.h>
#include "Helpers.h" #include "Helpers.h"
#include <cmath> #include <cmath>
@ -52,7 +51,7 @@ void fill(absl::Span<T> output, T value) noexcept
template<> template<>
void fill<float, true>(absl::Span<float> output, float value) noexcept; void fill<float, true>(absl::Span<float> output, float value) noexcept;
template<class Type, bool SIMD=SIMDConfig::useSIMD> template<class Type, bool SIMD=SIMDConfig::mathfuns>
void exp(absl::Span<const Type> input, absl::Span<Type> output) noexcept void exp(absl::Span<const Type> input, absl::Span<Type> output) noexcept
{ {
ASSERT(output.size() >= input.size()); ASSERT(output.size() >= input.size());
@ -64,7 +63,7 @@ void exp(absl::Span<const Type> input, absl::Span<Type> output) noexcept
template<> template<>
void exp<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept; void exp<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
template<class Type, bool SIMD=SIMDConfig::useSIMD> template<class Type, bool SIMD=SIMDConfig::mathfuns>
void log(absl::Span<const Type> input, absl::Span<Type> output) noexcept void log(absl::Span<const Type> input, absl::Span<Type> output) noexcept
{ {
ASSERT(output.size() >= input.size()); ASSERT(output.size() >= input.size());
@ -76,7 +75,7 @@ void log(absl::Span<const Type> input, absl::Span<Type> output) noexcept
template<> template<>
void log<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept; void log<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
template<class Type, bool SIMD=SIMDConfig::useSIMD> template<class Type, bool SIMD=SIMDConfig::mathfuns>
void sin(absl::Span<const Type> input, absl::Span<Type> output) noexcept void sin(absl::Span<const Type> input, absl::Span<Type> output) noexcept
{ {
ASSERT(output.size() >= input.size()); ASSERT(output.size() >= input.size());
@ -88,7 +87,7 @@ void sin(absl::Span<const Type> input, absl::Span<Type> output) noexcept
template<> template<>
void sin<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept; void sin<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
template<class Type, bool SIMD=SIMDConfig::useSIMD> template<class Type, bool SIMD=SIMDConfig::mathfuns>
void cos(absl::Span<const Type> input, absl::Span<Type> output) noexcept void cos(absl::Span<const Type> input, absl::Span<Type> output) noexcept
{ {
ASSERT(output.size() >= input.size()); ASSERT(output.size() >= input.size());
@ -109,9 +108,48 @@ void linearRamp(absl::Span<T> output, T start, T end);
template<class T, bool SIMD=SIMDConfig::useSIMD> template<class T, bool SIMD=SIMDConfig::useSIMD>
void exponentialRamp(absl::Span<T> output, T start, T end); void exponentialRamp(absl::Span<T> output, T start, T end);
template<class T, bool SIMD=SIMDConfig::useSIMD> template<class T, bool SIMD=SIMDConfig::gain>
void applyGain(T gain, absl::Span<T> output); void applyGain(T gain, absl::Span<const T> input, absl::Span<T> output) noexcept
{
ASSERT(input.size() <= output.size());
auto* in = input.begin();
auto* out = output.begin();
auto* sentinel = out + std::min(output.size(), input.size());
while (out < sentinel)
{
*out++ = gain * (*in++);
}
}
template<class T, bool SIMD=SIMDConfig::useSIMD> template<class T, bool SIMD=SIMDConfig::gain>
void applyGain(absl::Span<const T> gain, absl::Span<T> output); void applyGain(absl::Span<const T> gain, absl::Span<const T> input, absl::Span<T> output) noexcept
{
ASSERT(gain.size() == input.size());
ASSERT(input.size() <= output.size());
auto* in = input.begin();
auto* g = gain.begin();
auto* out = output.begin();
auto* sentinel = out + std::min(gain.size(), std::min(output.size(), input.size()));
while (out < sentinel)
{
*out++ = (*g++) * (*in++);
}
}
template<class T, bool SIMD=SIMDConfig::gain>
void applyGain(T gain, absl::Span<T> output) noexcept
{
applyGain<T, SIMD>(gain, output, output);
}
template<class T, bool SIMD=SIMDConfig::gain>
void applyGain(absl::Span<const T> gain, absl::Span<T> output) noexcept
{
applyGain<T, SIMD>(gain, output, output);
}
template<>
void applyGain<float, true>(float gain, absl::Span<const float> input, absl::Span<float> output) noexcept;
template<>
void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept;

165
sources/SIMDSSE.cpp
View file

@ -3,11 +3,42 @@
#include "x86intrin.h" #include "x86intrin.h"
#include "mathfuns/sse_mathfun.h" #include "mathfuns/sse_mathfun.h"
constexpr int TypeAlignment { 4 }; constexpr uintptr_t TypeAlignment { 4 };
constexpr uintptr_t TypeAlignmentMask { TypeAlignment - 1 };
using Type = float; using Type = float;
constexpr uintptr_t ByteAlignment { TypeAlignment * sizeof(Type) };
constexpr uintptr_t ByteAlignmentMask { ByteAlignment - 1 };
struct AlignmentSentinels { float* nextAligned; float* lastAligned; };
float* nextAligned(const float* ptr)
{
return reinterpret_cast<float*>( (reinterpret_cast<uintptr_t>(ptr) + ByteAlignmentMask) & (~ByteAlignmentMask) );
}
float* prevAligned(const float* ptr)
{
return reinterpret_cast<float*>( reinterpret_cast<uintptr_t>(ptr) & (~ByteAlignmentMask) );
}
bool unaligned(const float* ptr)
{
return (reinterpret_cast<uintptr_t>(ptr) & ByteAlignmentMask) != 0;
}
bool unaligned(const float* ptr1, const float* ptr2)
{
return unaligned(ptr1) || unaligned(ptr2);
}
bool unaligned(const float* ptr1, const float* ptr2, const float* ptr3)
{
return unaligned(ptr1) || unaligned(ptr2) || unaligned(ptr3);
}
template<> template<>
void readInterleaved<Type, true>(absl::Span<const Type> input, absl::Span<Type> outputLeft, absl::Span<Type> outputRight) noexcept void readInterleaved<float, true>(absl::Span<const float> input, absl::Span<float> outputLeft, absl::Span<float> outputRight) noexcept
{ {
// The size of the outputs is not big enough for the input... // The size of the outputs is not big enough for the input...
ASSERT(outputLeft.size() >= input.size() / 2); ASSERT(outputLeft.size() >= input.size() / 2);
@ -18,14 +49,21 @@ void readInterleaved<Type, true>(absl::Span<const Type> input, absl::Span<Type>
auto* in = input.begin(); auto* in = input.begin();
auto* lOut = outputLeft.begin(); auto* lOut = outputLeft.begin();
auto* rOut = outputRight.begin(); auto* rOut = outputRight.begin();
const int unalignedEnd = input.size() & (2 * TypeAlignment - 1);
const int lastAligned = input.size() - unalignedEnd; const auto size = std::min(input.size(), std::min(outputLeft.size() * 2, outputRight.size() * 2 ));
auto* inputSentinel = in + lastAligned; const auto* lastAligned = prevAligned(input.begin() + size - TypeAlignment);
while (in < inputSentinel && lOut < outputLeft.end() && rOut < outputRight.end())
while (unaligned(in, lOut, rOut) && in < lastAligned)
{ {
auto register0 = _mm_loadu_ps(in); *lOut++ = *in++;
*rOut++ = *in++;
}
while (in < lastAligned )
{
auto register0 = _mm_load_ps(in);
in += TypeAlignment; in += TypeAlignment;
auto register1 = _mm_loadu_ps(in); auto register1 = _mm_load_ps(in);
in += TypeAlignment; in += TypeAlignment;
auto register2 = register0; auto register2 = register0;
// register 2 holds the copy of register 0 that is going to get erased by the first operation // register 2 holds the copy of register 0 that is going to get erased by the first operation
@ -33,14 +71,13 @@ void readInterleaved<Type, true>(absl::Span<const Type> input, absl::Span<Type>
// "take 0 from a, take 2 from a, take 0 from b, take 2 from b" // "take 0 from a, take 2 from a, take 0 from b, take 2 from b"
register0 = _mm_shuffle_ps(register0, register1, 0b10001000); register0 = _mm_shuffle_ps(register0, register1, 0b10001000);
register1 = _mm_shuffle_ps(register2, register1, 0b11011101); register1 = _mm_shuffle_ps(register2, register1, 0b11011101);
_mm_storeu_ps(lOut, register0); _mm_store_ps(lOut, register0);
_mm_storeu_ps(rOut, register1); _mm_store_ps(rOut, register1);
lOut += TypeAlignment; lOut += TypeAlignment;
rOut += TypeAlignment; rOut += TypeAlignment;
} }
inputSentinel = input.end() - 1; while (in < input.end() - 1)
while (in < inputSentinel && lOut < outputLeft.end() && rOut < outputRight.end())
{ {
*lOut++ = *in++; *lOut++ = *in++;
*rOut++ = *in++; *rOut++ = *in++;
@ -48,7 +85,7 @@ void readInterleaved<Type, true>(absl::Span<const Type> input, absl::Span<Type>
} }
template<> template<>
void writeInterleaved<Type, true>(absl::Span<const Type> inputLeft, absl::Span<const Type> inputRight, absl::Span<Type> output) noexcept void writeInterleaved<float, true>(absl::Span<const float> inputLeft, absl::Span<const float> inputRight, absl::Span<float> output) noexcept
{ {
// The size of the output is not big enough for the inputs... // The size of the output is not big enough for the inputs...
ASSERT(inputLeft.size() <= output.size() / 2); ASSERT(inputLeft.size() <= output.size() / 2);
@ -58,48 +95,53 @@ void writeInterleaved<Type, true>(absl::Span<const Type> inputLeft, absl::Span<c
auto* rIn = inputRight.begin(); auto* rIn = inputRight.begin();
auto* out = output.begin(); auto* out = output.begin();
const int residualLeft = inputLeft.size() & (TypeAlignment - 1); const auto size = std::min(output.size(), std::min(inputLeft.size(), inputRight.size()) * 2);
const int residualRight = inputRight.size() & (TypeAlignment - 1); const auto* lastAligned = prevAligned(output.begin() + size - TypeAlignment);
const auto* leftSentinel = lIn + inputLeft.size() - residualLeft;
const auto* rightSentinel = rIn + inputRight.size() - residualRight;
const auto* outputSentinel = output.end() - 1;
while (lIn < leftSentinel && rIn < rightSentinel && out < outputSentinel) while (unaligned(out, rIn, lIn) && out < lastAligned)
{ {
const auto lInRegister = _mm_loadu_ps(lIn); *out++ = *lIn++;
const auto rInRegister = _mm_loadu_ps(rIn); *out++ = *rIn++;
}
while (out < lastAligned)
{
const auto lInRegister = _mm_load_ps(lIn);
const auto rInRegister = _mm_load_ps(rIn);
const auto outRegister1 = _mm_unpacklo_ps(lInRegister, rInRegister); const auto outRegister1 = _mm_unpacklo_ps(lInRegister, rInRegister);
_mm_storeu_ps(out, outRegister1); _mm_store_ps(out, outRegister1);
out += TypeAlignment; out += TypeAlignment;
const auto outRegister2 = _mm_unpackhi_ps(lInRegister, rInRegister); const auto outRegister2 = _mm_unpackhi_ps(lInRegister, rInRegister);
_mm_storeu_ps(out, outRegister2); _mm_store_ps(out, outRegister2);
out += TypeAlignment; out += TypeAlignment;
lIn += TypeAlignment; lIn += TypeAlignment;
rIn += TypeAlignment; rIn += TypeAlignment;
} }
while (lIn < inputLeft.end() && rIn < inputRight.end() && out < outputSentinel) while (out < output.end() - 1)
{ {
*out++ = *lIn++; *out++ = *lIn++;
*out++ = *rIn++; *out++ = *rIn++;
} }
} }
template<> template<>
void fill<float, true>(absl::Span<float> output, float value) noexcept void fill<float, true>(absl::Span<float> output, float value) noexcept
{ {
const auto mmValue = _mm_set_ps1(value); const auto mmValue = _mm_set_ps1(value);
auto* out = output.begin(); auto* out = output.begin();
const int residual = output.size() & (TypeAlignment - 1); const auto* lastAligned = prevAligned(output.end());
const auto* sentinel = output.end() - residual;
while (out < sentinel) // we should only need to test a single channel while (unaligned(out) && out < lastAligned)
*out++ = value;
while (out < lastAligned) // we should only need to test a single channel
{ {
_mm_storeu_ps(out, mmValue); _mm_store_ps(out, mmValue);
out += TypeAlignment; out += TypeAlignment;
} }
@ -117,8 +159,8 @@ void exp<float, true>(absl::Span<const float> input, absl::Span<float> output) n
while (in < sentinel) while (in < sentinel)
{ {
_mm_storeu_ps(out, exp_ps(_mm_loadu_ps(in))); _mm_storeu_ps(out, exp_ps(_mm_loadu_ps(in)));
out += 4; out += TypeAlignment;
in += 4; in += TypeAlignment;
} }
} }
@ -132,8 +174,8 @@ void cos<float, true>(absl::Span<const float> input, absl::Span<float> output) n
while (in < sentinel) while (in < sentinel)
{ {
_mm_storeu_ps(out, cos_ps(_mm_loadu_ps(in))); _mm_storeu_ps(out, cos_ps(_mm_loadu_ps(in)));
out += 4; out += TypeAlignment;
in += 4; in += TypeAlignment;
} }
} }
@ -147,11 +189,13 @@ void log<float, true>(absl::Span<const float> input, absl::Span<float> output) n
while (in < sentinel) while (in < sentinel)
{ {
_mm_storeu_ps(out, log_ps(_mm_loadu_ps(in))); _mm_storeu_ps(out, log_ps(_mm_loadu_ps(in)));
out += 4; out += TypeAlignment;
in += 4; in += TypeAlignment;
} }
} }
template<> template<>
void sin<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept void sin<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept
{ {
@ -162,7 +206,54 @@ void sin<float, true>(absl::Span<const float> input, absl::Span<float> output) n
while (in < sentinel) while (in < sentinel)
{ {
_mm_storeu_ps(out, sin_ps(_mm_loadu_ps(in))); _mm_storeu_ps(out, sin_ps(_mm_loadu_ps(in)));
out += 4; out += TypeAlignment;
in += 4; in += TypeAlignment;
} }
}
template<>
void applyGain<float, true>(float gain, absl::Span<const float> input, absl::Span<float> output) noexcept
{
auto* in = input.begin();
auto* out = output.begin();
const auto size = std::min(output.size(), input.size());
const auto* lastAligned = prevAligned(output.begin() + size);
const auto mmGain = _mm_set_ps1(gain);
while (unaligned(out, in) && out < lastAligned)
*out++ = gain * (*in++);
while (out < lastAligned)
{
_mm_store_ps(out, _mm_mul_ps(mmGain, _mm_load_ps(in)));
in += TypeAlignment;
out += TypeAlignment;
}
while (out < output.end())
*out++ = gain * (*in++);
}
template<>
void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept
{
auto* in = input.begin();
auto* out = output.begin();
auto* g = gain.begin();
const auto size = std::min(output.size(), std::min(input.size(), gain.size()));
const auto* lastAligned = prevAligned(output.begin() + size);
while (unaligned(out, in, g) && out < lastAligned)
*out++ = (*g++) * (*in++);
while (out < lastAligned)
{
_mm_store_ps(out, _mm_mul_ps(_mm_load_ps(g), _mm_load_ps(in)));
g += TypeAlignment;
in += TypeAlignment;
out += TypeAlignment;
}
while (out < output.end())
*out++ = (*g++) * (*in++);
} }

72
tests/SIMDHelpersT.cpp
View file

@ -265,4 +265,76 @@ TEST_CASE("[Helpers] Interleaved write SIMD vs Scalar")
writeInterleaved<float, false>(leftInput, rightInput, absl::MakeSpan(outputScalar)); writeInterleaved<float, false>(leftInput, rightInput, absl::MakeSpan(outputScalar));
writeInterleaved<float, true>(leftInput, rightInput, absl::MakeSpan(outputSIMD)); writeInterleaved<float, true>(leftInput, rightInput, absl::MakeSpan(outputSIMD));
REQUIRE( outputScalar == outputSIMD ); REQUIRE( outputScalar == outputSIMD );
}
TEST_CASE("[Helpers] Gain, single")
{
std::array<float, 5> input { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
std::array<float, 5> output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
std::array<float, 5> expected { fillValue, fillValue, fillValue, fillValue, fillValue };
applyGain<float, false>(fillValue, input, absl::MakeSpan(output));
REQUIRE( output == expected );
}
TEST_CASE("[Helpers] Gain, single and inplace")
{
std::array<float, 5> buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
std::array<float, 5> expected { fillValue, fillValue, fillValue, fillValue, fillValue };
applyGain<float, false>(fillValue, buffer, absl::MakeSpan(buffer));
REQUIRE( buffer == expected );
}
TEST_CASE("[Helpers] Gain, spans")
{
std::array<float, 5> input { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
std::array<float, 5> gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
std::array<float, 5> output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
std::array<float, 5> expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
applyGain<float, false>(gain, input, absl::MakeSpan(output));
REQUIRE( output == expected );
}
TEST_CASE("[Helpers] Gain, spans and inplace")
{
std::array<float, 5> buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
std::array<float, 5> gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
std::array<float, 5> expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
applyGain<float, false>(gain, buffer, absl::MakeSpan(buffer));
REQUIRE( buffer == expected );
}
TEST_CASE("[Helpers] Gain, single (SIMD)")
{
std::array<float, 5> input { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
std::array<float, 5> output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
std::array<float, 5> expected { fillValue, fillValue, fillValue, fillValue, fillValue };
applyGain<float, true>(fillValue, input, absl::MakeSpan(output));
REQUIRE( output == expected );
}
TEST_CASE("[Helpers] Gain, single and inplace (SIMD)")
{
std::array<float, 5> buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
std::array<float, 5> expected { fillValue, fillValue, fillValue, fillValue, fillValue };
applyGain<float, true>(fillValue, buffer, absl::MakeSpan(buffer));
REQUIRE( buffer == expected );
}
TEST_CASE("[Helpers] Gain, spans (SIMD)")
{
std::array<float, 5> input { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
std::array<float, 5> gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
std::array<float, 5> output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
std::array<float, 5> expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
applyGain<float, true>(gain, input, absl::MakeSpan(output));
REQUIRE( output == expected );
}
TEST_CASE("[Helpers] Gain, spans and inplace (SIMD)")
{
std::array<float, 5> buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
std::array<float, 5> gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
std::array<float, 5> expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
applyGain<float, true>(gain, buffer, absl::MakeSpan(buffer));
REQUIRE( buffer == expected );
} }