Lots of template work
This commit is contained in:
paul
parent
8e5c6650aa
commit
d2e55cf4b2
10 changed files with 463 additions and 55 deletions
|
|
@ -178,4 +178,7 @@ add_executable(bm_mathfuns benchmarks/BM_mathfuns.cpp sources/SIMDSSE.cpp)
|
|||
target_link_libraries(bm_mathfuns benchmark absl::span absl::span)
|
||||
if (UNIX)
|
||||
# target_compile_options(bm_math_loops PRIVATE -fopenmp)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
add_executable(bm_gain benchmarks/BM_gain.cpp sources/SIMDSSE.cpp)
|
||||
target_link_libraries(bm_gain benchmark absl::span)
|
||||
|
|
@ -22,7 +22,17 @@ static void Fill_float(benchmark::State& state) {
|
|||
std::mt19937 gen { rd() };
|
||||
std::uniform_real_distribution<float> dist { 1, 2 };
|
||||
for (auto _ : state) {
|
||||
fill<float, false>(buffer, dist(gen));
|
||||
fill<float, false>(absl::MakeSpan(buffer), dist(gen));
|
||||
}
|
||||
}
|
||||
|
||||
static void Fill_float_unaligned(benchmark::State& state) {
|
||||
Buffer<float> buffer (state.range(0));
|
||||
std::random_device rd { };
|
||||
std::mt19937 gen { rd() };
|
||||
std::uniform_real_distribution<float> dist { 1, 2 };
|
||||
for (auto _ : state) {
|
||||
fill<float, false>(absl::MakeSpan(buffer).subspan(1), dist(gen));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -32,7 +42,17 @@ static void Fill_float_SSE(benchmark::State& state) {
|
|||
std::mt19937 gen { rd() };
|
||||
std::uniform_real_distribution<float> dist { 1, 2 };
|
||||
for (auto _ : state) {
|
||||
fill<float, true>(buffer, dist(gen));
|
||||
fill<float, true>(absl::MakeSpan(buffer), dist(gen));
|
||||
}
|
||||
}
|
||||
|
||||
static void Fill_float_SSE_unaligned(benchmark::State& state) {
|
||||
Buffer<float> buffer (state.range(0));
|
||||
std::random_device rd { };
|
||||
std::mt19937 gen { rd() };
|
||||
std::uniform_real_distribution<float> dist { 1, 2 };
|
||||
for (auto _ : state) {
|
||||
fill<float, true>(absl::MakeSpan(buffer).subspan(1), dist(gen));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -42,12 +62,14 @@ static void Fill_double(benchmark::State& state) {
|
|||
std::mt19937 gen { rd() };
|
||||
std::uniform_real_distribution<double> dist { 1, 2 };
|
||||
for (auto _ : state) {
|
||||
fill<double>(buffer, dist(gen));
|
||||
fill<double>(absl::MakeSpan(buffer), dist(gen));
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK(Dummy)->Range((2<<6), (2<<16));
|
||||
BENCHMARK(Fill_float)->Range((2<<6), (2<<16));
|
||||
BENCHMARK(Fill_float_SSE)->Range((2<<6), (2<<16));
|
||||
BENCHMARK(Fill_float_unaligned)->Range((2<<6), (2<<16));
|
||||
BENCHMARK(Fill_float_SSE_unaligned)->Range((2<<6), (2<<16));
|
||||
BENCHMARK(Fill_double)->Range((2<<6), (2<<16));
|
||||
BENCHMARK_MAIN();
|
||||
120
benchmarks/BM_gain.cpp
Normal file
120
benchmarks/BM_gain.cpp
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
#include <benchmark/benchmark.h>
|
||||
#include <random>
|
||||
#include <numeric>
|
||||
#include <vector>
|
||||
#include <cmath>
|
||||
#include <iostream>
|
||||
#include "../sources/SIMDHelpers.h"
|
||||
|
||||
class GainSingle : public benchmark::Fixture {
|
||||
public:
|
||||
void SetUp(const ::benchmark::State& state) {
|
||||
std::random_device rd { };
|
||||
std::mt19937 gen { rd() };
|
||||
std::uniform_real_distribution<float> dist { 0, 1 };
|
||||
input = std::vector<float>(state.range(0));
|
||||
output = std::vector<float>(state.range(0));
|
||||
gain = dist(gen);
|
||||
std::generate(input.begin(), input.end(), [&]() { return dist(gen); });
|
||||
}
|
||||
|
||||
void TearDown(const ::benchmark::State& state [[maybe_unused]]) {
|
||||
|
||||
}
|
||||
|
||||
float gain;
|
||||
std::vector<float> input;
|
||||
std::vector<float> output;
|
||||
};
|
||||
|
||||
class GainArray : public benchmark::Fixture {
|
||||
public:
|
||||
void SetUp(const ::benchmark::State& state) {
|
||||
std::random_device rd { };
|
||||
std::mt19937 gen { rd() };
|
||||
std::uniform_real_distribution<float> dist { 0, 1 };
|
||||
input = std::vector<float>(state.range(0));
|
||||
output = std::vector<float>(state.range(0));
|
||||
gain = std::vector<float>(state.range(0));
|
||||
std::generate(gain.begin(), gain.end(), [&]() { return dist(gen); });
|
||||
std::generate(input.begin(), input.end(), [&]() { return dist(gen); });
|
||||
}
|
||||
|
||||
void TearDown(const ::benchmark::State& state [[maybe_unused]]) {
|
||||
|
||||
}
|
||||
|
||||
std::vector<float> gain;
|
||||
std::vector<float> input;
|
||||
std::vector<float> output;
|
||||
};
|
||||
|
||||
BENCHMARK_DEFINE_F(GainSingle, Straight)(benchmark::State& state) {
|
||||
for (auto _ : state)
|
||||
{
|
||||
for (int i = 0; i < state.range(0); ++i)
|
||||
output[i] = gain * input[i];
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(GainSingle, Scalar)(benchmark::State& state) {
|
||||
for (auto _ : state)
|
||||
{
|
||||
applyGain<float, false>(gain, input, absl::MakeSpan(output));
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(GainSingle, SIMD)(benchmark::State& state) {
|
||||
for (auto _ : state)
|
||||
{
|
||||
applyGain<float, true>(gain, input, absl::MakeSpan(output));
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(GainArray, Straight)(benchmark::State& state) {
|
||||
for (auto _ : state)
|
||||
{
|
||||
for (int i = 0; i < state.range(0); ++i)
|
||||
output[i] = gain[i] * input[i];
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(GainArray, Scalar)(benchmark::State& state) {
|
||||
for (auto _ : state)
|
||||
{
|
||||
applyGain<float, false>(gain, input, absl::MakeSpan(output));
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(GainArray, SIMD)(benchmark::State& state) {
|
||||
for (auto _ : state)
|
||||
{
|
||||
applyGain<float, true>(gain, input, absl::MakeSpan(output));
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(GainArray, Scalar_Unaligned)(benchmark::State& state) {
|
||||
for (auto _ : state)
|
||||
{
|
||||
applyGain<float, false>(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1));
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(GainArray, SIMD_Unaligned)(benchmark::State& state) {
|
||||
for (auto _ : state)
|
||||
{
|
||||
applyGain<float, true>(absl::MakeSpan(gain).subspan(1), absl::MakeSpan(input).subspan(1), absl::MakeSpan(output).subspan(1));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
BENCHMARK_REGISTER_F(GainSingle, Straight)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
|
||||
BENCHMARK_REGISTER_F(GainSingle, Scalar)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
|
||||
BENCHMARK_REGISTER_F(GainSingle, SIMD)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
|
||||
BENCHMARK_REGISTER_F(GainArray, Straight)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
|
||||
BENCHMARK_REGISTER_F(GainArray, Scalar)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
|
||||
BENCHMARK_REGISTER_F(GainArray, SIMD)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
|
||||
BENCHMARK_REGISTER_F(GainArray, Scalar_Unaligned)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
|
||||
BENCHMARK_REGISTER_F(GainArray, SIMD_Unaligned)->RangeMultiplier(4)->Range(1 << 2, 1 << 12);
|
||||
BENCHMARK_MAIN();
|
||||
|
|
@ -33,7 +33,7 @@ static void Scalar_Unaligned(benchmark::State& state) {
|
|||
Buffer<float> outputRight (state.range(0));
|
||||
std::iota(input.begin(), input.end(), 1.0f);
|
||||
for (auto _ : state) {
|
||||
readInterleaved<float, false>(absl::MakeSpan(input).subspan(1), absl::MakeSpan(outputLeft).subspan(1), absl::MakeSpan(outputRight).subspan(1));
|
||||
readInterleaved<float, false>(absl::MakeSpan(input).subspan(2), absl::MakeSpan(outputLeft), absl::MakeSpan(outputRight));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -43,7 +43,27 @@ static void SSE_Unaligned(benchmark::State& state) {
|
|||
Buffer<float> outputRight (state.range(0));
|
||||
std::iota(input.begin(), input.end(), 1.0f);
|
||||
for (auto _ : state) {
|
||||
readInterleaved<float, true>(absl::MakeSpan(input).subspan(1), absl::MakeSpan(outputLeft).subspan(1), absl::MakeSpan(outputRight).subspan(1));
|
||||
readInterleaved<float, true>(absl::MakeSpan(input).subspan(2), absl::MakeSpan(outputLeft), absl::MakeSpan(outputRight));
|
||||
}
|
||||
}
|
||||
|
||||
static void Scalar_Unaligned_2(benchmark::State& state) {
|
||||
Buffer<float> input (state.range(0) * 2);
|
||||
Buffer<float> outputLeft (state.range(0));
|
||||
Buffer<float> outputRight (state.range(0));
|
||||
std::iota(input.begin(), input.end(), 1.0f);
|
||||
for (auto _ : state) {
|
||||
readInterleaved<float, false>(absl::MakeSpan(input).subspan(2), absl::MakeSpan(outputLeft).subspan(1), absl::MakeSpan(outputRight).subspan(3));
|
||||
}
|
||||
}
|
||||
|
||||
static void SSE_Unaligned_2(benchmark::State& state) {
|
||||
Buffer<float> input (state.range(0) * 2);
|
||||
Buffer<float> outputLeft (state.range(0));
|
||||
Buffer<float> outputRight (state.range(0));
|
||||
std::iota(input.begin(), input.end(), 1.0f);
|
||||
for (auto _ : state) {
|
||||
readInterleaved<float, true>(absl::MakeSpan(input).subspan(2), absl::MakeSpan(outputLeft).subspan(1), absl::MakeSpan(outputRight).subspan(3));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -51,4 +71,6 @@ BENCHMARK(Scalar)->Range((8<<10), (8<<20));
|
|||
BENCHMARK(SSE)->Range((8<<10), (8<<20));
|
||||
BENCHMARK(Scalar_Unaligned)->Range((8<<10), (8<<20));
|
||||
BENCHMARK(SSE_Unaligned)->Range((8<<10), (8<<20));
|
||||
BENCHMARK(Scalar_Unaligned_2)->Range((8<<10), (8<<20));
|
||||
BENCHMARK(SSE_Unaligned_2)->Range((8<<10), (8<<20));
|
||||
BENCHMARK_MAIN();
|
||||
|
|
@ -36,7 +36,7 @@ static void Unaligned_Interleaved_Write(benchmark::State& state) {
|
|||
std::iota(inputLeft.begin(), inputLeft.end(), 1.0f);
|
||||
std::iota(inputRight.begin(), inputRight.end(), 1.0f);
|
||||
for (auto _ : state) {
|
||||
writeInterleaved<float, false>(absl::MakeSpan(inputLeft).subspan(1) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(1));
|
||||
writeInterleaved<float, false>(absl::MakeSpan(inputLeft).subspan(1) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(2));
|
||||
benchmark::DoNotOptimize(output);
|
||||
}
|
||||
}
|
||||
|
|
@ -48,7 +48,31 @@ static void Unaligned_Interleaved_Write_SSE(benchmark::State& state) {
|
|||
std::iota(inputLeft.begin(), inputLeft.end(), 1.0f);
|
||||
std::iota(inputRight.begin(), inputRight.end(), 1.0f);
|
||||
for (auto _ : state) {
|
||||
writeInterleaved<float, true>(absl::MakeSpan(inputLeft).subspan(1) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(1));
|
||||
writeInterleaved<float, true>(absl::MakeSpan(inputLeft).subspan(1) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(2));
|
||||
benchmark::DoNotOptimize(output);
|
||||
}
|
||||
}
|
||||
|
||||
static void Unaligned_Interleaved_Write_2(benchmark::State& state) {
|
||||
Buffer<float> inputLeft (state.range(0));
|
||||
Buffer<float> inputRight (state.range(0));
|
||||
Buffer<float> output (state.range(0) * 2);
|
||||
std::iota(inputLeft.begin(), inputLeft.end(), 1.0f);
|
||||
std::iota(inputRight.begin(), inputRight.end(), 1.0f);
|
||||
for (auto _ : state) {
|
||||
writeInterleaved<float, false>(absl::MakeSpan(inputLeft) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(2));
|
||||
benchmark::DoNotOptimize(output);
|
||||
}
|
||||
}
|
||||
|
||||
static void Unaligned_Interleaved_Write_SSE_2(benchmark::State& state) {
|
||||
Buffer<float> inputLeft (state.range(0));
|
||||
Buffer<float> inputRight (state.range(0));
|
||||
Buffer<float> output (state.range(0) * 2);
|
||||
std::iota(inputLeft.begin(), inputLeft.end(), 1.0f);
|
||||
std::iota(inputRight.begin(), inputRight.end(), 1.0f);
|
||||
for (auto _ : state) {
|
||||
writeInterleaved<float, true>(absl::MakeSpan(inputLeft) , absl::MakeSpan(inputRight).subspan(1), absl::MakeSpan(output).subspan(2));
|
||||
benchmark::DoNotOptimize(output);
|
||||
}
|
||||
}
|
||||
|
|
@ -57,4 +81,6 @@ BENCHMARK(Interleaved_Write)->Range((8<<10), (8<<20));
|
|||
BENCHMARK(Interleaved_Write_SSE)->Range((8<<10), (8<<20));
|
||||
BENCHMARK(Unaligned_Interleaved_Write)->Range((8<<10), (8<<20));
|
||||
BENCHMARK(Unaligned_Interleaved_Write_SSE)->Range((8<<10), (8<<20));
|
||||
BENCHMARK(Unaligned_Interleaved_Write_2)->Range((8<<10), (8<<20));
|
||||
BENCHMARK(Unaligned_Interleaved_Write_SSE_2)->Range((8<<10), (8<<20));
|
||||
BENCHMARK_MAIN();
|
||||
|
|
@ -26,6 +26,8 @@ namespace SIMDConfig
|
|||
constexpr bool writeInterleaved { true };
|
||||
constexpr bool readInterleaved { true };
|
||||
constexpr bool fill { false };
|
||||
constexpr bool gain { false };
|
||||
constexpr bool mathfuns { false };
|
||||
#if USE_SIMD
|
||||
constexpr bool useSIMD { true };
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -56,4 +56,16 @@ template<>
|
|||
void cos<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept
|
||||
{
|
||||
cos<float, false>(input, output);
|
||||
}
|
||||
|
||||
template<>
|
||||
void applyGain<float, true>(float gain, absl::Span<const float> input, absl::Span<float> output) noexcept
|
||||
{
|
||||
applyGain<float, false>(gain, input, output);
|
||||
}
|
||||
|
||||
template<>
|
||||
void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept
|
||||
{
|
||||
applyGain<float, false>(gain, input, output);
|
||||
}
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
|
||||
#include "absl/types/span.h"
|
||||
#include "Globals.h"
|
||||
#include <absl/types/span.h>
|
||||
#include "Helpers.h"
|
||||
#include <cmath>
|
||||
|
||||
|
|
@ -52,7 +51,7 @@ void fill(absl::Span<T> output, T value) noexcept
|
|||
template<>
|
||||
void fill<float, true>(absl::Span<float> output, float value) noexcept;
|
||||
|
||||
template<class Type, bool SIMD=SIMDConfig::useSIMD>
|
||||
template<class Type, bool SIMD=SIMDConfig::mathfuns>
|
||||
void exp(absl::Span<const Type> input, absl::Span<Type> output) noexcept
|
||||
{
|
||||
ASSERT(output.size() >= input.size());
|
||||
|
|
@ -64,7 +63,7 @@ void exp(absl::Span<const Type> input, absl::Span<Type> output) noexcept
|
|||
template<>
|
||||
void exp<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
|
||||
|
||||
template<class Type, bool SIMD=SIMDConfig::useSIMD>
|
||||
template<class Type, bool SIMD=SIMDConfig::mathfuns>
|
||||
void log(absl::Span<const Type> input, absl::Span<Type> output) noexcept
|
||||
{
|
||||
ASSERT(output.size() >= input.size());
|
||||
|
|
@ -76,7 +75,7 @@ void log(absl::Span<const Type> input, absl::Span<Type> output) noexcept
|
|||
template<>
|
||||
void log<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
|
||||
|
||||
template<class Type, bool SIMD=SIMDConfig::useSIMD>
|
||||
template<class Type, bool SIMD=SIMDConfig::mathfuns>
|
||||
void sin(absl::Span<const Type> input, absl::Span<Type> output) noexcept
|
||||
{
|
||||
ASSERT(output.size() >= input.size());
|
||||
|
|
@ -88,7 +87,7 @@ void sin(absl::Span<const Type> input, absl::Span<Type> output) noexcept
|
|||
template<>
|
||||
void sin<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept;
|
||||
|
||||
template<class Type, bool SIMD=SIMDConfig::useSIMD>
|
||||
template<class Type, bool SIMD=SIMDConfig::mathfuns>
|
||||
void cos(absl::Span<const Type> input, absl::Span<Type> output) noexcept
|
||||
{
|
||||
ASSERT(output.size() >= input.size());
|
||||
|
|
@ -109,9 +108,48 @@ void linearRamp(absl::Span<T> output, T start, T end);
|
|||
template<class T, bool SIMD=SIMDConfig::useSIMD>
|
||||
void exponentialRamp(absl::Span<T> output, T start, T end);
|
||||
|
||||
template<class T, bool SIMD=SIMDConfig::useSIMD>
|
||||
void applyGain(T gain, absl::Span<T> output);
|
||||
template<class T, bool SIMD=SIMDConfig::gain>
|
||||
void applyGain(T gain, absl::Span<const T> input, absl::Span<T> output) noexcept
|
||||
{
|
||||
ASSERT(input.size() <= output.size());
|
||||
auto* in = input.begin();
|
||||
auto* out = output.begin();
|
||||
auto* sentinel = out + std::min(output.size(), input.size());
|
||||
while (out < sentinel)
|
||||
{
|
||||
*out++ = gain * (*in++);
|
||||
}
|
||||
}
|
||||
|
||||
template<class T, bool SIMD=SIMDConfig::useSIMD>
|
||||
void applyGain(absl::Span<const T> gain, absl::Span<T> output);
|
||||
template<class T, bool SIMD=SIMDConfig::gain>
|
||||
void applyGain(absl::Span<const T> gain, absl::Span<const T> input, absl::Span<T> output) noexcept
|
||||
{
|
||||
ASSERT(gain.size() == input.size());
|
||||
ASSERT(input.size() <= output.size());
|
||||
auto* in = input.begin();
|
||||
auto* g = gain.begin();
|
||||
auto* out = output.begin();
|
||||
auto* sentinel = out + std::min(gain.size(), std::min(output.size(), input.size()));
|
||||
while (out < sentinel)
|
||||
{
|
||||
*out++ = (*g++) * (*in++);
|
||||
}
|
||||
}
|
||||
|
||||
template<class T, bool SIMD=SIMDConfig::gain>
|
||||
void applyGain(T gain, absl::Span<T> output) noexcept
|
||||
{
|
||||
applyGain<T, SIMD>(gain, output, output);
|
||||
}
|
||||
|
||||
template<class T, bool SIMD=SIMDConfig::gain>
|
||||
void applyGain(absl::Span<const T> gain, absl::Span<T> output) noexcept
|
||||
{
|
||||
applyGain<T, SIMD>(gain, output, output);
|
||||
}
|
||||
|
||||
template<>
|
||||
void applyGain<float, true>(float gain, absl::Span<const float> input, absl::Span<float> output) noexcept;
|
||||
|
||||
template<>
|
||||
void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept;
|
||||
|
|
@ -3,11 +3,42 @@
|
|||
#include "x86intrin.h"
|
||||
#include "mathfuns/sse_mathfun.h"
|
||||
|
||||
constexpr int TypeAlignment { 4 };
|
||||
constexpr uintptr_t TypeAlignment { 4 };
|
||||
constexpr uintptr_t TypeAlignmentMask { TypeAlignment - 1 };
|
||||
using Type = float;
|
||||
constexpr uintptr_t ByteAlignment { TypeAlignment * sizeof(Type) };
|
||||
constexpr uintptr_t ByteAlignmentMask { ByteAlignment - 1 };
|
||||
|
||||
|
||||
struct AlignmentSentinels { float* nextAligned; float* lastAligned; };
|
||||
|
||||
float* nextAligned(const float* ptr)
|
||||
{
|
||||
return reinterpret_cast<float*>( (reinterpret_cast<uintptr_t>(ptr) + ByteAlignmentMask) & (~ByteAlignmentMask) );
|
||||
}
|
||||
|
||||
float* prevAligned(const float* ptr)
|
||||
{
|
||||
return reinterpret_cast<float*>( reinterpret_cast<uintptr_t>(ptr) & (~ByteAlignmentMask) );
|
||||
}
|
||||
|
||||
bool unaligned(const float* ptr)
|
||||
{
|
||||
return (reinterpret_cast<uintptr_t>(ptr) & ByteAlignmentMask) != 0;
|
||||
}
|
||||
|
||||
bool unaligned(const float* ptr1, const float* ptr2)
|
||||
{
|
||||
return unaligned(ptr1) || unaligned(ptr2);
|
||||
}
|
||||
|
||||
bool unaligned(const float* ptr1, const float* ptr2, const float* ptr3)
|
||||
{
|
||||
return unaligned(ptr1) || unaligned(ptr2) || unaligned(ptr3);
|
||||
}
|
||||
|
||||
template<>
|
||||
void readInterleaved<Type, true>(absl::Span<const Type> input, absl::Span<Type> outputLeft, absl::Span<Type> outputRight) noexcept
|
||||
void readInterleaved<float, true>(absl::Span<const float> input, absl::Span<float> outputLeft, absl::Span<float> outputRight) noexcept
|
||||
{
|
||||
// The size of the outputs is not big enough for the input...
|
||||
ASSERT(outputLeft.size() >= input.size() / 2);
|
||||
|
|
@ -18,14 +49,21 @@ void readInterleaved<Type, true>(absl::Span<const Type> input, absl::Span<Type>
|
|||
auto* in = input.begin();
|
||||
auto* lOut = outputLeft.begin();
|
||||
auto* rOut = outputRight.begin();
|
||||
const int unalignedEnd = input.size() & (2 * TypeAlignment - 1);
|
||||
const int lastAligned = input.size() - unalignedEnd;
|
||||
auto* inputSentinel = in + lastAligned;
|
||||
while (in < inputSentinel && lOut < outputLeft.end() && rOut < outputRight.end())
|
||||
|
||||
const auto size = std::min(input.size(), std::min(outputLeft.size() * 2, outputRight.size() * 2 ));
|
||||
const auto* lastAligned = prevAligned(input.begin() + size - TypeAlignment);
|
||||
|
||||
while (unaligned(in, lOut, rOut) && in < lastAligned)
|
||||
{
|
||||
auto register0 = _mm_loadu_ps(in);
|
||||
*lOut++ = *in++;
|
||||
*rOut++ = *in++;
|
||||
}
|
||||
|
||||
while (in < lastAligned )
|
||||
{
|
||||
auto register0 = _mm_load_ps(in);
|
||||
in += TypeAlignment;
|
||||
auto register1 = _mm_loadu_ps(in);
|
||||
auto register1 = _mm_load_ps(in);
|
||||
in += TypeAlignment;
|
||||
auto register2 = register0;
|
||||
// register 2 holds the copy of register 0 that is going to get erased by the first operation
|
||||
|
|
@ -33,14 +71,13 @@ void readInterleaved<Type, true>(absl::Span<const Type> input, absl::Span<Type>
|
|||
// "take 0 from a, take 2 from a, take 0 from b, take 2 from b"
|
||||
register0 = _mm_shuffle_ps(register0, register1, 0b10001000);
|
||||
register1 = _mm_shuffle_ps(register2, register1, 0b11011101);
|
||||
_mm_storeu_ps(lOut, register0);
|
||||
_mm_storeu_ps(rOut, register1);
|
||||
_mm_store_ps(lOut, register0);
|
||||
_mm_store_ps(rOut, register1);
|
||||
lOut += TypeAlignment;
|
||||
rOut += TypeAlignment;
|
||||
}
|
||||
|
||||
inputSentinel = input.end() - 1;
|
||||
while (in < inputSentinel && lOut < outputLeft.end() && rOut < outputRight.end())
|
||||
while (in < input.end() - 1)
|
||||
{
|
||||
*lOut++ = *in++;
|
||||
*rOut++ = *in++;
|
||||
|
|
@ -48,7 +85,7 @@ void readInterleaved<Type, true>(absl::Span<const Type> input, absl::Span<Type>
|
|||
}
|
||||
|
||||
template<>
|
||||
void writeInterleaved<Type, true>(absl::Span<const Type> inputLeft, absl::Span<const Type> inputRight, absl::Span<Type> output) noexcept
|
||||
void writeInterleaved<float, true>(absl::Span<const float> inputLeft, absl::Span<const float> inputRight, absl::Span<float> output) noexcept
|
||||
{
|
||||
// The size of the output is not big enough for the inputs...
|
||||
ASSERT(inputLeft.size() <= output.size() / 2);
|
||||
|
|
@ -58,48 +95,53 @@ void writeInterleaved<Type, true>(absl::Span<const Type> inputLeft, absl::Span<c
|
|||
auto* rIn = inputRight.begin();
|
||||
auto* out = output.begin();
|
||||
|
||||
const int residualLeft = inputLeft.size() & (TypeAlignment - 1);
|
||||
const int residualRight = inputRight.size() & (TypeAlignment - 1);
|
||||
const auto* leftSentinel = lIn + inputLeft.size() - residualLeft;
|
||||
const auto* rightSentinel = rIn + inputRight.size() - residualRight;
|
||||
const auto* outputSentinel = output.end() - 1;
|
||||
const auto size = std::min(output.size(), std::min(inputLeft.size(), inputRight.size()) * 2);
|
||||
const auto* lastAligned = prevAligned(output.begin() + size - TypeAlignment);
|
||||
|
||||
while (lIn < leftSentinel && rIn < rightSentinel && out < outputSentinel)
|
||||
while (unaligned(out, rIn, lIn) && out < lastAligned)
|
||||
{
|
||||
const auto lInRegister = _mm_loadu_ps(lIn);
|
||||
const auto rInRegister = _mm_loadu_ps(rIn);
|
||||
*out++ = *lIn++;
|
||||
*out++ = *rIn++;
|
||||
}
|
||||
|
||||
while (out < lastAligned)
|
||||
{
|
||||
const auto lInRegister = _mm_load_ps(lIn);
|
||||
const auto rInRegister = _mm_load_ps(rIn);
|
||||
|
||||
const auto outRegister1 = _mm_unpacklo_ps(lInRegister, rInRegister);
|
||||
_mm_storeu_ps(out, outRegister1);
|
||||
_mm_store_ps(out, outRegister1);
|
||||
out += TypeAlignment;
|
||||
|
||||
const auto outRegister2 = _mm_unpackhi_ps(lInRegister, rInRegister);
|
||||
_mm_storeu_ps(out, outRegister2);
|
||||
_mm_store_ps(out, outRegister2);
|
||||
out += TypeAlignment;
|
||||
|
||||
lIn += TypeAlignment;
|
||||
rIn += TypeAlignment;
|
||||
}
|
||||
|
||||
while (lIn < inputLeft.end() && rIn < inputRight.end() && out < outputSentinel)
|
||||
while (out < output.end() - 1)
|
||||
{
|
||||
|
||||
*out++ = *lIn++;
|
||||
*out++ = *rIn++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template<>
|
||||
void fill<float, true>(absl::Span<float> output, float value) noexcept
|
||||
{
|
||||
const auto mmValue = _mm_set_ps1(value);
|
||||
auto* out = output.begin();
|
||||
const int residual = output.size() & (TypeAlignment - 1);
|
||||
const auto* sentinel = output.end() - residual;
|
||||
const auto* lastAligned = prevAligned(output.end());
|
||||
|
||||
while (out < sentinel) // we should only need to test a single channel
|
||||
while (unaligned(out) && out < lastAligned)
|
||||
*out++ = value;
|
||||
|
||||
while (out < lastAligned) // we should only need to test a single channel
|
||||
{
|
||||
_mm_storeu_ps(out, mmValue);
|
||||
_mm_store_ps(out, mmValue);
|
||||
out += TypeAlignment;
|
||||
}
|
||||
|
||||
|
|
@ -117,8 +159,8 @@ void exp<float, true>(absl::Span<const float> input, absl::Span<float> output) n
|
|||
while (in < sentinel)
|
||||
{
|
||||
_mm_storeu_ps(out, exp_ps(_mm_loadu_ps(in)));
|
||||
out += 4;
|
||||
in += 4;
|
||||
out += TypeAlignment;
|
||||
in += TypeAlignment;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -132,8 +174,8 @@ void cos<float, true>(absl::Span<const float> input, absl::Span<float> output) n
|
|||
while (in < sentinel)
|
||||
{
|
||||
_mm_storeu_ps(out, cos_ps(_mm_loadu_ps(in)));
|
||||
out += 4;
|
||||
in += 4;
|
||||
out += TypeAlignment;
|
||||
in += TypeAlignment;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -147,11 +189,13 @@ void log<float, true>(absl::Span<const float> input, absl::Span<float> output) n
|
|||
while (in < sentinel)
|
||||
{
|
||||
_mm_storeu_ps(out, log_ps(_mm_loadu_ps(in)));
|
||||
out += 4;
|
||||
in += 4;
|
||||
out += TypeAlignment;
|
||||
in += TypeAlignment;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
template<>
|
||||
void sin<float, true>(absl::Span<const float> input, absl::Span<float> output) noexcept
|
||||
{
|
||||
|
|
@ -162,7 +206,54 @@ void sin<float, true>(absl::Span<const float> input, absl::Span<float> output) n
|
|||
while (in < sentinel)
|
||||
{
|
||||
_mm_storeu_ps(out, sin_ps(_mm_loadu_ps(in)));
|
||||
out += 4;
|
||||
in += 4;
|
||||
out += TypeAlignment;
|
||||
in += TypeAlignment;
|
||||
}
|
||||
}
|
||||
|
||||
template<>
|
||||
void applyGain<float, true>(float gain, absl::Span<const float> input, absl::Span<float> output) noexcept
|
||||
{
|
||||
auto* in = input.begin();
|
||||
auto* out = output.begin();
|
||||
const auto size = std::min(output.size(), input.size());
|
||||
const auto* lastAligned = prevAligned(output.begin() + size);
|
||||
const auto mmGain = _mm_set_ps1(gain);
|
||||
|
||||
while (unaligned(out, in) && out < lastAligned)
|
||||
*out++ = gain * (*in++);
|
||||
|
||||
while (out < lastAligned)
|
||||
{
|
||||
_mm_store_ps(out, _mm_mul_ps(mmGain, _mm_load_ps(in)));
|
||||
in += TypeAlignment;
|
||||
out += TypeAlignment;
|
||||
}
|
||||
|
||||
while (out < output.end())
|
||||
*out++ = gain * (*in++);
|
||||
}
|
||||
|
||||
template<>
|
||||
void applyGain<float, true>(absl::Span<const float> gain, absl::Span<const float> input, absl::Span<float> output) noexcept
|
||||
{
|
||||
auto* in = input.begin();
|
||||
auto* out = output.begin();
|
||||
auto* g = gain.begin();
|
||||
const auto size = std::min(output.size(), std::min(input.size(), gain.size()));
|
||||
const auto* lastAligned = prevAligned(output.begin() + size);
|
||||
|
||||
while (unaligned(out, in, g) && out < lastAligned)
|
||||
*out++ = (*g++) * (*in++);
|
||||
|
||||
while (out < lastAligned)
|
||||
{
|
||||
_mm_store_ps(out, _mm_mul_ps(_mm_load_ps(g), _mm_load_ps(in)));
|
||||
g += TypeAlignment;
|
||||
in += TypeAlignment;
|
||||
out += TypeAlignment;
|
||||
}
|
||||
|
||||
while (out < output.end())
|
||||
*out++ = (*g++) * (*in++);
|
||||
}
|
||||
|
|
@ -265,4 +265,76 @@ TEST_CASE("[Helpers] Interleaved write SIMD vs Scalar")
|
|||
writeInterleaved<float, false>(leftInput, rightInput, absl::MakeSpan(outputScalar));
|
||||
writeInterleaved<float, true>(leftInput, rightInput, absl::MakeSpan(outputSIMD));
|
||||
REQUIRE( outputScalar == outputSIMD );
|
||||
}
|
||||
|
||||
TEST_CASE("[Helpers] Gain, single")
|
||||
{
|
||||
std::array<float, 5> input { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
|
||||
std::array<float, 5> output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
|
||||
std::array<float, 5> expected { fillValue, fillValue, fillValue, fillValue, fillValue };
|
||||
applyGain<float, false>(fillValue, input, absl::MakeSpan(output));
|
||||
REQUIRE( output == expected );
|
||||
}
|
||||
|
||||
TEST_CASE("[Helpers] Gain, single and inplace")
|
||||
{
|
||||
std::array<float, 5> buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
|
||||
std::array<float, 5> expected { fillValue, fillValue, fillValue, fillValue, fillValue };
|
||||
applyGain<float, false>(fillValue, buffer, absl::MakeSpan(buffer));
|
||||
REQUIRE( buffer == expected );
|
||||
}
|
||||
|
||||
TEST_CASE("[Helpers] Gain, spans")
|
||||
{
|
||||
std::array<float, 5> input { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
|
||||
std::array<float, 5> gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
|
||||
std::array<float, 5> output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
|
||||
std::array<float, 5> expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
|
||||
applyGain<float, false>(gain, input, absl::MakeSpan(output));
|
||||
REQUIRE( output == expected );
|
||||
}
|
||||
|
||||
TEST_CASE("[Helpers] Gain, spans and inplace")
|
||||
{
|
||||
std::array<float, 5> buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
|
||||
std::array<float, 5> gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
|
||||
std::array<float, 5> expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
|
||||
applyGain<float, false>(gain, buffer, absl::MakeSpan(buffer));
|
||||
REQUIRE( buffer == expected );
|
||||
}
|
||||
|
||||
TEST_CASE("[Helpers] Gain, single (SIMD)")
|
||||
{
|
||||
std::array<float, 5> input { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
|
||||
std::array<float, 5> output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
|
||||
std::array<float, 5> expected { fillValue, fillValue, fillValue, fillValue, fillValue };
|
||||
applyGain<float, true>(fillValue, input, absl::MakeSpan(output));
|
||||
REQUIRE( output == expected );
|
||||
}
|
||||
|
||||
TEST_CASE("[Helpers] Gain, single and inplace (SIMD)")
|
||||
{
|
||||
std::array<float, 5> buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
|
||||
std::array<float, 5> expected { fillValue, fillValue, fillValue, fillValue, fillValue };
|
||||
applyGain<float, true>(fillValue, buffer, absl::MakeSpan(buffer));
|
||||
REQUIRE( buffer == expected );
|
||||
}
|
||||
|
||||
TEST_CASE("[Helpers] Gain, spans (SIMD)")
|
||||
{
|
||||
std::array<float, 5> input { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
|
||||
std::array<float, 5> gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
|
||||
std::array<float, 5> output { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
|
||||
std::array<float, 5> expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
|
||||
applyGain<float, true>(gain, input, absl::MakeSpan(output));
|
||||
REQUIRE( output == expected );
|
||||
}
|
||||
|
||||
TEST_CASE("[Helpers] Gain, spans and inplace (SIMD)")
|
||||
{
|
||||
std::array<float, 5> buffer { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
|
||||
std::array<float, 5> gain { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
|
||||
std::array<float, 5> expected { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f };
|
||||
applyGain<float, true>(gain, buffer, absl::MakeSpan(buffer));
|
||||
REQUIRE( buffer == expected );
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue