dolphin/Source/Core/VideoCommon/CPUCullImpl.h

// Copyright 2022 Dolphin Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#if defined(USE_FMA)
#define VECTOR_NAMESPACE CPUCull_FMA
#elif defined(USE_AVX)
#define VECTOR_NAMESPACE CPUCull_AVX
#elif defined(USE_SSE41)
#define VECTOR_NAMESPACE CPUCull_SSE41
#elif defined(USE_SSE3)
#define VECTOR_NAMESPACE CPUCull_SSE3
#elif defined(USE_SSE)
#define VECTOR_NAMESPACE CPUCull_SSE
#elif defined(USE_NEON)
#define VECTOR_NAMESPACE CPUCull_NEON
#elif defined(NO_SIMD)
#define VECTOR_NAMESPACE CPUCull_Scalar
#else
#error This file is meant to be used by CPUCull.cpp only!
#endif

#if defined(__GNUC__) && defined(USE_FMA) && !(defined(__AVX__) && defined(__FMA__))
#define ATTR_TARGET __attribute__((target("avx,fma")))
#elif defined(__GNUC__) && defined(USE_AVX) && !defined(__AVX__)
#define ATTR_TARGET __attribute__((target("avx")))
#elif defined(__GNUC__) && defined(USE_SSE41) && !defined(__SSE4_1__)
#define ATTR_TARGET __attribute__((target("sse4.1")))
#elif defined(__GNUC__) && defined(USE_SSE3) && !defined(__SSE3__)
#define ATTR_TARGET __attribute__((target("sse3")))
#else
#define ATTR_TARGET
#endif

namespace VECTOR_NAMESPACE
{
#if defined(USE_SSE)
typedef __m128 Vector;
#elif defined(USE_NEON)
typedef float32x4_t Vector;
#else
struct alignas(16) Vector
{
  float x, y, z, w;
};
#endif
static_assert(sizeof(Vector) == 16);

#ifdef USE_NEON
ATTR_TARGET DOLPHIN_FORCE_INLINE static Vector vsetr_f32(float x, float y, float z, float w)
{
  float tmp[4] = {x, y, z, w};
  return vld1q_f32(tmp);
}
ATTR_TARGET DOLPHIN_FORCE_INLINE static void vuzp12q_f32(Vector& a, Vector& b)
{
  Vector tmp = vuzp2q_f32(a, b);
  a = vuzp1q_f32(a, b);
  b = tmp;
}
#endif
#ifdef USE_SSE
template <int i>
ATTR_TARGET DOLPHIN_FORCE_INLINE static Vector vector_broadcast(Vector v)
{
  return _mm_shuffle_ps(v, v, _MM_SHUFFLE(i, i, i, i));
}
#endif
#ifdef USE_AVX
template <int i>
ATTR_TARGET DOLPHIN_FORCE_INLINE static __m256 vector_broadcast(__m256 v)
{
  return _mm256_shuffle_ps(v, v, _MM_SHUFFLE(i, i, i, i));
}
#endif

#ifdef USE_AVX
ATTR_TARGET DOLPHIN_FORCE_INLINE static void TransposeYMM(__m256& o0, __m256& o1,  //
                                                          __m256& o2, __m256& o3)
{
  __m256d tmp0 = _mm256_castps_pd(_mm256_unpacklo_ps(o0, o1));
  __m256d tmp1 = _mm256_castps_pd(_mm256_unpacklo_ps(o2, o3));
  __m256d tmp2 = _mm256_castps_pd(_mm256_unpackhi_ps(o0, o1));
  __m256d tmp3 = _mm256_castps_pd(_mm256_unpackhi_ps(o2, o3));
  o0 = _mm256_castpd_ps(_mm256_unpacklo_pd(tmp0, tmp1));
  o1 = _mm256_castpd_ps(_mm256_unpackhi_pd(tmp0, tmp1));
  o2 = _mm256_castpd_ps(_mm256_unpacklo_pd(tmp2, tmp3));
  o3 = _mm256_castpd_ps(_mm256_unpackhi_pd(tmp2, tmp3));
}

ATTR_TARGET DOLPHIN_FORCE_INLINE static void LoadTransposedYMM(const void* source, __m256& o0,
                                                               __m256& o1, __m256& o2, __m256& o3)
{
  const Vector* vsource = static_cast<const Vector*>(source);
  o0 = _mm256_broadcast_ps(&vsource[0]);
  o1 = _mm256_broadcast_ps(&vsource[1]);
  o2 = _mm256_broadcast_ps(&vsource[2]);
  o3 = _mm256_broadcast_ps(&vsource[3]);
  TransposeYMM(o0, o1, o2, o3);
}

ATTR_TARGET DOLPHIN_FORCE_INLINE static void LoadPosYMM(const void* sourcel, const void* sourceh,
                                                        __m256& o0, __m256& o1, __m256& o2)
{
  const Vector* vsourcel = static_cast<const Vector*>(sourcel);
  const Vector* vsourceh = static_cast<const Vector*>(sourceh);
  o0 = _mm256_insertf128_ps(_mm256_castps128_ps256(vsourcel[0]), vsourceh[0], 1);
  o1 = _mm256_insertf128_ps(_mm256_castps128_ps256(vsourcel[1]), vsourceh[1], 1);
  o2 = _mm256_insertf128_ps(_mm256_castps128_ps256(vsourcel[2]), vsourceh[2], 1);
}

ATTR_TARGET DOLPHIN_FORCE_INLINE static void
LoadTransposedPosYMM(const void* source, __m256& o0, __m256& o1, __m256& o2, __m256& o3)
{
  const Vector* vsource = static_cast<const Vector*>(source);
  o0 = _mm256_broadcast_ps(&vsource[0]);
  o1 = _mm256_broadcast_ps(&vsource[1]);
  o2 = _mm256_broadcast_ps(&vsource[2]);
  o3 = _mm256_setr_ps(0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 1.0f);
  TransposeYMM(o0, o1, o2, o3);
}

ATTR_TARGET DOLPHIN_FORCE_INLINE static __m256 ApplyMatrixYMM(__m256 v, __m256 m0, __m256 m1,
                                                              __m256 m2, __m256 m3)
{
  __m256 output = _mm256_mul_ps(vector_broadcast<0>(v), m0);
#ifdef USE_FMA
  output = _mm256_fmadd_ps(vector_broadcast<1>(v), m1, output);
  output = _mm256_fmadd_ps(vector_broadcast<2>(v), m2, output);
  output = _mm256_fmadd_ps(vector_broadcast<3>(v), m3, output);
#else
  output = _mm256_add_ps(output, _mm256_mul_ps(vector_broadcast<1>(v), m1));
  output = _mm256_add_ps(output, _mm256_mul_ps(vector_broadcast<2>(v), m2));
  output = _mm256_add_ps(output, _mm256_mul_ps(vector_broadcast<3>(v), m3));
#endif
  return output;
}

ATTR_TARGET DOLPHIN_FORCE_INLINE static __m256
TransformVertexNoTransposeYMM(__m256 vertex, __m256 pos0, __m256 pos1, __m256 pos2,  //
                              __m256 proj0, __m256 proj1, __m256 proj2, __m256 proj3)
{
  __m256 mul0 = _mm256_mul_ps(vertex, pos0);
  __m256 mul1 = _mm256_mul_ps(vertex, pos1);
  __m256 mul2 = _mm256_mul_ps(vertex, pos2);
  __m256 mul3 = _mm256_setr_ps(0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 1.0f);
  __m256 output = _mm256_hadd_ps(_mm256_hadd_ps(mul0, mul1), _mm256_hadd_ps(mul2, mul3));
  return ApplyMatrixYMM(output, proj0, proj1, proj2, proj3);
}

template <bool PositionHas3Elems>
ATTR_TARGET DOLPHIN_FORCE_INLINE static __m256
TransformVertexYMM(__m256 vertex, __m256 pos0, __m256 pos1, __m256 pos2, __m256 pos3,  //
                   __m256 proj0, __m256 proj1, __m256 proj2, __m256 proj3)
{
  __m256 output = pos3;  // vertex.w is always 1.0
#ifdef USE_FMA
  output = _mm256_fmadd_ps(vector_broadcast<0>(vertex), pos0, output);
  output = _mm256_fmadd_ps(vector_broadcast<1>(vertex), pos1, output);
  if constexpr (PositionHas3Elems)
    output = _mm256_fmadd_ps(vector_broadcast<2>(vertex), pos2, output);
#else
  output = _mm256_add_ps(output, _mm256_mul_ps(vector_broadcast<0>(vertex), pos0));
  output = _mm256_add_ps(output, _mm256_mul_ps(vector_broadcast<1>(vertex), pos1));
  if constexpr (PositionHas3Elems)
    output = _mm256_add_ps(output, _mm256_mul_ps(vector_broadcast<2>(vertex), pos2));
#endif
  return ApplyMatrixYMM(output, proj0, proj1, proj2, proj3);
}

template <bool PositionHas3Elems, bool PerVertexPosMtx>
ATTR_TARGET DOLPHIN_FORCE_INLINE static __m256
LoadTransform2Vertices(const u8* v0data, const u8* v1data,                  //
                       __m256 pos0, __m256 pos1, __m256 pos2, __m256 pos3,  //
                       __m256 proj0, __m256 proj1, __m256 proj2, __m256 proj3)
{
  __m256 v01;
  if constexpr (PerVertexPosMtx)
  {
    // Vertex data layout always starts with posmtx data if available, then position data
    // Convenient for us, that means offsets are always fixed
    u32 v0idx = v0data[0] & 0x3f;
    u32 v1idx = v1data[0] & 0x3f;
    v0data += sizeof(u32);
    v1data += sizeof(u32);

    const float* v0fdata = reinterpret_cast<const float*>(v0data);
    const float* v1fdata = reinterpret_cast<const float*>(v1data);

    LoadPosYMM(&xfmem.posMatrices[v0idx * 4], &xfmem.posMatrices[v1idx * 4], pos0, pos1, pos2);

    if constexpr (PositionHas3Elems)
    {
      __m256 base = _mm256_set1_ps(1.0f);
      v01 = _mm256_blend_ps(_mm256_loadu2_m128(v1fdata, v0fdata), base, 0x88);
    }
    else
    {
      __m256 base = _mm256_unpacklo_ps(_mm256_setzero_ps(), _mm256_set1_ps(1.0f));
      __m256 v1 = _mm256_castpd_ps(_mm256_broadcast_sd(reinterpret_cast<const double*>(v1data)));
      __m128 v0 = _mm_loadl_pi(_mm_setzero_ps(), reinterpret_cast<const __m64*>(v0data));
      v01 = _mm256_blend_ps(_mm256_castps128_ps256(v0), v1, 0x30);
      v01 = _mm256_blend_ps(v01, base, 0xcc);
    }

    v01 = TransformVertexNoTransposeYMM(v01, pos0, pos1, pos2, proj0, proj1, proj2, proj3);
  }
  else
  {
    const float* v0fdata = reinterpret_cast<const float*>(v0data);
    const float* v1fdata = reinterpret_cast<const float*>(v1data);
    if constexpr (PositionHas3Elems)
    {
      v01 = _mm256_loadu2_m128(v1fdata, v0fdata);
    }
    else
    {
      __m256 v1 = _mm256_castpd_ps(_mm256_broadcast_sd(reinterpret_cast<const double*>(v1data)));
      __m128 v0 = _mm_loadl_pi(_mm_setzero_ps(), reinterpret_cast<const __m64*>(v0data));
      v01 = _mm256_blend_ps(_mm256_castps128_ps256(v0), v1, 0x30);
    }

#ifdef __clang__
    // Clang's optimizer is dumb, yay
    // It sees TransformVertexYMM doing broadcasts and is like
    // "let's broadcast *before* we combine v0 and v1!  Then we can use vbroadcastss!"
    // Prevent it from "optimizing" here
    asm("" : "+x"(v01)::);
#endif

    v01 = TransformVertexYMM<PositionHas3Elems>(v01, pos0, pos1, pos2, pos3,  //
                                                proj0, proj1, proj2, proj3);
  }

  return v01;
}

#endif

#ifndef USE_AVX
// Note: Assumes 16-byte aligned source
ATTR_TARGET DOLPHIN_FORCE_INLINE static void LoadTransposed(const void* source, Vector& o0,
                                                            Vector& o1, Vector& o2, Vector& o3)
{
#if defined(USE_SSE)
  const Vector* vsource = static_cast<const Vector*>(source);
  o0 = vsource[0];
  o1 = vsource[1];
  o2 = vsource[2];
  o3 = vsource[3];
  _MM_TRANSPOSE4_PS(o0, o1, o2, o3);
#elif defined(USE_NEON)
  float32x4x4_t ld = vld4q_f32(static_cast<const float*>(source));
  o0 = ld.val[0];
  o1 = ld.val[1];
  o2 = ld.val[2];
  o3 = ld.val[3];
#else
  const Vector* vsource = static_cast<const Vector*>(source);
  // clang-format off
  o0.x = vsource[0].x; o0.y = vsource[1].x; o0.z = vsource[2].x; o0.w = vsource[3].x;
  o1.x = vsource[0].y; o1.y = vsource[1].y; o1.z = vsource[2].y; o1.w = vsource[3].y;
  o2.x = vsource[0].z; o2.y = vsource[1].z; o2.z = vsource[2].z; o2.w = vsource[3].z;
  o3.x = vsource[0].w; o3.y = vsource[1].w; o3.z = vsource[2].w; o3.w = vsource[3].w;
  // clang-format on
#endif
}

ATTR_TARGET DOLPHIN_FORCE_INLINE static void LoadTransposedPos(const void* source, Vector& o0,
                                                               Vector& o1, Vector& o2, Vector& o3)
{
  const Vector* vsource = static_cast<const Vector*>(source);
#if defined(USE_SSE)
  o0 = vsource[0];
  o1 = vsource[1];
  o2 = vsource[2];
  o3 = _mm_setr_ps(0.0f, 0.0f, 0.0f, 1.0f);
  _MM_TRANSPOSE4_PS(o0, o1, o2, o3);
#elif defined(USE_NEON)
  float32x4x2_t ld01 = vld2q_f32(static_cast<const float*>(source));
  o0 = ld01.val[0];
  o1 = ld01.val[1];
  o2 = vsource[2];
  o3 = vsetr_f32(0.0f, 0.0f, 0.0f, 1.0f);
  vuzp12q_f32(o2, o3);
  vuzp12q_f32(o0, o2);
  vuzp12q_f32(o1, o3);
#else
  // clang-format off
  o0.x = vsource[0].x; o0.y = vsource[1].x; o0.z = vsource[2].x; o0.w = 0.0f;
  o1.x = vsource[0].y; o1.y = vsource[1].y; o1.z = vsource[2].y; o1.w = 0.0f;
  o2.x = vsource[0].z; o2.y = vsource[1].z; o2.z = vsource[2].z; o2.w = 0.0f;
  o3.x = vsource[0].w; o3.y = vsource[1].w; o3.z = vsource[2].w; o3.w = 1.0f;
  // clang-format on
#endif
}
#endif

#ifndef USE_NEON
ATTR_TARGET DOLPHIN_FORCE_INLINE static void LoadPos(const void* source,  //
                                                     Vector& o0, Vector& o1, Vector& o2)
{
  const Vector* vsource = static_cast<const Vector*>(source);
  o0 = vsource[0];
  o1 = vsource[1];
  o2 = vsource[2];
}
#endif

ATTR_TARGET DOLPHIN_FORCE_INLINE static Vector ApplyMatrix(Vector v, Vector m0, Vector m1,
                                                           Vector m2, Vector m3)
{
#if defined(USE_SSE)
  Vector output = _mm_mul_ps(vector_broadcast<0>(v), m0);
#ifdef USE_FMA
  output = _mm_fmadd_ps(vector_broadcast<1>(v), m1, output);
  output = _mm_fmadd_ps(vector_broadcast<2>(v), m2, output);
  output = _mm_fmadd_ps(vector_broadcast<3>(v), m3, output);
#else
  output = _mm_add_ps(output, _mm_mul_ps(vector_broadcast<1>(v), m1));
  output = _mm_add_ps(output, _mm_mul_ps(vector_broadcast<2>(v), m2));
  output = _mm_add_ps(output, _mm_mul_ps(vector_broadcast<3>(v), m3));
#endif
  return output;
#elif defined(USE_NEON)
  Vector output = vmulq_laneq_f32(m0, v, 0);
  output = vfmaq_laneq_f32(output, m1, v, 1);
  output = vfmaq_laneq_f32(output, m2, v, 2);
  output = vfmaq_laneq_f32(output, m3, v, 3);
  return output;
#else
  Vector output;
  output.x = v.x * m0.x + v.y * m1.x + v.z * m2.x + v.w * m3.x;
  output.y = v.x * m0.y + v.y * m1.y + v.z * m2.y + v.w * m3.y;
  output.z = v.x * m0.z + v.y * m1.z + v.z * m2.z + v.w * m3.z;
  output.w = v.x * m0.w + v.y * m1.w + v.z * m2.w + v.w * m3.w;
  return output;
#endif
}

#ifndef USE_NEON
ATTR_TARGET DOLPHIN_FORCE_INLINE static Vector
TransformVertexNoTranspose(Vector vertex, Vector pos0, Vector pos1, Vector pos2,  //
                           Vector proj0, Vector proj1, Vector proj2, Vector proj3)
{
#ifdef USE_SSE
  Vector mul0 = _mm_mul_ps(vertex, pos0);
  Vector mul1 = _mm_mul_ps(vertex, pos1);
  Vector mul2 = _mm_mul_ps(vertex, pos2);
  Vector mul3 = _mm_setr_ps(0.0f, 0.0f, 0.0f, 1.0f);
#ifdef USE_SSE3
  Vector output = _mm_hadd_ps(_mm_hadd_ps(mul0, mul1), _mm_hadd_ps(mul2, mul3));
#else
  Vector t0 = _mm_add_ps(_mm_unpacklo_ps(mul0, mul2), _mm_unpackhi_ps(mul0, mul2));
  Vector t1 = _mm_add_ps(_mm_unpacklo_ps(mul1, mul3), _mm_unpackhi_ps(mul1, mul3));
  Vector output = _mm_add_ps(_mm_unpacklo_ps(t0, t1), _mm_unpackhi_ps(t0, t1));
#endif
#else
  Vector output;
  output.x = vertex.x * pos0.x + vertex.y * pos0.y + vertex.z * pos0.z + vertex.w * pos0.w;
  output.y = vertex.x * pos1.x + vertex.y * pos1.y + vertex.z * pos1.z + vertex.w * pos1.w;
  output.z = vertex.x * pos2.x + vertex.y * pos2.y + vertex.z * pos2.z + vertex.w * pos2.w;
  output.w = 1.0f;
#endif
  output = ApplyMatrix(output, proj0, proj1, proj2, proj3);
  return output;
}
#endif

template <bool PositionHas3Elems>
ATTR_TARGET DOLPHIN_FORCE_INLINE static Vector
TransformVertex(Vector vertex, Vector pos0, Vector pos1, Vector pos2, Vector pos3,  //
                Vector proj0, Vector proj1, Vector proj2, Vector proj3)
{
  Vector output = pos3;  // vertex.w is always 1.0
#if defined(USE_FMA)
  output = _mm_fmadd_ps(vector_broadcast<0>(vertex), pos0, output);
  output = _mm_fmadd_ps(vector_broadcast<1>(vertex), pos1, output);
  if constexpr (PositionHas3Elems)
    output = _mm_fmadd_ps(vector_broadcast<2>(vertex), pos2, output);
#elif defined(USE_SSE)
  output = _mm_add_ps(output, _mm_mul_ps(vector_broadcast<0>(vertex), pos0));
  output = _mm_add_ps(output, _mm_mul_ps(vector_broadcast<1>(vertex), pos1));
  if constexpr (PositionHas3Elems)
    output = _mm_add_ps(output, _mm_mul_ps(vector_broadcast<2>(vertex), pos2));
#elif defined(USE_NEON)
  output = vfmaq_laneq_f32(output, pos0, vertex, 0);
  output = vfmaq_laneq_f32(output, pos1, vertex, 1);
  if constexpr (PositionHas3Elems)
    output = vfmaq_laneq_f32(output, pos2, vertex, 2);
#else
  output.x += vertex.x * pos0.x + vertex.y * pos1.x;
  output.y += vertex.x * pos0.y + vertex.y * pos1.y;
  output.z += vertex.x * pos0.z + vertex.y * pos1.z;
  if constexpr (PositionHas3Elems)
  {
    output.x += vertex.z * pos2.x;
    output.y += vertex.z * pos2.y;
    output.z += vertex.z * pos2.z;
  }
#endif
  output = ApplyMatrix(output, proj0, proj1, proj2, proj3);
  return output;
}

template <bool PositionHas3Elems, bool PerVertexPosMtx>
ATTR_TARGET DOLPHIN_FORCE_INLINE static Vector
LoadTransformVertex(const u8* data, Vector pos0, Vector pos1, Vector pos2, Vector pos3,
                    Vector proj0, Vector proj1, Vector proj2, Vector proj3)
{
  Vector vertex;
  if constexpr (PerVertexPosMtx)
  {
    // Vertex data layout always starts with posmtx data if available, then position data
    // Convenient for us, that means offsets are always fixed
    u32 idx = data[0] & 0x3f;
    data += sizeof(u32);

    const float* fdata = reinterpret_cast<const float*>(data);

#ifdef USE_NEON
    LoadTransposedPos(&xfmem.posMatrices[idx * 4], pos0, pos1, pos2, pos3);

    if constexpr (PositionHas3Elems)
    {
      vertex = vld1q_f32(fdata);
    }
    else
    {
      vertex = vcombine_f32(vld1_f32(fdata), vdup_n_f32(0.0f));
    }

    vertex = TransformVertex<PositionHas3Elems>(vertex, pos0, pos1, pos2, pos3,  //
                                                proj0, proj1, proj2, proj3);
#else
    LoadPos(&xfmem.posMatrices[idx * 4], pos0, pos1, pos2);

    if constexpr (PositionHas3Elems)
    {
#if defined(USE_SSE)
#ifdef USE_SSE41
      Vector base = _mm_set1_ps(1.0f);
      vertex = _mm_blend_ps(_mm_loadu_ps(fdata), base, 8);
#else
      Vector base = _mm_setr_ps(0.0f, 0.0f, 0.0f, 1.0f);
      Vector mask = _mm_castsi128_ps(_mm_setr_epi32(-1, -1, -1, 0));
      vertex = _mm_or_ps(_mm_and_ps(_mm_loadu_ps(fdata), mask), base);
#endif
#else
      vertex.x = fdata[0];
      vertex.y = fdata[1];
      vertex.z = fdata[2];
      vertex.w = 1.0f;
#endif
    }
    else
    {
#if defined(USE_SSE)
      Vector base = _mm_setr_ps(0.0f, 0.0f, 0.0f, 1.0f);
      vertex = _mm_loadl_pi(base, reinterpret_cast<const __m64*>(fdata));
#else
      vertex.x = fdata[0];
      vertex.y = fdata[1];
      vertex.z = 0.0f;
      vertex.w = 1.0f;
#endif
    }

    vertex = TransformVertexNoTranspose(vertex, pos0, pos1, pos2, proj0, proj1, proj2, proj3);
#endif
  }
  else
  {
    const float* fdata = reinterpret_cast<const float*>(data);
    if constexpr (PositionHas3Elems)
    {
#if defined(USE_SSE)
      vertex = _mm_loadu_ps(fdata);
#elif defined(USE_NEON)
      vertex = vld1q_f32(fdata);
#else
      vertex.x = fdata[0];
      vertex.y = fdata[1];
      vertex.z = fdata[2];
      vertex.w = 1.0f;
#endif
    }
    else
    {
#if defined(USE_SSE)
      vertex = _mm_loadl_pi(_mm_setzero_ps(), reinterpret_cast<const __m64*>(fdata));
#elif defined(USE_NEON)
      vertex = vcombine_f32(vld1_f32(fdata), vdup_n_f32(0.0f));
#else
      vertex.x = fdata[0];
      vertex.y = fdata[1];
      vertex.z = 0.0f;
      vertex.w = 1.0f;
#endif
    }

    vertex = TransformVertex<PositionHas3Elems>(vertex, pos0, pos1, pos2, pos3,  //
                                                proj0, proj1, proj2, proj3);
  }

  return vertex;
}

template <bool PositionHas3Elems, bool PerVertexPosMtx>
ATTR_TARGET static void TransformVertices(void* output, const void* vertices, u32 stride, int count)
{
  const VertexShaderManager& vsmanager = Core::System::GetInstance().GetVertexShaderManager();
  const u8* cvertices = static_cast<const u8*>(vertices);
  Vector* voutput = static_cast<Vector*>(output);
  u32 idx = g_main_cp_state.matrix_index_a.PosNormalMtxIdx & 0x3f;
#ifdef USE_AVX
  __m256 proj0, proj1, proj2, proj3;
  __m256 pos0, pos1, pos2, pos3;
  LoadTransposedYMM(vsmanager.constants.projection.data(), proj0, proj1, proj2, proj3);
  LoadTransposedPosYMM(&xfmem.posMatrices[idx * 4], pos0, pos1, pos2, pos3);
  for (int i = 1; i < count; i += 2)
  {
    const u8* v0data = cvertices;
    const u8* v1data = cvertices + stride;
    __m256 v01 = LoadTransform2Vertices<PositionHas3Elems, PerVertexPosMtx>(
        v0data, v1data, pos0, pos1, pos2, pos3, proj0, proj1, proj2, proj3);
    _mm256_store_ps(reinterpret_cast<float*>(voutput), v01);
    cvertices += stride * 2;
    voutput += 2;
  }
  if (count & 1)
  {
    *voutput = LoadTransformVertex<PositionHas3Elems, PerVertexPosMtx>(
        cvertices,                                                     //
        _mm256_castps256_ps128(pos0), _mm256_castps256_ps128(pos1),    //
        _mm256_castps256_ps128(pos2), _mm256_castps256_ps128(pos3),    //
        _mm256_castps256_ps128(proj0), _mm256_castps256_ps128(proj1),  //
        _mm256_castps256_ps128(proj2), _mm256_castps256_ps128(proj3));
  }
#else
  Vector proj0, proj1, proj2, proj3;
  Vector pos0, pos1, pos2, pos3;
  LoadTransposed(vsmanager.constants.projection.data(), proj0, proj1, proj2, proj3);
  LoadTransposedPos(&xfmem.posMatrices[idx * 4], pos0, pos1, pos2, pos3);
  for (int i = 0; i < count; i++)
  {
    *voutput = LoadTransformVertex<PositionHas3Elems, PerVertexPosMtx>(
        cvertices, pos0, pos1, pos2, pos3, proj0, proj1, proj2, proj3);
    cvertices += stride;
    voutput += 1;
  }
#endif
}

template <CullMode Mode>
ATTR_TARGET DOLPHIN_FORCE_INLINE static bool CullTriangle(const CPUCull::TransformedVertex& a,
                                                          const CPUCull::TransformedVertex& b,
                                                          const CPUCull::TransformedVertex& c)
{
  if (Mode == CullMode::All)
    return true;

  Vector va = reinterpret_cast<const Vector&>(a);
  Vector vb = reinterpret_cast<const Vector&>(b);
  Vector vc = reinterpret_cast<const Vector&>(c);

  // See videosoftware Clipper.cpp

#if defined(USE_SSE)
  Vector wxzya = _mm_shuffle_ps(va, va, _MM_SHUFFLE(1, 2, 0, 3));
  Vector wxzyc = _mm_shuffle_ps(vc, vc, _MM_SHUFFLE(1, 2, 0, 3));
  Vector ywzxb = _mm_shuffle_ps(vb, vb, _MM_SHUFFLE(0, 2, 3, 1));
  Vector part0 = _mm_mul_ps(va, wxzyc);
  Vector part1 = _mm_mul_ps(vc, wxzya);
  Vector part2 = _mm_mul_ps(_mm_sub_ps(part0, part1), ywzxb);
#ifdef USE_SSE3
  Vector part3 = _mm_movehdup_ps(part2);
#else
  Vector part3 = vector_broadcast<1>(part2);
#endif
  Vector part4 = vector_broadcast<3>(part2);
  Vector part5 = _mm_add_ss(_mm_add_ss(part2, part3), part4);
  float normal_z_dir;
  _mm_store_ss(&normal_z_dir, part5);
#elif defined(USE_NEON)
  Vector zero = vdupq_n_f32(0.0f);
  Vector wx0ya = vextq_f32(va, vzip1q_f32(va, zero), 3);
  Vector wx0yc = vextq_f32(vc, vzip1q_f32(vc, zero), 3);
  Vector ywxxb = vuzp2q_f32(vb, vdupq_laneq_f32(vb, 0));
  Vector part0 = vmulq_f32(va, wx0yc);
  Vector part1 = vmulq_f32(vc, wx0ya);
  Vector part2 = vmulq_f32(vsubq_f32(part0, part1), ywxxb);
  float normal_z_dir = vaddvq_f32(part2);
#else
  float normal_z_dir = (c.w * a.x - a.w * c.x) * b.y +  //
                       (c.x * a.y - a.x * c.y) * b.w +  //
                       (c.y * a.w - a.y * c.w) * b.x;
#endif
  bool cull = false;
  switch (Mode)
  {
  case CullMode::None:
    cull = normal_z_dir == 0;
    break;
  case CullMode::Front:
    cull = normal_z_dir <= 0;
    break;
  case CullMode::Back:
    cull = normal_z_dir >= 0;
    break;
  case CullMode::All:
    cull = true;
    break;
  }
  if (cull)
    return true;

#if defined(USE_SSE)
  Vector xyab = _mm_unpacklo_ps(va, vb);
  Vector zwab = _mm_unpackhi_ps(va, vb);
  Vector allx = _mm_shuffle_ps(xyab, vc, _MM_SHUFFLE(0, 0, 1, 0));
  Vector ally = _mm_shuffle_ps(xyab, vc, _MM_SHUFFLE(1, 1, 3, 2));
  Vector allpw = _mm_shuffle_ps(zwab, vc, _MM_SHUFFLE(3, 3, 3, 2));
  Vector allnw = _mm_xor_ps(allpw, _mm_set1_ps(-0.0f));
  __m128i x_gt_pw = _mm_castps_si128(_mm_cmple_ps(allpw, allx));
  __m128i y_gt_pw = _mm_castps_si128(_mm_cmple_ps(allpw, ally));
  __m128i x_lt_nw = _mm_castps_si128(_mm_cmplt_ps(allx, allnw));
  __m128i y_lt_nw = _mm_castps_si128(_mm_cmplt_ps(ally, allnw));
  __m128i any_out_of_bounds = _mm_packs_epi16(_mm_packs_epi32(x_lt_nw, y_lt_nw),  //
                                              _mm_packs_epi32(x_gt_pw, y_gt_pw));
  cull |= 0 != _mm_movemask_epi8(_mm_cmpeq_epi32(_mm_set1_epi32(~0), any_out_of_bounds));
#elif defined(USE_NEON)
  float64x2_t xyab = vreinterpretq_f64_f32(vzip1q_f32(va, vb));
  float64x2_t xycc = vreinterpretq_f64_f32(vzip1q_f32(vc, vc));
  float32x4_t allx = vreinterpretq_f32_f64(vzip1q_f64(xyab, xycc));
  float32x4_t ally = vreinterpretq_f32_f64(vzip2q_f64(xyab, xycc));
  float32x4_t allpw = vextq_f32(vzip2q_f32(va, vb), vdupq_laneq_f32(vc, 3), 2);
  float32x4_t allnw = vnegq_f32(allpw);
  uint16x8_t x_gt_pw = vreinterpretq_u16_u32(vcgtq_f32(allx, allpw));
  uint16x8_t y_gt_pw = vreinterpretq_u16_u32(vcgtq_f32(ally, allpw));
  uint16x8_t x_lt_nw = vreinterpretq_u16_u32(vcltq_f32(allx, allnw));
  uint16x8_t y_lt_nw = vreinterpretq_u16_u32(vcltq_f32(ally, allnw));
  uint8x16_t lt_nw = vreinterpretq_u8_u16(vuzp1q_u16(x_lt_nw, y_lt_nw));
  uint8x16_t gt_pw = vreinterpretq_u8_u16(vuzp1q_u16(x_gt_pw, y_gt_pw));
  uint32x4_t any_out_of_bounds = vreinterpretq_u32_u8(vuzp1q_u8(lt_nw, gt_pw));
  cull |= 0xFFFFFFFF == vmaxvq_u32(any_out_of_bounds);
#else
  cull |= a.x < -a.w && b.x < -b.w && c.x < -c.w;
  cull |= a.y < -a.w && b.y < -b.w && c.y < -c.w;
  cull |= a.x > a.w && b.x > b.w && c.x > c.w;
  cull |= a.y > a.w && b.y > b.w && c.y > c.w;
#endif

  return cull;
}

template <OpcodeDecoder::Primitive Primitive, CullMode Mode>
ATTR_TARGET static bool AreAllVerticesCulled(const CPUCull::TransformedVertex* transformed,
                                             int count)
{
  switch (Primitive)
  {
  case OpcodeDecoder::Primitive::GX_DRAW_QUADS:
  case OpcodeDecoder::Primitive::GX_DRAW_QUADS_2:
  {
    int i = 3;
    for (; i < count; i += 4)
    {
      if (!CullTriangle<Mode>(transformed[i - 3], transformed[i - 2], transformed[i - 1]))
        return false;
      if (!CullTriangle<Mode>(transformed[i - 3], transformed[i - 1], transformed[i - 0]))
        return false;
    }
    // three vertices remaining, so render a triangle
    if (i == count)
    {
      if (!CullTriangle<Mode>(transformed[i - 3], transformed[i - 2], transformed[i - 1]))
        return false;
    }
    break;
  }
  case OpcodeDecoder::Primitive::GX_DRAW_TRIANGLES:
    for (int i = 2; i < count; i += 3)
    {
      if (!CullTriangle<Mode>(transformed[i - 2], transformed[i - 1], transformed[i - 0]))
        return false;
    }
    break;
  case OpcodeDecoder::Primitive::GX_DRAW_TRIANGLE_STRIP:
  {
    bool wind = false;
    for (int i = 2; i < count; ++i)
    {
      if (!CullTriangle<Mode>(transformed[i - 2], transformed[i - !wind], transformed[i - wind]))
        return false;
      wind = !wind;
    }
    break;
  }
  case OpcodeDecoder::Primitive::GX_DRAW_TRIANGLE_FAN:
    for (int i = 2; i < count; ++i)
    {
      if (!CullTriangle<Mode>(transformed[0], transformed[i - 1], transformed[i]))
        return false;
    }
    break;
  }

  return true;
}

}  // namespace VECTOR_NAMESPACE

#undef ATTR_TARGET
#undef VECTOR_NAMESPACE