kc3-lang/SDL

diff --git a/src/audio/SDL_audio.c b/src/audio/SDL_audio.c
index fa2a2b8..a62ef42 100644
--- a/src/audio/SDL_audio.c
+++ b/src/audio/SDL_audio.c
@@ -843,6 +843,8 @@ SDL_GetAudioDriver(int index)
     return NULL;
 }
 
+extern void SDL_ChooseAudioConverters(void);
+
 int
 SDL_AudioInit(const char *driver_name)
 {
@@ -857,6 +859,8 @@ SDL_AudioInit(const char *driver_name)
     SDL_zero(current_audio);
     SDL_zero(open_devices);
 
+    SDL_ChooseAudioConverters();
+
     /* Select the proper audio driver */
     if (driver_name == NULL) {
         driver_name = SDL_getenv("SDL_AUDIODRIVER");
diff --git a/src/audio/SDL_audio_c.h b/src/audio/SDL_audio_c.h
index 62fd6bc..8a8af13 100644
--- a/src/audio/SDL_audio_c.h
+++ b/src/audio/SDL_audio_c.h
@@ -53,16 +53,17 @@ extern SDL_AudioFormat SDL_NextAudioFormat(void);
 /* Function to calculate the size and silence for a SDL_AudioSpec */
 extern void SDL_CalculateAudioSpec(SDL_AudioSpec * spec);
 
-void SDLCALL SDL_Convert_S8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format);
-void SDLCALL SDL_Convert_U8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format);
-void SDLCALL SDL_Convert_S16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format);
-void SDLCALL SDL_Convert_U16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format);
-void SDLCALL SDL_Convert_S32_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format);
-void SDLCALL SDL_Convert_F32_to_S8(SDL_AudioCVT *cvt, SDL_AudioFormat format);
-void SDLCALL SDL_Convert_F32_to_U8(SDL_AudioCVT *cvt, SDL_AudioFormat format);
-void SDLCALL SDL_Convert_F32_to_S16(SDL_AudioCVT *cvt, SDL_AudioFormat format);
-void SDLCALL SDL_Convert_F32_to_U16(SDL_AudioCVT *cvt, SDL_AudioFormat format);
-void SDLCALL SDL_Convert_F32_to_S32(SDL_AudioCVT *cvt, SDL_AudioFormat format);
+/* These pointers get set during init to various SIMD implementations. */
+extern SDL_AudioFilter SDL_Convert_S8_to_F32;
+extern SDL_AudioFilter SDL_Convert_U8_to_F32;
+extern SDL_AudioFilter SDL_Convert_S16_to_F32;
+extern SDL_AudioFilter SDL_Convert_U16_to_F32;
+extern SDL_AudioFilter SDL_Convert_S32_to_F32;
+extern SDL_AudioFilter SDL_Convert_F32_to_S8;
+extern SDL_AudioFilter SDL_Convert_F32_to_U8;
+extern SDL_AudioFilter SDL_Convert_F32_to_S16;
+extern SDL_AudioFilter SDL_Convert_F32_to_U16;
+extern SDL_AudioFilter SDL_Convert_F32_to_S32;
 
 
 /* SDL_AudioStream is a new audio conversion interface. It
diff --git a/src/audio/SDL_audiotypecvt.c b/src/audio/SDL_audiotypecvt.c
index cebfca6..505e25a 100644
--- a/src/audio/SDL_audiotypecvt.c
+++ b/src/audio/SDL_audiotypecvt.c
@@ -22,14 +22,55 @@
 #include "../SDL_internal.h"
 #include "SDL_audio.h"
 #include "SDL_audio_c.h"
+#include "SDL_cpuinfo.h"
 #include "SDL_assert.h"
 
+/* !!! FIXME: write NEON code. */
+#define HAVE_NEON_INTRINSICS 0
+
+/* !!! FIXME: wire this up to the configure script, etc. */
+#define HAVE_SSE2_INTRINSICS 0
+
+#if HAVE_SSE2_INTRINSICS
+#include <emmintrin.h>
+#endif
+
+#if defined(__x86_64__) && HAVE_SSE2_INTRINSICS
+#define NEED_SCALAR_CONVERTER_FALLBACKS 0  /* x86_64 guarantees SSE2. */
+#elif __MACOSX__ && HAVE_SSE2_INTRINSICS
+#define NEED_SCALAR_CONVERTER_FALLBACKS 0  /* Mac OS X/Intel guarantees SSE2. */
+#elif defined(__ARM_ARCH) && (__ARM_ARCH >= 8) && HAVE_NEON_INTRINSICS
+#define NEED_SCALAR_CONVERTER_FALLBACKS 0  /* ARMv8+ promise NEON. */
+#elif defined(__APPLE__) && defined(__ARM_ARCH) && (__ARM_ARCH >= 7) && HAVE_NEON_INTRINSICS
+#define NEED_SCALAR_CONVERTER_FALLBACKS 0  /* All Apple ARMv7 chips promise NEON support. */
+#endif
+
+/* Set to zero if platform is guaranteed to use a SIMD codepath here. */
+#ifndef NEED_SCALAR_CONVERTER_FALLBACKS
+#define NEED_SCALAR_CONVERTER_FALLBACKS 1
+#endif
+
+/* Function pointers set to a CPU-specific implementation. */
+SDL_AudioFilter SDL_Convert_S8_to_F32 = NULL;
+SDL_AudioFilter SDL_Convert_U8_to_F32 = NULL;
+SDL_AudioFilter SDL_Convert_S16_to_F32 = NULL;
+SDL_AudioFilter SDL_Convert_U16_to_F32 = NULL;
+SDL_AudioFilter SDL_Convert_S32_to_F32 = NULL;
+SDL_AudioFilter SDL_Convert_F32_to_S8 = NULL;
+SDL_AudioFilter SDL_Convert_F32_to_U8 = NULL;
+SDL_AudioFilter SDL_Convert_F32_to_S16 = NULL;
+SDL_AudioFilter SDL_Convert_F32_to_U16 = NULL;
+SDL_AudioFilter SDL_Convert_F32_to_S32 = NULL;
+
+
 #define DIVBY127 0.0078740157480315f
 #define DIVBY32767 3.05185094759972e-05f
 #define DIVBY2147483647 4.6566128752458e-10f
 
-void SDLCALL
-SDL_Convert_S8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+
+#if NEED_SCALAR_CONVERTER_FALLBACKS
+static void SDLCALL
+SDL_Convert_S8_to_F32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format)
 {
     const Sint8 *src = ((const Sint8 *) (cvt->buf + cvt->len_cvt)) - 1;
     float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1;
@@ -47,8 +88,8 @@ SDL_Convert_S8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
     }
 }
 
-void SDLCALL
-SDL_Convert_U8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+static void SDLCALL
+SDL_Convert_U8_to_F32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format)
 {
     const Uint8 *src = ((const Uint8 *) (cvt->buf + cvt->len_cvt)) - 1;
     float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1;
@@ -66,8 +107,8 @@ SDL_Convert_U8_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
     }
 }
 
-void SDLCALL
-SDL_Convert_S16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+static void SDLCALL
+SDL_Convert_S16_to_F32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format)
 {
     const Sint16 *src = ((const Sint16 *) (cvt->buf + cvt->len_cvt)) - 1;
     float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1;
@@ -85,8 +126,8 @@ SDL_Convert_S16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
     }
 }
 
-void SDLCALL
-SDL_Convert_U16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+static void SDLCALL
+SDL_Convert_U16_to_F32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format)
 {
     const Uint16 *src = ((const Uint16 *) (cvt->buf + cvt->len_cvt)) - 1;
     float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1;
@@ -104,8 +145,8 @@ SDL_Convert_U16_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
     }
 }
 
-void SDLCALL
-SDL_Convert_S32_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+static void SDLCALL
+SDL_Convert_S32_to_F32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format)
 {
     const Sint32 *src = (const Sint32 *) cvt->buf;
     float *dst = (float *) cvt->buf;
@@ -122,8 +163,8 @@ SDL_Convert_S32_to_F32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
     }
 }
 
-void SDLCALL
-SDL_Convert_F32_to_S8(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+static void SDLCALL
+SDL_Convert_F32_to_S8_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format)
 {
     const float *src = (const float *) cvt->buf;
     Sint8 *dst = (Sint8 *) cvt->buf;
@@ -141,8 +182,8 @@ SDL_Convert_F32_to_S8(SDL_AudioCVT *cvt, SDL_AudioFormat format)
     }
 }
 
-void SDLCALL
-SDL_Convert_F32_to_U8(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+static void SDLCALL
+SDL_Convert_F32_to_U8_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format)
 {
     const float *src = (const float *) cvt->buf;
     Uint8 *dst = (Uint8 *) cvt->buf;
@@ -160,8 +201,8 @@ SDL_Convert_F32_to_U8(SDL_AudioCVT *cvt, SDL_AudioFormat format)
     }
 }
 
-void SDLCALL
-SDL_Convert_F32_to_S16(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+static void SDLCALL
+SDL_Convert_F32_to_S16_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format)
 {
     const float *src = (const float *) cvt->buf;
     Sint16 *dst = (Sint16 *) cvt->buf;
@@ -179,8 +220,8 @@ SDL_Convert_F32_to_S16(SDL_AudioCVT *cvt, SDL_AudioFormat format)
     }
 }
 
-void SDLCALL
-SDL_Convert_F32_to_U16(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+static void SDLCALL
+SDL_Convert_F32_to_U16_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format)
 {
     const float *src = (const float *) cvt->buf;
     Uint16 *dst = (Uint16 *) cvt->buf;
@@ -198,8 +239,8 @@ SDL_Convert_F32_to_U16(SDL_AudioCVT *cvt, SDL_AudioFormat format)
     }
 }
 
-void SDLCALL
-SDL_Convert_F32_to_S32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+static void SDLCALL
+SDL_Convert_F32_to_S32_Scalar(SDL_AudioCVT *cvt, SDL_AudioFormat format)
 {
     const float *src = (const float *) cvt->buf;
     Sint32 *dst = (Sint32 *) cvt->buf;
@@ -215,5 +256,532 @@ SDL_Convert_F32_to_S32(SDL_AudioCVT *cvt, SDL_AudioFormat format)
         cvt->filters[cvt->filter_index](cvt, AUDIO_S32SYS);
     }
 }
+#endif
+
+
+#if HAVE_SSE2_INTRINSICS
+static void SDLCALL
+SDL_Convert_S8_to_F32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+{
+    const Sint8 *src = ((const Sint8 *) (cvt->buf + cvt->len_cvt)) - 1;
+    float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1;
+    int i;
+
+    LOG_DEBUG_CONVERT("AUDIO_S8", "AUDIO_F32 (using SSE2)");
+
+    /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */
+    for (i = cvt->len_cvt; i && (((size_t) (dst-15)) & 15); --i, --src, --dst) {
+        *dst = (((float) *src) * DIVBY127);
+    }
+
+    src -= 15; dst -= 15;  /* adjust to read SSE blocks from the start. */
+    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
+
+    /* Make sure src is aligned too. */
+    if ((((size_t) src) & 15) == 0) {
+        /* Aligned! Do SSE blocks as long as we have 16 bytes available. */
+        const __m128i *mmsrc = (const __m128i *) src;
+        const __m128i zero = _mm_setzero_si128();
+        const __m128 divby127 = _mm_set1_ps(DIVBY127);
+        while (i >= 16) {   /* 16 * 8-bit */
+            const __m128i bytes = _mm_load_si128(mmsrc);  /* get 16 sint8 into an XMM register. */
+            /* treat as int16, shift left to clear every other sint16, then back right with sign-extend. Now sint16. */
+            const __m128i shorts1 = _mm_srai_epi16(_mm_slli_epi16(bytes, 8), 8);
+            /* right-shift-sign-extend gets us sint16 with the other set of values. */
+            const __m128i shorts2 = _mm_srai_epi16(bytes, 8);
+            /* unpack against zero to make these int32, shift to make them sign-extend, convert to float, multiply. Whew! */
+            const __m128 floats1 = _mm_mul_ps(_mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpacklo_epi16(shorts1, zero), 16), 16)), divby127);
+            const __m128 floats2 = _mm_mul_ps(_mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpacklo_epi16(shorts2, zero), 16), 16)), divby127);
+            const __m128 floats3 = _mm_mul_ps(_mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpackhi_epi16(shorts1, zero), 16), 16)), divby127);
+            const __m128 floats4 = _mm_mul_ps(_mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(_mm_unpackhi_epi16(shorts2, zero), 16), 16)), divby127);
+            /* Interleave back into correct order, store. */
+            _mm_store_ps(dst, _mm_unpacklo_ps(floats1, floats2));
+            _mm_store_ps(dst+4, _mm_unpackhi_ps(floats1, floats2));
+            _mm_store_ps(dst+8, _mm_unpacklo_ps(floats3, floats4));
+            _mm_store_ps(dst+12, _mm_unpackhi_ps(floats3, floats4));
+            i -= 16; mmsrc--; dst -= 16;
+        }
+
+        src = (const Sint8 *) mmsrc;
+    }
+
+    src += 15; dst += 15;  /* adjust for any scalar finishing. */
+
+    /* Finish off any leftovers with scalar operations. */
+    while (i) {
+        *dst = (((float) *src) * DIVBY127);
+        i--; src--; dst--;
+    }
+
+    cvt->len_cvt *= 4;
+    if (cvt->filters[++cvt->filter_index]) {
+        cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS);
+    }
+}
+
+static void SDLCALL
+SDL_Convert_U8_to_F32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+{
+    const Uint8 *src = ((const Uint8 *) (cvt->buf + cvt->len_cvt)) - 1;
+    float *dst = ((float *) (cvt->buf + cvt->len_cvt * 4)) - 1;
+    int i;
+
+    LOG_DEBUG_CONVERT("AUDIO_U8", "AUDIO_F32 (using SSE2)");
+
+    /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */
+    for (i = cvt->len_cvt; i && (((size_t) (dst-15)) & 15); --i, --src, --dst) {
+        *dst = ((((float) *src) * DIVBY127) - 1.0f);
+    }
+
+    src -= 15; dst -= 15;  /* adjust to read SSE blocks from the start. */
+    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
+
+    /* Make sure src is aligned too. */
+    if ((((size_t) src) & 15) == 0) {
+        /* Aligned! Do SSE blocks as long as we have 16 bytes available. */
+        const __m128i *mmsrc = (const __m128i *) src;
+        const __m128i zero = _mm_setzero_si128();
+        const __m128 divby127 = _mm_set1_ps(DIVBY127);
+        const __m128 minus1 = _mm_set1_ps(-1.0f);
+        while (i >= 16) {   /* 16 * 8-bit */
+            const __m128i bytes = _mm_load_si128(mmsrc);  /* get 16 uint8 into an XMM register. */
+            /* treat as int16, shift left to clear every other sint16, then back right with zero-extend. Now uint16. */
+            const __m128i shorts1 = _mm_srli_epi16(_mm_slli_epi16(bytes, 8), 8);
+            /* right-shift-zero-extend gets us uint16 with the other set of values. */
+            const __m128i shorts2 = _mm_srli_epi16(bytes, 8);
+            /* unpack against zero to make these int32, convert to float, multiply, add. Whew! */
+            /* Note that AVX2 can do floating point multiply+add in one instruction, fwiw. SSE2 cannot. */
+            const __m128 floats1 = _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpacklo_epi16(shorts1, zero)), divby127), minus1);
+            const __m128 floats2 = _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpacklo_epi16(shorts2, zero)), divby127), minus1);
+            const __m128 floats3 = _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpackhi_epi16(shorts1, zero)), divby127), minus1);
+            const __m128 floats4 = _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpackhi_epi16(shorts2, zero)), divby127), minus1);
+            /* Interleave back into correct order, store. */
+            _mm_store_ps(dst, _mm_unpacklo_ps(floats1, floats2));
+            _mm_store_ps(dst+4, _mm_unpackhi_ps(floats1, floats2));
+            _mm_store_ps(dst+8, _mm_unpacklo_ps(floats3, floats4));
+            _mm_store_ps(dst+12, _mm_unpackhi_ps(floats3, floats4));
+            i -= 16; mmsrc--; dst -= 16;
+        }
+
+        src = (const Uint8 *) mmsrc;
+    }
+
+    src += 15; dst += 15;  /* adjust for any scalar finishing. */
+
+    /* Finish off any leftovers with scalar operations. */
+    while (i) {
+        *dst = ((((float) *src) * DIVBY127) - 1.0f);
+        i--; src--; dst--;
+    }
+
+    cvt->len_cvt *= 4;
+    if (cvt->filters[++cvt->filter_index]) {
+        cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS);
+    }
+}
+
+static void SDLCALL
+SDL_Convert_S16_to_F32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+{
+    const Sint16 *src = ((const Sint16 *) (cvt->buf + cvt->len_cvt)) - 1;
+    float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1;
+    int i;
+
+    LOG_DEBUG_CONVERT("AUDIO_S16", "AUDIO_F32 (using SSE2)");
+
+    /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */
+    for (i = cvt->len_cvt / sizeof (Sint16); i && (((size_t) (dst-7)) & 15); --i, --src, --dst) {
+        *dst = (((float) *src) * DIVBY32767);
+    }
+
+    src -= 7; dst -= 7;  /* adjust to read SSE blocks from the start. */
+    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
+
+    /* Make sure src is aligned too. */
+    if ((((size_t) src) & 15) == 0) {
+        /* Aligned! Do SSE blocks as long as we have 16 bytes available. */
+        const __m128 divby32767 = _mm_set1_ps(DIVBY32767);
+        while (i >= 8) {   /* 8 * 16-bit */
+            const __m128i ints = _mm_load_si128((__m128i const *) src);  /* get 8 sint16 into an XMM register. */
+            /* treat as int32, shift left to clear every other sint16, then back right with sign-extend. Now sint32. */
+            const __m128i a = _mm_srai_epi32(_mm_slli_epi32(ints, 16), 16);
+            /* right-shift-sign-extend gets us sint32 with the other set of values. */
+            const __m128i b = _mm_srai_epi32(ints, 16);
+            /* Interleave these back into the right order, convert to float, multiply, store. */
+            _mm_store_ps(dst, _mm_mul_ps(_mm_cvtepi32_ps(_mm_unpacklo_epi32(a, b)), divby32767));
+            _mm_store_ps(dst+4, _mm_mul_ps(_mm_cvtepi32_ps(_mm_unpackhi_epi32(a, b)), divby32767));
+            i -= 8; src -= 8; dst -= 8;
+        }
+    }
+
+    src += 7; dst += 7;  /* adjust for any scalar finishing. */
+
+    /* Finish off any leftovers with scalar operations. */
+    while (i) {
+        *dst = (((float) *src) * DIVBY32767);
+        i--; src--; dst--;
+    }
+
+    cvt->len_cvt *= 2;
+    if (cvt->filters[++cvt->filter_index]) {
+        cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS);
+    }
+}
+
+static void SDLCALL
+SDL_Convert_U16_to_F32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+{
+    const Uint16 *src = ((const Uint16 *) (cvt->buf + cvt->len_cvt)) - 1;
+    float *dst = ((float *) (cvt->buf + cvt->len_cvt * 2)) - 1;
+    int i;
+
+    LOG_DEBUG_CONVERT("AUDIO_U16", "AUDIO_F32 (using SSE2)");
+
+    /* Get dst aligned to 16 bytes (since buffer is growing, we don't have to worry about overreading from src) */
+    for (i = cvt->len_cvt / sizeof (Sint16); i && (((size_t) (dst-7)) & 15); --i, --src, --dst) {
+        *dst = ((((float) *src) * DIVBY32767) - 1.0f);
+    }
+
+    src -= 7; dst -= 7;  /* adjust to read SSE blocks from the start. */
+    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
+
+    /* Make sure src is aligned too. */
+    if ((((size_t) src) & 15) == 0) {
+        /* Aligned! Do SSE blocks as long as we have 16 bytes available. */
+        const __m128 divby32767 = _mm_set1_ps(DIVBY32767);
+        const __m128 minus1 = _mm_set1_ps(1.0f);
+        while (i >= 8) {   /* 8 * 16-bit */
+            const __m128i ints = _mm_load_si128((__m128i const *) src);  /* get 8 sint16 into an XMM register. */
+            /* treat as int32, shift left to clear every other sint16, then back right with zero-extend. Now sint32. */
+            const __m128i a = _mm_srli_epi32(_mm_slli_epi32(ints, 16), 16);
+            /* right-shift-sign-extend gets us sint32 with the other set of values. */
+            const __m128i b = _mm_srli_epi32(ints, 16);
+            /* Interleave these back into the right order, convert to float, multiply, store. */
+            _mm_store_ps(dst, _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpacklo_epi32(a, b)), divby32767), minus1));
+            _mm_store_ps(dst+4, _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(_mm_unpackhi_epi32(a, b)), divby32767), minus1));
+            i -= 8; src -= 8; dst -= 8;
+        }
+    }
+
+    src += 7; dst += 7;  /* adjust for any scalar finishing. */
+
+    /* Finish off any leftovers with scalar operations. */
+    while (i) {
+        *dst = ((((float) *src) * DIVBY32767) - 1.0f);
+        i--; src--; dst--;
+    }
+
+    cvt->len_cvt *= 2;
+    if (cvt->filters[++cvt->filter_index]) {
+        cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS);
+    }
+}
+
+static void SDLCALL
+SDL_Convert_S32_to_F32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+{
+    const Sint32 *src = (const Sint32 *) cvt->buf;
+    float *dst = (float *) cvt->buf;
+    int i;
+
+    LOG_DEBUG_CONVERT("AUDIO_S32", "AUDIO_F32 (using SSE2)");
+
+    /* Get dst aligned to 16 bytes */
+    for (i = cvt->len_cvt / sizeof (Sint32); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
+        *dst = (float) (((double) *src) * DIVBY2147483647);
+    }
+
+    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
+    SDL_assert(!i || ((((size_t) src) & 15) == 0));
+
+    {
+        /* Aligned! Do SSE blocks as long as we have 16 bytes available. */
+        const __m128d divby2147483647 = _mm_set1_pd(DIVBY2147483647);
+        const __m128i *mmsrc = (const __m128i *) src;
+        while (i >= 4) {   /* 4 * sint32 */
+            const __m128i ints = _mm_load_si128(mmsrc);
+            /* bitshift the whole register over, so _mm_cvtepi32_pd can read the top ints in the bottom of the vector. */
+            const __m128d doubles1 = _mm_mul_pd(_mm_cvtepi32_pd(_mm_bsrli_si128(ints, 8)), divby2147483647);
+            const __m128d doubles2 = _mm_mul_pd(_mm_cvtepi32_pd(ints), divby2147483647);
+            /* convert to float32, bitshift/or to get these into a vector to store. */
+            _mm_store_ps(dst, _mm_castsi128_ps(_mm_or_si128(_mm_bslli_si128(_mm_castps_si128(_mm_cvtpd_ps(doubles1)), 8), _mm_castps_si128(_mm_cvtpd_ps(doubles2)))));
+            i -= 4; mmsrc++; dst += 4;
+        }
+        src = (const Sint32 *) mmsrc;
+    }
+
+    /* Finish off any leftovers with scalar operations. */
+    while (i) {
+        *dst = (float) (((double) *src) * DIVBY2147483647);
+        i--; src++; dst++;
+    }
+
+    if (cvt->filters[++cvt->filter_index]) {
+        cvt->filters[cvt->filter_index](cvt, AUDIO_F32SYS);
+    }
+}
+
+static void SDLCALL
+SDL_Convert_F32_to_S8_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+{
+    const float *src = (const float *) cvt->buf;
+    Sint8 *dst = (Sint8 *) cvt->buf;
+    int i;
+
+    LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S8 (using SSE2)");
+
+    /* Get dst aligned to 16 bytes */
+    for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
+        *dst = (Sint8) (*src * 127.0f);
+    }
+
+    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
+
+    /* Make sure src is aligned too. */
+    if ((((size_t) src) & 15) == 0) {
+        /* Aligned! Do SSE blocks as long as we have 16 bytes available. */
+        const __m128 mulby127 = _mm_set1_ps(127.0f);
+        __m128i *mmdst = (__m128i *) dst;
+        while (i >= 16) {   /* 16 * float32 */
+            const __m128i ints1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src), mulby127));  /* load 4 floats, convert to sint32 */
+            const __m128i ints2 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src+4), mulby127));  /* load 4 floats, convert to sint32 */
+            const __m128i ints3 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src+8), mulby127));  /* load 4 floats, convert to sint32 */
+            const __m128i ints4 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src+12), mulby127));  /* load 4 floats, convert to sint32 */
+            _mm_store_si128(mmdst, _mm_packs_epi16(_mm_packs_epi32(ints1, ints2), _mm_packs_epi32(ints3, ints4)));  /* pack down, store out. */
+            i -= 16; src += 16; mmdst++;
+        }
+        dst = (Sint8 *) mmdst;
+    }
+
+    /* Finish off any leftovers with scalar operations. */
+    while (i) {
+        *dst = (Sint8) (*src * 127.0f);
+        i--; src++; dst++;
+    }
+
+    cvt->len_cvt /= 4;
+    if (cvt->filters[++cvt->filter_index]) {
+        cvt->filters[cvt->filter_index](cvt, AUDIO_S8);
+    }
+}
+
+static void SDLCALL
+SDL_Convert_F32_to_U8_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+{
+    const float *src = (const float *) cvt->buf;
+    Uint8 *dst = (Uint8 *) cvt->buf;
+    int i;
+
+    LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_U8 (using SSE2)");
+
+    /* Get dst aligned to 16 bytes */
+    for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
+        *dst = (Uint8) ((*src + 1.0f) * 127.0f);
+    }
+
+    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
+
+    /* Make sure src is aligned too. */
+    if ((((size_t) src) & 15) == 0) {
+        /* Aligned! Do SSE blocks as long as we have 16 bytes available. */
+        const __m128 add1 = _mm_set1_ps(1.0f);
+        const __m128 mulby127 = _mm_set1_ps(127.0f);
+        __m128i *mmdst = (__m128i *) dst;
+        while (i >= 16) {   /* 16 * float32 */
+            const __m128i ints1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_add_ps(_mm_load_ps(src), add1), mulby127));  /* load 4 floats, convert to sint32 */
+            const __m128i ints2 = _mm_cvtps_epi32(_mm_mul_ps(_mm_add_ps(_mm_load_ps(src+4), add1), mulby127));  /* load 4 floats, convert to sint32 */
+            const __m128i ints3 = _mm_cvtps_epi32(_mm_mul_ps(_mm_add_ps(_mm_load_ps(src+8), add1), mulby127));  /* load 4 floats, convert to sint32 */
+            const __m128i ints4 = _mm_cvtps_epi32(_mm_mul_ps(_mm_add_ps(_mm_load_ps(src+12), add1), mulby127));  /* load 4 floats, convert to sint32 */
+            _mm_store_si128(mmdst, _mm_packus_epi16(_mm_packs_epi32(ints1, ints2), _mm_packs_epi32(ints3, ints4)));  /* pack down, store out. */
+            i -= 16; src += 16; mmdst++;
+        }
+        dst = (Uint8 *) mmdst;
+    }
+
+    /* Finish off any leftovers with scalar operations. */
+    while (i) {
+        *dst = (Uint8) ((*src + 1.0f) * 127.0f);
+        i--; src++; dst++;
+    }
+
+    cvt->len_cvt /= 4;
+    if (cvt->filters[++cvt->filter_index]) {
+        cvt->filters[cvt->filter_index](cvt, AUDIO_U8);
+    }
+}
+
+static void SDLCALL
+SDL_Convert_F32_to_S16_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+{
+    const float *src = (const float *) cvt->buf;
+    Sint16 *dst = (Sint16 *) cvt->buf;
+    int i;
+
+    LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S16 (using SSE2)");
+
+    /* Get dst aligned to 16 bytes */
+    for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
+        *dst = (Sint16) (*src * 32767.0f);
+    }
+
+    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
+
+    /* Make sure src is aligned too. */
+    if ((((size_t) src) & 15) == 0) {
+        /* Aligned! Do SSE blocks as long as we have 16 bytes available. */
+        const __m128 mulby32767 = _mm_set1_ps(32767.0f);
+        __m128i *mmdst = (__m128i *) dst;
+        while (i >= 8) {   /* 8 * float32 */
+            const __m128i ints1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src), mulby32767));  /* load 4 floats, convert to sint32 */
+            const __m128i ints2 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src+4), mulby32767));  /* load 4 floats, convert to sint32 */
+            _mm_store_si128(mmdst, _mm_packs_epi32(ints1, ints2));  /* pack to sint16, store out. */
+            i -= 8; src += 8; mmdst++;
+        }
+        dst = (Sint16 *) mmdst;
+    }
+
+    /* Finish off any leftovers with scalar operations. */
+    while (i) {
+        *dst = (((float) *src) * DIVBY32767);
+        i--; src++; dst++;
+    }
+
+    cvt->len_cvt /= 2;
+    if (cvt->filters[++cvt->filter_index]) {
+        cvt->filters[cvt->filter_index](cvt, AUDIO_S16SYS);
+    }
+}
+
+static void SDLCALL
+SDL_Convert_F32_to_U16_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+{
+    const float *src = (const float *) cvt->buf;
+    Uint16 *dst = (Uint16 *) cvt->buf;
+    int i;
+
+    LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_U16 (using SSE2)");
+
+    /* Get dst aligned to 16 bytes */
+    for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
+        *dst = (Uint16) ((*src + 1.0f) * 32767.0f);
+    }
+
+    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
+
+    /* Make sure src is aligned too. */
+    if ((((size_t) src) & 15) == 0) {
+        /* Aligned! Do SSE blocks as long as we have 16 bytes available. */
+        /* This calculates differently than the scalar path because SSE2 can't
+           pack int32 data down to unsigned int16. _mm_packs_epi32 does signed
+           saturation, so that would corrupt our data. _mm_packus_epi32 exists,
+           but not before SSE 4.1. So we convert from float to sint16, packing
+           that down with legit signed saturation, and then xor the top bit
+           against 1. This results in the correct unsigned 16-bit value, even
+           though it looks like dark magic. */
+        const __m128 mulby32767 = _mm_set1_ps(32767.0f);
+        const __m128i topbit = _mm_set1_epi16(-32768);
+        __m128i *mmdst = (__m128i *) dst;
+        while (i >= 8) {   /* 8 * float32 */
+            const __m128i ints1 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src), mulby32767));  /* load 4 floats, convert to sint32 */
+            const __m128i ints2 = _mm_cvtps_epi32(_mm_mul_ps(_mm_load_ps(src+4), mulby32767));  /* load 4 floats, convert to sint32 */
+            _mm_store_si128(mmdst, _mm_xor_si128(_mm_packs_epi32(ints1, ints2), topbit));  /* pack to sint16, xor top bit, store out. */
+            i -= 8; src += 8; mmdst++;
+        }
+        dst = (Uint16 *) mmdst;
+    }
+
+    /* Finish off any leftovers with scalar operations. */
+    while (i) {
+        *dst = (Uint16) ((*src + 1.0f) * 32767.0f);
+        i--; src++; dst++;
+    }
+
+    cvt->len_cvt /= 2;
+    if (cvt->filters[++cvt->filter_index]) {
+        cvt->filters[cvt->filter_index](cvt, AUDIO_U16SYS);
+    }
+}
+
+static void SDLCALL
+SDL_Convert_F32_to_S32_SSE2(SDL_AudioCVT *cvt, SDL_AudioFormat format)
+{
+    const float *src = (const float *) cvt->buf;
+    Sint32 *dst = (Sint32 *) cvt->buf;
+    int i;
+
+    LOG_DEBUG_CONVERT("AUDIO_F32", "AUDIO_S32 (using SSE2)");
+
+    /* Get dst aligned to 16 bytes */
+    for (i = cvt->len_cvt / sizeof (float); i && (((size_t) dst) & 15); --i, ++src, ++dst) {
+        *dst = (Sint32) (((double) *src) * 2147483647.0);
+    }
+
+    SDL_assert(!i || ((((size_t) dst) & 15) == 0));
+    SDL_assert(!i || ((((size_t) src) & 15) == 0));
+
+    {
+        /* Aligned! Do SSE blocks as long as we have 16 bytes available. */
+        const __m128d mulby2147483647 = _mm_set1_pd(2147483647.0);
+        __m128i *mmdst = (__m128i *) dst;
+        while (i >= 4) {   /* 4 * float32 */
+            const __m128 floats = _mm_load_ps(src);
+            /* bitshift the whole register over, so _mm_cvtps_pd can read the top floats in the bottom of the vector. */
+            const __m128d doubles1 = _mm_mul_pd(_mm_cvtps_pd(_mm_castsi128_ps(_mm_bsrli_si128(_mm_castps_si128(floats), 8))), mulby2147483647);
+            const __m128d doubles2 = _mm_mul_pd(_mm_cvtps_pd(floats), mulby2147483647);
+            _mm_store_si128(mmdst, _mm_or_si128(_mm_bslli_si128(_mm_cvtpd_epi32(doubles1), 8), _mm_cvtpd_epi32(doubles2)));
+            i -= 4; src += 4; mmdst++;
+        }
+        dst = (Sint32 *) mmdst;
+    }
+
+    /* Finish off any leftovers with scalar operations. */
+    while (i) {
+        *dst = (Sint32) (((double) *src) * 2147483647.0);
+        i--; src++; dst++;
+    }
+
+    if (cvt->filters[++cvt->filter_index]) {
+        cvt->filters[cvt->filter_index](cvt, AUDIO_S32SYS);
+    }
+}
+#endif
+
+
+void SDL_ChooseAudioConverters(void)
+{
+    static SDL_bool converters_chosen = SDL_FALSE;
+
+    if (converters_chosen) {
+        return;
+    }
+
+    #define SET_CONVERTER_FUNCS(fntype) \
+        SDL_Convert_S8_to_F32 = SDL_Convert_S8_to_F32_##fntype; \
+        SDL_Convert_U8_to_F32 = SDL_Convert_U8_to_F32_##fntype; \
+        SDL_Convert_S16_to_F32 = SDL_Convert_S16_to_F32_##fntype; \
+        SDL_Convert_U16_to_F32 = SDL_Convert_U16_to_F32_##fntype; \
+        SDL_Convert_S32_to_F32 = SDL_Convert_S32_to_F32_##fntype; \
+        SDL_Convert_F32_to_S8 = SDL_Convert_F32_to_S8_##fntype; \
+        SDL_Convert_F32_to_U8 = SDL_Convert_F32_to_U8_##fntype; \
+        SDL_Convert_F32_to_S16 = SDL_Convert_F32_to_S16_##fntype; \
+        SDL_Convert_F32_to_U16 = SDL_Convert_F32_to_U16_##fntype; \
+        SDL_Convert_F32_to_S32 = SDL_Convert_F32_to_S32_##fntype; \
+        converters_chosen = SDL_TRUE
+
+    #if HAVE_SSE2_INTRINSICS
+    if (SDL_HasSSE2()) {
+        SET_CONVERTER_FUNCS(SSE2);
+        return;
+    }
+    #endif
+
+    #if NEED_SCALAR_CONVERTER_FALLBACKS
+    SET_CONVERTER_FUNCS(Scalar);
+    #endif
+
+    #undef SET_CONVERTER_FUNCS
+
+    SDL_assert(converters_chosen == SDL_TRUE);
+}
 
 /* vi: set ts=4 sw=4 expandtab: */