From 1319c4533a7d3a2562907d043d238bfe3d51d335 Mon Sep 17 00:00:00 2001
From: Peter Meerwald
Date: Tue, 23 Oct 2012 17:54:57 +0200
Subject: [PATCH] core: Add ARM NEON optimized sample conversion code
final:
* includes some minor style fixes and build-time changes to allow
building a single binary for neon and non-neon systems
v4:
* fix for sample length < 4
v3:
* convert from intrinsics to inline assembly
v2:
* load and store data with vld1/vld1q and vst1/vst1q, resp., to work
around alignment issues of compiler-generated vldmia instruction
* remove redundant check for NEON flags
Ubuntu/Linaro gcc 4.6.3
arm-linux-gnueabi-gcc -O2 -mcpu=cortex-a8 -mfloat-abi=softfp -mfpu=neon
runtime on beagle-xm:
D: [pulseaudio] sconv_neon.c: checking NEON sconv_s16le_from_float
I: [pulseaudio] sconv_neon.c: NEON: 3754 usec.
I: [pulseaudio] sconv_neon.c: ref: 58594 usec.
D: [pulseaudio] sconv_neon.c: checking NEON sconv_s16le_to_float
I: [pulseaudio] sconv_neon.c: NEON: 1831 usec.
I: [pulseaudio] sconv_neon.c: ref: 10528 usec.
I: [pulseaudio] sconv_neon.c: Initialising ARM NEON optimized conversions.
conversion may be off by one for some samples due to rounding issues
---
configure.ac | 28 ++++++++++
src/Makefile.am | 10 +++-
src/pulsecore/cpu-arm.c | 4 ++
src/pulsecore/cpu-arm.h | 8 +++
src/pulsecore/sconv_neon.c | 110 +++++++++++++++++++++++++++++++++++++
src/tests/cpu-test.c | 34 ++++++++++++
6 files changed, 193 insertions(+), 1 deletion(-)
create mode 100644 src/pulsecore/sconv_neon.c
diff --git a/configure.ac b/configure.ac
index 2154429b..c91c5720 100644
--- a/configure.ac
+++ b/configure.ac
@@ -317,6 +317,34 @@ case $host in
;;
esac
+#### NEON optimisations ####
+AC_ARG_ENABLE([neon-opt],
+ AS_HELP_STRING([--enable-neon-opt], [Enable NEON optimisations on ARM CPUs that support it]))
+
+AS_IF([test "x$enable_neon_opt" != "xno"],
+ [save_CFLAGS="$CFLAGS"; CFLAGS="$CFLAGS -mfpu=neon"
+ AC_COMPILE_IFELSE(
+ AC_LANG_PROGRAM([], []),
+ [
+ HAVE_NEON=1
+ NEON_CFLAGS="-mfpu=neon"
+ ],
+ [
+ HAVE_NEON=0
+ NEON_CFLAGS=
+ ])
+ CFLAGS="$save_CFLAGS"
+ ],
+ [HAVE_NEON=0])
+
+AS_IF([test "x$enable_neon_opt" = "xyes" && test "x$HAVE_NEON" = "x0"],
+ [AC_MSG_ERROR([*** Compiler does not support -mfpu=neon])])
+
+AC_SUBST(HAVE_NEON)
+AC_SUBST(NEON_CFLAGS)
+AM_CONDITIONAL([HAVE_NEON], [test "x$HAVE_NEON" = x1])
+AS_IF([test "x$HAVE_NEON" = "x1"], AC_DEFINE([HAVE_NEON], 1, [Have NEON support?]))
+
#### libtool stuff ####
diff --git a/src/Makefile.am b/src/Makefile.am
index f80c448a..84598b22 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -813,6 +813,7 @@ libpulsedsp_la_LDFLAGS = $(AM_LDFLAGS) -avoid-version -disable-static
###################################
lib_LTLIBRARIES += libpulsecore-@PA_MAJORMINOR@.la
+noinst_LTLIBRARIES =
# Pure core stuff
libpulsecore_@PA_MAJORMINOR@_la_SOURCES = \
@@ -870,6 +871,13 @@ libpulsecore_@PA_MAJORMINOR@_la_CFLAGS = $(AM_CFLAGS) $(SERVER_CFLAGS) $(LIBSAMP
libpulsecore_@PA_MAJORMINOR@_la_LDFLAGS = $(AM_LDFLAGS) -avoid-version
libpulsecore_@PA_MAJORMINOR@_la_LIBADD = $(AM_LIBADD) $(LIBLTDL) $(LIBSAMPLERATE_LIBS) $(LIBSPEEX_LIBS) $(LIBSNDFILE_LIBS) $(WINSOCK_LIBS) $(LTLIBICONV) libpulsecommon-@PA_MAJORMINOR@.la libpulse.la libpulsecore-foreign.la
+if HAVE_NEON
+noinst_LTLIBRARIES += libpulsecore_sconv_neon.la
+libpulsecore_sconv_neon_la_SOURCES = pulsecore/sconv_neon.c
+libpulsecore_sconv_neon_la_CFLAGS = $(AM_CFLAGS) $(NEON_CFLAGS)
+libpulsecore_@PA_MAJORMINOR@_la_LIBADD += libpulsecore_sconv_neon.la
+endif
+
if HAVE_ORC
ORC_SOURCE += pulsecore/svolume
libpulsecore_@PA_MAJORMINOR@_la_SOURCES += pulsecore/svolume_orc.c
@@ -909,7 +917,7 @@ libpulsecore_@PA_MAJORMINOR@_la_SOURCES += pulsecore/database-simple.c
endif
# We split the foreign code off to not be annoyed by warnings we don't care about
-noinst_LTLIBRARIES = libpulsecore-foreign.la
+noinst_LTLIBRARIES += libpulsecore-foreign.la
libpulsecore_foreign_la_SOURCES = \
pulsecore/ffmpeg/resample2.c pulsecore/ffmpeg/avcodec.h pulsecore/ffmpeg/dsputil.h
diff --git a/src/pulsecore/cpu-arm.c b/src/pulsecore/cpu-arm.c
index 934c7fe9..05668f18 100644
--- a/src/pulsecore/cpu-arm.c
+++ b/src/pulsecore/cpu-arm.c
@@ -142,6 +142,10 @@ pa_bool_t pa_cpu_init_arm(pa_cpu_arm_flag_t *flags) {
if (*flags & PA_CPU_ARM_V6)
pa_volume_func_init_arm(*flags);
+#ifdef HAVE_NEON
+ if (*flags & PA_CPU_ARM_NEON)
+ pa_convert_func_init_neon(*flags);
+#endif
return TRUE;
diff --git a/src/pulsecore/cpu-arm.h b/src/pulsecore/cpu-arm.h
index 91aa659e..d2d3f5c7 100644
--- a/src/pulsecore/cpu-arm.h
+++ b/src/pulsecore/cpu-arm.h
@@ -26,6 +26,10 @@
#include
#include
+#ifndef PACKAGE
+#error "Please include config.h before including this file!"
+#endif
+
typedef enum pa_cpu_arm_flag {
PA_CPU_ARM_V6 = (1 << 0),
PA_CPU_ARM_V7 = (1 << 1),
@@ -41,4 +45,8 @@ pa_bool_t pa_cpu_init_arm(pa_cpu_arm_flag_t *flags);
/* some optimized functions */
void pa_volume_func_init_arm(pa_cpu_arm_flag_t flags);
+#ifdef HAVE_NEON
+void pa_convert_func_init_neon(pa_cpu_arm_flag_t flags);
+#endif
+
#endif /* foocpuarmhfoo */
diff --git a/src/pulsecore/sconv_neon.c b/src/pulsecore/sconv_neon.c
new file mode 100644
index 00000000..81dd97ea
--- /dev/null
+++ b/src/pulsecore/sconv_neon.c
@@ -0,0 +1,110 @@
+/***
+ This file is part of PulseAudio.
+
+ Copyright 2012 Peter Meerwald
+
+ PulseAudio is free software; you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published
+ by the Free Software Foundation; either version 2.1 of the License,
+ or (at your option) any later version.
+
+ PulseAudio is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+***/
+
+#ifdef HAVE_CONFIG_H
+#include
+#endif
+
+#include
+
+#include
+#include
+
+#include "cpu-arm.h"
+#include "sconv.h"
+
+#include
+#include
+
+static void pa_sconv_s16le_from_f32ne_neon(unsigned n, const float *src, int16_t *dst) {
+ unsigned i = n & 3;
+
+ __asm__ __volatile__ (
+ "movs %[n], %[n], lsr #2 \n\t"
+ "beq 2f \n\t"
+
+ "vdup.f32 q2, %[plusone] \n\t"
+ "vneg.f32 q3, q2 \n\t"
+ "vdup.f32 q4, %[scale] \n\t"
+ "vdup.u32 q5, %[mask] \n\t"
+ "vdup.f32 q6, %[half] \n\t"
+
+ "1: \n\t"
+ "vld1.32 {q0}, [%[src]]! \n\t"
+ "vmin.f32 q0, q0, q2 \n\t" /* clamp */
+ "vmax.f32 q0, q0, q3 \n\t"
+ "vmul.f32 q0, q0, q4 \n\t" /* scale */
+ "vand.u32 q1, q0, q5 \n\t"
+ "vorr.u32 q1, q1, q6 \n\t" /* round */
+ "vadd.f32 q0, q0, q1 \n\t"
+ "vcvt.s32.f32 q0, q0 \n\t" /* narrow */
+ "vmovn.i32 d0, q0 \n\t"
+ "subs %[n], %[n], #1 \n\t"
+ "vst1.16 {d0}, [%[dst]]! \n\t"
+ "bgt 1b \n\t"
+
+ "2: \n\t"
+
+ : [dst] "+r" (dst), [src] "+r" (src), [n] "+r" (n) /* output operands (or input operands that get modified) */
+ : [plusone] "r" (1.0f), [scale] "r" (32767.0f), [half] "r" (0.5f), [mask] "r" (0x80000000) /* input operands */
+ : "memory", "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6" /* clobber list */
+ );
+
+ /* leftovers */
+ while (i--) {
+ *dst++ = (int16_t) lrintf(PA_CLAMP_UNLIKELY(*src, -1.0f, 1.0f) * 0x7FFF);
+ src++;
+ }
+}
+
+static void pa_sconv_s16le_to_f32ne_neon(unsigned n, const int16_t *src, float *dst) {
+ unsigned i = n & 3;
+
+ const float invscale = 1.0f / 0x7FFF;
+
+ __asm__ __volatile__ (
+ "movs %[n], %[n], lsr #2 \n\t"
+ "beq 2f \n\t"
+
+ "vdup.f32 q1, %[invscale] \n\t"
+
+ "1: \n\t"
+ "vld1.16 {d0}, [%[src]]! \n\t"
+ "vmovl.s16 q0, d0 \n\t"
+ "vcvt.f32.s32 q0, q0 \n\t"
+ "vmul.f32 q0, q0, q1 \n\t"
+ "subs %[n], %[n], #1 \n\t"
+ "vst1.32 {q0}, [%[dst]]! \n\t"
+ "bgt 1b \n\t"
+
+ "2: \n\t"
+
+ : [dst] "+r" (dst), [src] "+r" (src), [n] "+r" (n) /* output operands (or input operands that get modified) */
+ : [invscale] "r" (invscale) /* input operands */
+ : "memory", "cc", "q0", "q1" /* clobber list */
+ );
+
+ /* leftovers */
+ while (i--) {
+ *dst++ = *src++ * invscale;
+ }
+}
+
+void pa_convert_func_init_neon(pa_cpu_arm_flag_t flags) {
+ pa_log_info("Initialising ARM NEON optimized conversions.");
+ pa_set_convert_from_float32ne_function(PA_SAMPLE_S16LE, (pa_convert_func_t) pa_sconv_s16le_from_f32ne_neon);
+ pa_set_convert_to_float32ne_function(PA_SAMPLE_S16LE, (pa_convert_func_t) pa_sconv_s16le_to_f32ne_neon);
+}
diff --git a/src/tests/cpu-test.c b/src/tests/cpu-test.c
index 5b83d84c..56b0cd7c 100644
--- a/src/tests/cpu-test.c
+++ b/src/tests/cpu-test.c
@@ -282,6 +282,35 @@ START_TEST (sconv_sse_test) {
END_TEST
#endif /* defined (__i386__) || defined (__amd64__) */
+#if defined (__arm__) && defined (__linux__)
+START_TEST (sconv_neon_test) {
+ pa_cpu_arm_flag_t flags = 0;
+ pa_convert_func_t orig_func, neon_func;
+
+ pa_cpu_get_arm_flags(&flags);
+
+ if (!(flags & PA_CPU_ARM_NEON)) {
+ pa_log_info("NEON not supported. Skipping");
+ return;
+ }
+
+ orig_func = pa_get_convert_from_float32ne_function(PA_SAMPLE_S16LE);
+ pa_convert_func_init_neon(flags);
+ neon_func = pa_get_convert_from_float32ne_function(PA_SAMPLE_S16LE);
+
+ pa_log_debug("Checking NEON sconv (s16 -> float)");
+ run_conv_test_float_to_s16(neon_func, orig_func, 0, TRUE, FALSE);
+ run_conv_test_float_to_s16(neon_func, orig_func, 1, TRUE, FALSE);
+ run_conv_test_float_to_s16(neon_func, orig_func, 2, TRUE, FALSE);
+ run_conv_test_float_to_s16(neon_func, orig_func, 3, TRUE, FALSE);
+ run_conv_test_float_to_s16(neon_func, orig_func, 4, TRUE, FALSE);
+ run_conv_test_float_to_s16(neon_func, orig_func, 5, TRUE, FALSE);
+ run_conv_test_float_to_s16(neon_func, orig_func, 6, TRUE, FALSE);
+ run_conv_test_float_to_s16(neon_func, orig_func, 7, TRUE, TRUE);
+}
+END_TEST
+#endif /* defined (__arm__) && defined (__linux__) */
+
#undef SAMPLES
#undef TIMES
/* End conversion tests */
@@ -313,6 +342,11 @@ int main(int argc, char *argv[]) {
tc = tcase_create("sconv");
#if defined (__i386__) || defined (__amd64__)
tcase_add_test(tc, sconv_sse_test);
+#endif
+#if defined (__arm__) && defined (__linux__)
+#if HAVE_NEON
+ tcase_add_test(tc, sconv_neon_test);
+#endif
#endif
suite_add_tcase(s, tc);
--
2.39.2